From ec8ae759c270add73538a5d79f233b8613a5efe2 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 3 May 2022 21:10:55 +1000 Subject: [PATCH 001/135] notebook dev, adds nest coefficient in logit.py --- notebooks/example_dev-Copy1.ipynb | 4295 +++++++++++++++++ notebooks/frozen_rand_mode_choice_dev.ipynb | 4199 ++++++++++++++++ ...ested_logit_frozen_rand_individ_util.ipynb | 463 ++ 3 files changed, 8957 insertions(+) create mode 100644 notebooks/example_dev-Copy1.ipynb create mode 100644 notebooks/frozen_rand_mode_choice_dev.ipynb create mode 100644 notebooks/nested_logit_frozen_rand_individ_util.ipynb diff --git a/notebooks/example_dev-Copy1.ipynb b/notebooks/example_dev-Copy1.ipynb new file mode 100644 index 0000000000..95993575f5 --- /dev/null +++ b/notebooks/example_dev-Copy1.ipynb @@ -0,0 +1,4295 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", + "metadata": {}, + "source": [ + "# Copy of previous dev work\n", + "keep to make sure I can quickly re-trace my steps and understand Asim steps, but keep current dev notebook clean. Should probably start to properly pull out things during dev once they are clear.\n", + "\n", + "Problem below: scale of error term on lower levels needs to be given by nest, not the case atm" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.385153Z", + "iopub.status.busy": "2022-05-01T10:17:57.384881Z", + "iopub.status.idle": "2022-05-01T10:17:57.534433Z", + "shell.execute_reply": "2022-05-01T10:17:57.533096Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3dba451-1e10-403e-8614-35d57e6577f4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.536623Z", + "iopub.status.busy": "2022-05-01T10:17:57.536012Z", + "iopub.status.idle": "2022-05-01T10:17:57.542755Z", + "shell.execute_reply": "2022-05-01T10:17:57.541685Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:00.350944Z", + "iopub.status.busy": "2022-05-01T10:18:00.350730Z", + "iopub.status.idle": "2022-05-01T10:18:12.760977Z", + "shell.execute_reply": "2022-05-01T10:18:12.760013Z", + "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "from datetime import datetime\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from numpy.random import default_rng\n", + "\n", + "\n", + "from activitysim.cli import run\n", + "from activitysim.core import inject\n", + "from activitysim.core import tracing\n", + "from activitysim.core import config\n", + "from activitysim.core import pipeline\n", + "from activitysim.core import mem\n", + "from activitysim.core import chunk\n", + "from activitysim.core import simulate\n", + "from activitysim.core import logit\n", + "from activitysim.abm.models.util.mode import mode_choice_simulate\n", + "from activitysim.abm.models.util import estimation\n", + "from activitysim.core import expressions\n", + "from activitysim.core.util import assign_in_place" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:16.944537Z", + "iopub.status.busy": "2022-05-01T10:18:16.944291Z", + "iopub.status.idle": "2022-05-01T10:18:17.124764Z", + "shell.execute_reply": "2022-05-01T10:18:17.123725Z", + "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "pd.set_option(\"max_columns\", 500)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:17.839947Z", + "iopub.status.busy": "2022-05-01T10:18:17.839070Z", + "iopub.status.idle": "2022-05-01T10:18:18.019676Z", + "shell.execute_reply": "2022-05-01T10:18:18.018689Z", + "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_example_mtc\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:18.894533Z", + "iopub.status.busy": "2022-05-01T10:18:18.894303Z", + "iopub.status.idle": "2022-05-01T10:18:19.078807Z", + "shell.execute_reply": "2022-05-01T10:18:19.077951Z", + "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "os.chdir(example_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:22.111723Z", + "iopub.status.busy": "2022-05-01T10:18:22.111490Z", + "iopub.status.idle": "2022-05-01T10:18:22.297437Z", + "shell.execute_reply": "2022-05-01T10:18:22.296501Z", + "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "run.add_run_args(parser)\n", + "# args = parser.parse_args()\n", + "# parser.parse_args(['--sum', '7', '-1', '42'])\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", + "#run.run(args) # 10mins full example run" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "7c2e2f00-f396-4016-8545-dd959f232468", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:23.454776Z", + "iopub.status.busy": "2022-05-01T10:18:23.454538Z", + "iopub.status.idle": "2022-05-01T10:18:23.637711Z", + "shell.execute_reply": "2022-05-01T10:18:23.636943Z", + "shell.execute_reply.started": "2022-05-01T10:18:23.454751Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "if not inject.is_injectable('preload_injectables'):\n", + " from activitysim import abm # register abm steps and other abm-specific injectables\n", + "\n", + "run.handle_standard_args(args) # possibly update injectables\n", + "\n", + "resume_after = \"trip_scheduling\"\n", + "\n", + "\n", + "#####\n", + "## rest below replaces the next two lines\n", + "## pipeline.run(models=config.setting('models'), resume_after=resume_after)\n", + "## pipeline.close_pipeline()\n", + "#####\n", + "\n", + "# pipeline.open_pipeline(resume_after)\n", + "\n", + "# # preload any bulky injectables (e.g. skims) not in pipeline\n", + "# inject.get_injectable('preload_injectables', None)\n", + "\n", + "# #pipeline.run_model(resume_after) # from here on\n", + "# model_name = \"trip_mode_choice\"\n", + "# pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "# step_name = model_name\n", + "# args = {}\n", + "\n", + "# checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "\n", + "# inject.set_step_args(args)\n", + "\n", + "# pipeline.orca.run([step_name])\n", + "\n", + "# inject.set_step_args(None)\n", + "\n", + "# pipeline._PIPELINE.rng().end_step(model_name)\n", + "# pipeline.add_checkpoint(model_name)\n", + "\n", + "# # add checkpoint with final tables even if not intermediate checkpointing\n", + "# if not pipeline.intermediate_checkpoint():\n", + "# pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + "# pipeline.close_pipeline()" + ] + }, + { + "cell_type": "markdown", + "id": "6045438f-d1eb-4b5c-9737-185798b7f97f", + "metadata": {}, + "source": [ + "## trip mode choice by hand" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "600ae244-3e6c-4b66-8d39-aa9f6f60b378", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:26.399145Z", + "iopub.status.busy": "2022-05-01T10:18:26.398687Z", + "iopub.status.idle": "2022-05-01T10:18:26.583256Z", + "shell.execute_reply": "2022-05-01T10:18:26.582233Z", + "shell.execute_reply.started": "2022-05-01T10:18:26.399105Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#inject.get_table('trips').to_frame() #pipeline.orca.get_raw_table('trips').to_frame()\n", + "#inject.get_table('tours_merged').to_frame() #pipeline.orca.get_raw_table('tours_merged').to_frame()\n", + "#inject.get_injectable('network_los')" + ] + }, + { + "cell_type": "code", + "execution_count": 373, + "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:28:53.643404Z", + "iopub.status.busy": "2021-09-01T02:28:53.643055Z", + "iopub.status.idle": "2021-09-01T02:29:01.223951Z", + "shell.execute_reply": "2021-09-01T02:29:01.222865Z", + "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n", + "trip_mode_choice tour_type 'eatout' (33 trips)\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n", + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", + "trip_mode_choice tour_type 'othmaint' (46 trips)\n", + "trip_mode_choice tour_type 'school' (37 trips)\n", + "trip_mode_choice tour_type 'shopping' (77 trips)\n", + "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'univ' (26 trips)\n", + "trip_mode_choice tour_type 'work' (168 trips)\n", + "Done\n" + ] + } + ], + "source": [ + "pipeline.open_pipeline(resume_after)\n", + "\n", + "# preload any bulky injectables (e.g. skims) not in pipeline\n", + "inject.get_injectable('preload_injectables', None)\n", + "\n", + "#pipeline.run_model(resume_after) # from here on\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "#####\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "\n", + "trips_df = trips.to_frame()\n", + "print(\"Running with %d trips\", trips_df.shape[0])\n", + "\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "\n", + "\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "\n", + "\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "\n", + "\n", + "\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "\n", + "do_these_purposes = None #['escort']\n", + "\n", + "choices_list = []\n", + "for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + "\n", + " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", + " continue\n", + "\n", + " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " (primary_purpose, len(trips_segment.index), ))\n", + "\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + " \n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + "################ Replace wrapper function\n", + "# choices = mode_choice_simulate(\n", + "# choosers=trips_segment,\n", + "# spec=simulate.eval_coefficients(model_spec, coefficients, estimator),\n", + "# nest_spec=simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label),\n", + "# skims=skims,\n", + "# locals_d=locals_dict,\n", + "# chunk_size=chunk_size,\n", + "# mode_column_name=mode_column_name,\n", + "# logsum_column_name=logsum_column_name,\n", + "# trace_label=segment_trace_label,\n", + "# trace_choice_name='trip_mode_choice',\n", + "# estimator=estimator)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simulate.simple_simulate(\n", + " choosers=trips_segment,\n", + " spec=spec,\n", + " nest_spec=nest_spec,\n", + " skims=skims,\n", + " locals_d=locals_dict,\n", + " chunk_size=chunk_size,\n", + " want_logsums=logsum_column_name is not None,\n", + " trace_label=segment_trace_label,\n", + " trace_choice_name='trip_mode_choice',\n", + " estimator=estimator,\n", + " trace_column_names=None)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + "\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + "\n", + " alts = spec.columns\n", + " choices[mode_column_name] = \\\n", + " choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", + "################\n", + "\n", + "\n", + " choices_list.append(choices)\n", + "\n", + "choices_df_asim = pd.concat(choices_list)\n", + "\n", + "# update trips table with choices (and potionally logssums)\n", + "trips_df = trips.to_frame()\n", + "\n", + "if (do_these_purposes is not None):\n", + " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", + "\n", + "assign_in_place(trips_df, choices_df)\n", + "assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(\"Done\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "7eae2854-38bd-4da0-a161-3a15bbd17177", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:30:57.562848Z", + "iopub.status.busy": "2021-08-31T10:30:57.562615Z", + "iopub.status.idle": "2021-08-31T10:30:57.806655Z", + "shell.execute_reply": "2021-08-31T10:30:57.805135Z", + "shell.execute_reply.started": "2021-08-31T10:30:57.562824Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
person_idhousehold_idtour_idprimary_purposetrip_numoutboundtrip_countpurposedestinationorigindestination_logsumdeparttrip_modemode_choice_logsum
trip_id
13724872141844130403617156090escort1True1escort710NaN7.0WALK11.435800
13724872541844130403617156090escort1False1Home107NaN7.0WALK11.480440
21138820164447638676126423525escort1True1escort1116NaN5.0WALK_LOC4.789158
21138820564447638676126423525escort1False1Home1611NaN6.0WALK_LOC5.050171
80638840124585011173905100798550escort1True1escort168NaN15.0WALK_LOC6.451457
80638840524585011173905100798550escort1False1Home816NaN16.0WALK_LOC6.446188
\n", + "
" + ], + "text/plain": [ + " person_id household_id tour_id primary_purpose trip_num \\\n", + "trip_id \n", + "137248721 418441 304036 17156090 escort 1 \n", + "137248725 418441 304036 17156090 escort 1 \n", + "211388201 644476 386761 26423525 escort 1 \n", + "211388205 644476 386761 26423525 escort 1 \n", + "806388401 2458501 1173905 100798550 escort 1 \n", + "806388405 2458501 1173905 100798550 escort 1 \n", + "\n", + " outbound trip_count purpose destination origin \\\n", + "trip_id \n", + "137248721 True 1 escort 7 10 \n", + "137248725 False 1 Home 10 7 \n", + "211388201 True 1 escort 11 16 \n", + "211388205 False 1 Home 16 11 \n", + "806388401 True 1 escort 16 8 \n", + "806388405 False 1 Home 8 16 \n", + "\n", + " destination_logsum depart trip_mode mode_choice_logsum \n", + "trip_id \n", + "137248721 NaN 7.0 WALK 11.435800 \n", + "137248725 NaN 7.0 WALK 11.480440 \n", + "211388201 NaN 5.0 WALK_LOC 4.789158 \n", + "211388205 NaN 6.0 WALK_LOC 5.050171 \n", + "806388401 NaN 15.0 WALK_LOC 6.451457 \n", + "806388405 NaN 16.0 WALK_LOC 6.446188 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca3206ce-df97-42d9-83ce-dc4067ea5675", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6c2752e-68a8-4319-85c2-fe6bf1fd6d59", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3851df2c-bb8c-43b6-8a27-3b04a7f85a50", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b596af94-85d1-4d25-99b2-c19292397882", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "8eebea31-499c-41ca-8411-883a88ca800a", + "metadata": {}, + "source": [ + "## nested dev" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:01.113060Z", + "iopub.status.busy": "2021-08-31T10:31:01.112829Z", + "iopub.status.idle": "2021-08-31T10:31:01.359912Z", + "shell.execute_reply": "2021-08-31T10:31:01.358948Z", + "shell.execute_reply.started": "2021-08-31T10:31:01.113034Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "spec = simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + "nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "36c100b3-3e39-4950-a586-4d42be695eaa", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:01.775068Z", + "iopub.status.busy": "2021-08-31T10:31:01.774846Z", + "iopub.status.idle": "2021-08-31T10:31:01.975816Z", + "shell.execute_reply": "2021-08-31T10:31:01.974499Z", + "shell.execute_reply.started": "2021-08-31T10:31:01.775044Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'root',\n", + " 'coefficient': 1.0,\n", + " 'alternatives': [{'name': 'AUTO',\n", + " 'coefficient': 0.72,\n", + " 'alternatives': [{'name': 'DRIVEALONE',\n", + " 'coefficient': 0.35,\n", + " 'alternatives': ['DRIVEALONEFREE', 'DRIVEALONEPAY']},\n", + " {'name': 'SHAREDRIDE2',\n", + " 'coefficient': 0.35,\n", + " 'alternatives': ['SHARED2FREE', 'SHARED2PAY']},\n", + " {'name': 'SHAREDRIDE3',\n", + " 'coefficient': 0.35,\n", + " 'alternatives': ['SHARED3FREE', 'SHARED3PAY']}]},\n", + " {'name': 'NONMOTORIZED',\n", + " 'coefficient': 0.72,\n", + " 'alternatives': ['WALK', 'BIKE']},\n", + " {'name': 'TRANSIT',\n", + " 'coefficient': 0.72,\n", + " 'alternatives': [{'name': 'WALKACCESS',\n", + " 'coefficient': 0.5,\n", + " 'alternatives': ['WALK_LOC',\n", + " 'WALK_LRF',\n", + " 'WALK_EXP',\n", + " 'WALK_HVY',\n", + " 'WALK_COM']},\n", + " {'name': 'DRIVEACCESS',\n", + " 'coefficient': 0.5,\n", + " 'alternatives': ['DRIVE_LOC',\n", + " 'DRIVE_LRF',\n", + " 'DRIVE_EXP',\n", + " 'DRIVE_HVY',\n", + " 'DRIVE_COM']}]},\n", + " {'name': 'RIDEHAIL',\n", + " 'coefficient': 0.36,\n", + " 'alternatives': ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']}]}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nest_spec" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "f442cf50-e85c-4991-af39-68a6839b146e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:03.459794Z", + "iopub.status.busy": "2021-08-31T10:31:03.459560Z", + "iopub.status.idle": "2021-08-31T10:31:03.660759Z", + "shell.execute_reply": "2021-08-31T10:31:03.658857Z", + "shell.execute_reply.started": "2021-08-31T10:31:03.459767Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] alternatives: ['AUTO', 'NONMOTORIZED', 'TRANSIT', 'RIDEHAIL']\n", + "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] alternatives: ['DRIVEALONE', 'SHAREDRIDE2', 'SHAREDRIDE3']\n", + "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] alternatives: ['DRIVEALONEFREE', 'DRIVEALONEPAY']\n", + "Nest name: DRIVEALONEFREE level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE'] alternatives: None\n", + "Nest name: DRIVEALONEPAY level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY'] alternatives: None\n", + "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2'] alternatives: ['SHARED2FREE', 'SHARED2PAY']\n", + "Nest name: SHARED2FREE level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2FREE'] alternatives: None\n", + "Nest name: SHARED2PAY level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2PAY'] alternatives: None\n", + "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3'] alternatives: ['SHARED3FREE', 'SHARED3PAY']\n", + "Nest name: SHARED3FREE level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3FREE'] alternatives: None\n", + "Nest name: SHARED3PAY level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3PAY'] alternatives: None\n", + "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] alternatives: ['WALK', 'BIKE']\n", + "Nest name: WALK level: 3 coefficient: 0 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'WALK'] alternatives: None\n", + "Nest name: BIKE level: 3 coefficient: 0 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'BIKE'] alternatives: None\n", + "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] alternatives: ['WALKACCESS', 'DRIVEACCESS']\n", + "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS'] alternatives: ['WALK_LOC', 'WALK_LRF', 'WALK_EXP', 'WALK_HVY', 'WALK_COM']\n", + "Nest name: WALK_LOC level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LOC'] alternatives: None\n", + "Nest name: WALK_LRF level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LRF'] alternatives: None\n", + "Nest name: WALK_EXP level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_EXP'] alternatives: None\n", + "Nest name: WALK_HVY level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_HVY'] alternatives: None\n", + "Nest name: WALK_COM level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_COM'] alternatives: None\n", + "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS'] alternatives: ['DRIVE_LOC', 'DRIVE_LRF', 'DRIVE_EXP', 'DRIVE_HVY', 'DRIVE_COM']\n", + "Nest name: DRIVE_LOC level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LOC'] alternatives: None\n", + "Nest name: DRIVE_LRF level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LRF'] alternatives: None\n", + "Nest name: DRIVE_EXP level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_EXP'] alternatives: None\n", + "Nest name: DRIVE_HVY level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_HVY'] alternatives: None\n", + "Nest name: DRIVE_COM level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_COM'] alternatives: None\n", + "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] alternatives: ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']\n", + "Nest name: TAXI level: 3 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TAXI'] alternatives: None\n", + "Nest name: TNC_SINGLE level: 3 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SINGLE'] alternatives: None\n", + "Nest name: TNC_SHARED level: 3 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SHARED'] alternatives: None\n" + ] + } + ], + "source": [ + "for nest in logit.each_nest(nest_spec):\n", + " nest.print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d509b34d-a755-48ca-8079-386c14023d69", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 349, + "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:59:35.109079Z", + "iopub.status.busy": "2021-09-01T01:59:35.108781Z", + "iopub.status.idle": "2021-09-01T01:59:35.319301Z", + "shell.execute_reply": "2021-09-01T01:59:35.318402Z", + "shell.execute_reply.started": "2021-09-01T01:59:35.109050Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_dev(choosers, spec, nest_spec,\n", + " skims=None, locals_d=None,\n", + " chunk_size=0, custom_chooser=None,\n", + " log_alt_losers=False,\n", + " want_logsums=False,\n", + " estimator=None,\n", + " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", + " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", + " assert len(choosers) > 0\n", + " result_list = []\n", + " # segment by person type and pick the right spec for each person type\n", + " for i, chooser_chunk, chunk_trace_label \\\n", + " in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", + " # the following replaces choices = _simple_simulate(...)\n", + " if skims is not None:\n", + " simulate.set_skim_wrapper_targets(choosers, skims)\n", + "\n", + " # only do this for nested, logit is straight forward\n", + " assert nest_spec is not None\n", + " choices = simulate.eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser,\n", + " log_alt_losers=log_alt_losers,\n", + " want_logsums=want_logsums,\n", + " estimator=estimator,\n", + " trace_label=trace_label,\n", + " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", + "\n", + "\n", + " result_list.append(choices)\n", + " chunk.log_df(trace_label, f'result_list', result_list)\n", + "\n", + " if len(result_list) > 1:\n", + " choices = pd.concat(result_list)\n", + " assert len(choices.index == len(choosers.index))\n", + " return choices\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "05e06aa4-e943-4c34-ad1b-09c19c800bdc", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:35:06.100477Z", + "iopub.status.busy": "2021-08-31T10:35:06.100054Z", + "iopub.status.idle": "2021-08-31T10:35:06.974578Z", + "shell.execute_reply": "2021-08-31T10:35:06.973620Z", + "shell.execute_reply.started": "2021-08-31T10:35:06.100438Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "primary_purpose = 'escort'\n", + "trips_segment = trips_merged.loc[trips_merged.primary_purpose == primary_purpose].copy()\n", + "assert trips_segment.index.name == 'trip_id' # name index so tracing knows how to slice\n", + "coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "locals_dict = {}\n", + "locals_dict.update(constants)\n", + "locals_dict.update(coefficients)\n", + "segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + "locals_dict.update(skims)\n", + "\n", + "spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + "nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + "choices = simple_simulate_dev(\n", + " choosers=trips_segment,\n", + " spec=spec,\n", + " nest_spec=nest_spec,\n", + " skims=skims,\n", + " locals_d=locals_dict,\n", + " chunk_size=chunk_size,\n", + " want_logsums=logsum_column_name is not None,\n", + " trace_label=segment_trace_label,\n", + " trace_choice_name='trip_mode_choice',\n", + " estimator=estimator,\n", + " trace_column_names=None)\n", + "# for consistency, always return dataframe, whether or not logsums were requested\n", + "if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + "\n", + "choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + "\n", + "alts = spec.columns\n", + "choices[mode_column_name] = \\\n", + " choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07ef8fbc-beb2-4e57-9d56-939719026d58", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "c60656db-075a-4eab-96b3-fc54eb785075", + "metadata": {}, + "source": [ + "## eval_nl dev" + ] + }, + { + "cell_type": "code", + "execution_count": 372, + "id": "56c8923c-e946-48d8-b964-87695c30163d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:27:40.290389Z", + "iopub.status.busy": "2021-09-01T02:27:40.290139Z", + "iopub.status.idle": "2021-09-01T02:27:40.977526Z", + "shell.execute_reply": "2021-09-01T02:27:40.976594Z", + "shell.execute_reply.started": "2021-09-01T02:27:40.290363Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# replaces simple_simulate_dev:\n", + "\n", + "want_logsums = True\n", + "trace_choice_name='trip_mode_choice'\n", + "trace_column_names=None\n", + "log_alt_losers=False\n", + "\n", + "tr_label = tracing.extend_trace_label(segment_trace_label, 'simple_simulate')\n", + "assert len(trips_segment) > 0\n", + "result_list = []\n", + "# segment by person type and pick the right spec for each person type\n", + "for i, chooser_chunk, chunk_trace_label \\\n", + " in chunk.adaptive_chunked_choosers(trips_segment, chunk_size, tr_label):\n", + " # the following replaces choices = _simple_simulate(...)\n", + " if skims is not None:\n", + " simulate.set_skim_wrapper_targets(trips_segment, skims)\n", + "\n", + " # only do this for nested, logit is straight forward\n", + " assert nest_spec is not None\n", + " choices = simulate.eval_nl(trips_segment, spec, nest_spec, locals_dict, custom_chooser=None,\n", + " log_alt_losers=log_alt_losers,\n", + " want_logsums=want_logsums,\n", + " estimator=estimator,\n", + " trace_label=tr_label,\n", + " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", + "\n", + "\n", + " result_list.append(choices)\n", + "\n", + "if len(result_list) > 1:\n", + " choices = pd.concat(result_list)\n", + "assert len(choices.index == len(trips_segment.index))" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "id": "cbc2dfde-1bd9-4b03-96d9-ca9b7222d36c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T12:52:34.544865Z", + "iopub.status.busy": "2021-08-31T12:52:34.544586Z", + "iopub.status.idle": "2021-08-31T12:52:35.159615Z", + "shell.execute_reply": "2021-08-31T12:52:35.158682Z", + "shell.execute_reply.started": "2021-08-31T12:52:34.544840Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "locals_d = locals_dict\n", + "custom_chooser=None\n", + "\n", + "trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + "logit.validate_nest_spec(nest_spec, trace_label)\n", + "\n", + "raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + "nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + "\n", + "nested_probabilities = \\\n", + " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", + "\n", + "logsums = pd.Series(np.log(nested_exp_utilities.root), index=trips_segment.index)\n", + "\n", + "# global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + "base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", + "\n", + "choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "id": "0cd9593b-b3fd-41f6-a7ca-4c5b73b63571", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T12:52:45.509057Z", + "iopub.status.busy": "2021-08-31T12:52:45.508819Z", + "iopub.status.idle": "2021-08-31T12:52:45.721956Z", + "shell.execute_reply": "2021-08-31T12:52:45.721064Z", + "shell.execute_reply.started": "2021-08-31T12:52:45.509021Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 0.990308\n", + "137248725 0.108433\n", + "211388201 0.806612\n", + "211388205 0.195994\n", + "806388401 0.854020\n", + "806388405 0.660550\n", + "dtype: float64" + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rands" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "id": "b467330b-f1cc-43bc-9325-c6dfeaa9b364", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T12:50:51.735273Z", + "iopub.status.busy": "2021-08-31T12:50:51.734962Z", + "iopub.status.idle": "2021-08-31T12:50:52.023440Z", + "shell.execute_reply": "2021-08-31T12:50:52.022582Z", + "shell.execute_reply.started": "2021-08-31T12:50:51.735231Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.798480-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.360270-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.435800
137248725-1.732971-3966.018685-0.606540-0.324578-3964.610293-0.113602-3966.009714-7930.295428-inf0.26146615.945036-1368.16910311.480426-2737.292402-5512.292402-5512.292402-5512.292402-5512.292402-inf-2737.782457-5522.136798-5522.136798-5516.076156-5525.080218-inf-inf-24.341318-25.104291-24.140831-8.40687011.480440
211388201-3966.000225-7930.285939-inf-24.120135-3988.405850-8.442047-29.480718-3993.766432-10.318251-5.975663-4.216264-1381.397292-3.03571013.295720-2761.150907-2761.150907-2761.150907-2761.1509076.647860-2761.648763-5543.048131-5543.048131-2769.692185-2773.867648-inf4.786459-15.861515-3.864497-5.068360-1.2967574.789158
211388205-3967.791945-7932.077659-inf-25.129596-3989.415310-8.795359-30.177275-3994.462989-10.562046-6.219063-4.216264-1381.397292-3.03571014.022438-2760.977562-2760.977562-2760.977562-2760.9775627.011219-2761.294814-5546.132585-5546.132585-2764.829835-2774.132982-inf5.048078-15.776424-3.845077-5.005482-1.2860945.050171
806388401-3966.492870-7930.778584-inf-12.618378-3976.904092-4.416432-14.908422-3979.194136-5.217948-2.9129776.192499-1378.4807234.45859917.513379-2755.845870-2755.208370-2755.728926-2755.7930938.756690-2757.559519-5540.685990-5540.685990-5542.838278-5546.780761-inf6.304817-19.143696-19.495908-16.361074-5.8539176.451457
806388405-3969.835821-7934.121535-inf-14.515916-3978.801631-5.080571-16.227796-3980.513510-5.679728-3.3428055.998749-1378.5194734.31909917.553369-2755.987742-2755.350242-2755.870797-2755.9349648.776685-2757.571454-5542.673358-5542.673358-5537.799228-5547.515590-inf6.319213-18.886576-19.495133-16.300395-5.8285406.446188
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", + "trip_id \n", + "137248721 -1.798480 -3966.084195 -0.629468 -0.389611 \n", + "137248725 -1.732971 -3966.018685 -0.606540 -0.324578 \n", + "211388201 -3966.000225 -7930.285939 -inf -24.120135 \n", + "211388205 -3967.791945 -7932.077659 -inf -25.129596 \n", + "806388401 -3966.492870 -7930.778584 -inf -12.618378 \n", + "806388405 -3969.835821 -7934.121535 -inf -14.515916 \n", + "\n", + " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", + "trip_id \n", + "137248721 -3964.675326 -0.136364 -3966.074556 -7930.360270 -inf \n", + "137248725 -3964.610293 -0.113602 -3966.009714 -7930.295428 -inf \n", + "211388201 -3988.405850 -8.442047 -29.480718 -3993.766432 -10.318251 \n", + "211388205 -3989.415310 -8.795359 -30.177275 -3994.462989 -10.562046 \n", + "806388401 -3976.904092 -4.416432 -14.908422 -3979.194136 -5.217948 \n", + "806388405 -3978.801631 -5.080571 -16.227796 -3980.513510 -5.679728 \n", + "\n", + " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", + "trip_id \n", + "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", + "137248725 0.261466 15.945036 -1368.169103 11.480426 -2737.292402 \n", + "211388201 -5.975663 -4.216264 -1381.397292 -3.035710 13.295720 \n", + "211388205 -6.219063 -4.216264 -1381.397292 -3.035710 14.022438 \n", + "806388401 -2.912977 6.192499 -1378.480723 4.458599 17.513379 \n", + "806388405 -3.342805 5.998749 -1378.519473 4.319099 17.553369 \n", + "\n", + " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", + "trip_id \n", + "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", + "137248725 -5512.292402 -5512.292402 -5512.292402 -5512.292402 -inf \n", + "211388201 -2761.150907 -2761.150907 -2761.150907 -2761.150907 6.647860 \n", + "211388205 -2760.977562 -2760.977562 -2760.977562 -2760.977562 7.011219 \n", + "806388401 -2755.845870 -2755.208370 -2755.728926 -2755.793093 8.756690 \n", + "806388405 -2755.987742 -2755.350242 -2755.870797 -2755.934964 8.776685 \n", + "\n", + " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", + "trip_id \n", + "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", + "137248725 -2737.782457 -5522.136798 -5522.136798 -5516.076156 -5525.080218 \n", + "211388201 -2761.648763 -5543.048131 -5543.048131 -2769.692185 -2773.867648 \n", + "211388205 -2761.294814 -5546.132585 -5546.132585 -2764.829835 -2774.132982 \n", + "806388401 -2757.559519 -5540.685990 -5540.685990 -5542.838278 -5546.780761 \n", + "806388405 -2757.571454 -5542.673358 -5542.673358 -5537.799228 -5547.515590 \n", + "\n", + " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", + "trip_id \n", + "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", + "137248725 -inf -inf -24.341318 -25.104291 -24.140831 -8.406870 \n", + "211388201 -inf 4.786459 -15.861515 -3.864497 -5.068360 -1.296757 \n", + "211388205 -inf 5.048078 -15.776424 -3.845077 -5.005482 -1.286094 \n", + "806388401 -inf 6.304817 -19.143696 -19.495908 -16.361074 -5.853917 \n", + "806388405 -inf 6.319213 -18.886576 -19.495133 -16.300395 -5.828540 \n", + "\n", + " root \n", + "trip_id \n", + "137248721 11.435800 \n", + "137248725 11.480440 \n", + "211388201 4.789158 \n", + "211388205 5.050171 \n", + "806388401 6.451457 \n", + "806388405 6.446188 " + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# root is logsum here, no need to take log again as for exponentiated utils\n", + "simulate.compute_nested_exp_utilities(raw_utilities, nest_spec, exponentiate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "5c9ba018-167d-497f-ac65-f7c49036ee58", + "metadata": {}, + "source": [ + "#### scale of utils, including emus" + ] + }, + { + "cell_type": "code", + "execution_count": 283, + "id": "9febf6ef-7c6b-4f3c-bdd9-aa3326a82e95", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:23:00.020038Z", + "iopub.status.busy": "2021-08-31T23:23:00.019377Z", + "iopub.status.idle": "2021-08-31T23:23:00.220889Z", + "shell.execute_reply": "2021-08-31T23:23:00.220154Z", + "shell.execute_reply.started": "2021-08-31T23:23:00.019997Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "nests_ = list(logit.each_nest(nest_spec))" + ] + }, + { + "cell_type": "code", + "execution_count": 284, + "id": "48ff920e-0c9b-4b69-9cee-eb3c3427bd4f", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:23:01.085713Z", + "iopub.status.busy": "2021-08-31T23:23:01.085486Z", + "iopub.status.idle": "2021-08-31T23:23:01.283509Z", + "shell.execute_reply": "2021-08-31T23:23:01.282444Z", + "shell.execute_reply.started": "2021-08-31T23:23:01.085685Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] alternatives: ['DRIVEALONE', 'SHAREDRIDE2', 'SHAREDRIDE3']\n", + "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] alternatives: ['DRIVEALONEFREE', 'DRIVEALONEPAY']\n", + "Nest name: DRIVEALONEFREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE'] alternatives: None\n", + "Nest name: DRIVEALONEPAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY'] alternatives: None\n" + ] + } + ], + "source": [ + "#list(map(lambda x: x.print(), nests_));\n", + "# have a look at \n", + "nests_[1].print()\n", + "nests_[2].print()\n", + "nests_[3].print()\n", + "nests_[4].print()" + ] + }, + { + "cell_type": "code", + "execution_count": 312, + "id": "7b7e453d-da45-4751-8889-561a349bd373", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:22.371464Z", + "iopub.status.busy": "2021-09-01T01:11:22.371192Z", + "iopub.status.idle": "2021-09-01T01:11:22.593424Z", + "shell.execute_reply": "2021-09-01T01:11:22.592472Z", + "shell.execute_reply.started": "2021-09-01T01:11:22.371415Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation belowi s equivalent.\n", + "def compute_nested_utilities(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) / nest.product_of_coefficients #coefficient\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", + " return nested_utilities\n", + "\n", + "\n", + "utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", + "\n", + "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", + "seed = 9326543345\n", + "rng = default_rng(seed=seed)\n", + "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", + "rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", + "\n", + "probs_arr = utils_df - np.log(-np.log(rands))\n", + "\n", + "# Now replace the following with recursive nested choice\n", + "# choices = np.argmax(probs_arr, axis=1)\n", + "# choices = pd.Series(choices, index=utils_df.index)\n", + "# rands = pd.Series(np.asanyarray(rands).flatten(), index=utils_df.index)" + ] + }, + { + "cell_type": "code", + "execution_count": 313, + "id": "39b71df9-12ac-4d9c-8134-43d22f7e6a6f", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:23.703299Z", + "iopub.status.busy": "2021-09-01T01:11:23.703088Z", + "iopub.status.idle": "2021-09-01T01:11:23.937077Z", + "shell.execute_reply": "2021-09-01T01:11:23.935627Z", + "shell.execute_reply.started": "2021-09-01T01:11:23.703274Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.79848-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.36027-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.4358
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", + "trip_id \n", + "137248721 -1.79848 -3966.084195 -0.629468 -0.389611 \n", + "\n", + " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", + "trip_id \n", + "137248721 -3964.675326 -0.136364 -3966.074556 -7930.36027 -inf \n", + "\n", + " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", + "trip_id \n", + "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", + "\n", + " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", + "trip_id \n", + "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", + "\n", + " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", + "trip_id \n", + "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", + "\n", + " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", + "trip_id \n", + "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", + "\n", + " root \n", + "trip_id \n", + "137248721 11.4358 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-0.717963-3964.2474012.7217010.230429-3964.716506-0.274584-3965.126773-7930.610173-inf1.73337615.473633-1368.52552614.717027-2736.517471-5510.738449-5512.038516-5512.559241-5511.584293-inf-2738.143234-5518.606969-5519.009587-5519.830735-5521.525966-inf-inf-25.519792-21.698963-25.267185-7.44465211.689443
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", + "trip_id \n", + "137248721 -0.717963 -3964.247401 2.721701 0.230429 \n", + "\n", + " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", + "trip_id \n", + "137248721 -3964.716506 -0.274584 -3965.126773 -7930.610173 -inf \n", + "\n", + " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", + "trip_id \n", + "137248721 1.733376 15.473633 -1368.525526 14.717027 -2736.517471 \n", + "\n", + " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", + "trip_id \n", + "137248721 -5510.738449 -5512.038516 -5512.559241 -5511.584293 -inf \n", + "\n", + " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", + "trip_id \n", + "137248721 -2738.143234 -5518.606969 -5519.009587 -5519.830735 -5521.525966 \n", + "\n", + " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", + "trip_id \n", + "137248721 -inf -inf -25.519792 -21.698963 -25.267185 -7.444652 \n", + "\n", + " root \n", + "trip_id \n", + "137248721 11.689443 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(utils_df.head(1), probs_arr.head(1))" + ] + }, + { + "cell_type": "markdown", + "id": "2f080150-c15a-4059-8a26-42a8c0072606", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:16:40.014024Z", + "iopub.status.busy": "2021-08-31T23:16:40.013770Z", + "iopub.status.idle": "2021-08-31T23:16:40.236326Z", + "shell.execute_reply": "2021-08-31T23:16:40.235553Z", + "shell.execute_reply.started": "2021-08-31T23:16:40.013973Z" + }, + "tags": [] + }, + "source": [ + "### make choice at each level" + ] + }, + { + "cell_type": "code", + "execution_count": 315, + "id": "2ed71b54-67e4-4087-957a-75f4fa184144", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:43.778081Z", + "iopub.status.busy": "2021-09-01T01:11:43.777757Z", + "iopub.status.idle": "2021-09-01T01:11:43.975332Z", + "shell.execute_reply": "2021-09-01T01:11:43.974588Z", + "shell.execute_reply.started": "2021-09-01T01:11:43.778055Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_alternatives(nests, name):\n", + " alts = list(filter(lambda x: x.name == name, nests))\n", + " assert len(alts) == 1, f\"{len(alts)} not one\"\n", + " alts = alts[0].alternatives\n", + " return alts\n", + "\n", + "def recursive_choice(row, columns, nest_levels, nests):\n", + " choices = row[columns].idxmax() #axis=1).values[0]\n", + " next_level_columns = get_alternatives(nests, choices)\n", + " #print(f\"{choices} leads to columns {next_level_columns}\")\n", + " if next_level_columns is None:\n", + " return choices \n", + " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", + " return new_choice" + ] + }, + { + "cell_type": "code", + "execution_count": 316, + "id": "30424562-2627-446b-abd5-7c763c52060e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:49.444289Z", + "iopub.status.busy": "2021-09-01T01:11:49.444002Z", + "iopub.status.idle": "2021-09-01T01:11:49.664214Z", + "shell.execute_reply": "2021-09-01T01:11:49.663088Z", + "shell.execute_reply.started": "2021-09-01T01:11:49.444266Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n" + ] + } + ], + "source": [ + "# group nests by level:\n", + "depth = np.max([x.level for x in nests_])\n", + "print(depth)\n", + "nest_levels = {x: [] for x in range(1, depth+1)}\n", + "for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + "assert len(nest_levels[1]) == 1\n", + "assert nest_levels[1][0] == 'root'" + ] + }, + { + "cell_type": "code", + "execution_count": 326, + "id": "bae6e7a8-d7c3-42a2-a82f-874b1e59a0b6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:56:38.377472Z", + "iopub.status.busy": "2021-09-01T01:56:38.377253Z", + "iopub.status.idle": "2021-09-01T01:56:38.607858Z", + "shell.execute_reply": "2021-09-01T01:56:38.606913Z", + "shell.execute_reply.started": "2021-09-01T01:56:38.377447Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 WALK\n", + "137248725 WALK\n", + "211388201 WALK_LOC\n", + "211388205 WALK_LOC\n", + "806388401 WALK\n", + "806388405 WALK\n", + "dtype: object" + ] + }, + "execution_count": 326, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests_), axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 321, + "id": "fc315f63-542f-46fd-8d70-b7a1fcd492c6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:53:15.500933Z", + "iopub.status.busy": "2021-09-01T01:53:15.500709Z", + "iopub.status.idle": "2021-09-01T01:53:15.703201Z", + "shell.execute_reply": "2021-09-01T01:53:15.702357Z", + "shell.execute_reply.started": "2021-09-01T01:53:15.500909Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#display(utils_df, probs_arr)" + ] + }, + { + "cell_type": "markdown", + "id": "8a45e1c0-0a32-4f7a-8803-7559d71abefd", + "metadata": {}, + "source": [ + "### some tests\n", + "seeds, average?" + ] + }, + { + "cell_type": "code", + "execution_count": 333, + "id": "f0e00814-2a14-4da1-bc6c-89b52b7fb286", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:58:26.002425Z", + "iopub.status.busy": "2021-09-01T01:58:26.002200Z", + "iopub.status.idle": "2021-09-01T01:58:26.247954Z", + "shell.execute_reply": "2021-09-01T01:58:26.247054Z", + "shell.execute_reply.started": "2021-09-01T01:58:26.002401Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 WALK\n", + "137248725 WALK\n", + "211388201 WALK_LOC\n", + "211388205 WALK_LOC\n", + "806388401 WALK\n", + "806388405 WALK\n", + "dtype: object" + ] + }, + "execution_count": 333, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", + "seed = 9326543345\n", + "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", + "\n", + "def make_choice(utils_df, nests, nest_levels, seed=None):\n", + " rng = default_rng(seed=seed)\n", + " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", + " probs_arr = utils_df - np.log(-np.log(rands))\n", + " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", + " return choices\n", + " \n", + "make_choice(utils_df, nests_, nest_levels, seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 348, + "id": "bca659db-4c88-4287-8f5e-9c9516e3ef41", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:59:00.410062Z", + "iopub.status.busy": "2021-09-01T01:59:00.409811Z", + "iopub.status.idle": "2021-09-01T01:59:00.621940Z", + "shell.execute_reply": "2021-09-01T01:59:00.621156Z", + "shell.execute_reply.started": "2021-09-01T01:59:00.410037Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 WALK\n", + "137248725 WALK\n", + "211388201 WALK_LOC\n", + "211388205 WALK_LOC\n", + "806388401 WALK_LOC\n", + "806388405 WALK_LOC\n", + "dtype: object" + ] + }, + "execution_count": 348, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "make_choice(utils_df, nests_, nest_levels)" + ] + }, + { + "cell_type": "code", + "execution_count": 453, + "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:55:59.542669Z", + "iopub.status.busy": "2021-09-01T04:55:59.542360Z", + "iopub.status.idle": "2021-09-01T04:55:59.771722Z", + "shell.execute_reply": "2021-09-01T04:55:59.770801Z", + "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + " nested_probabilities = \\\n", + " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", + " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) \n", + " return base_probabilities\n", + "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 451, + "id": "054fd9b1-72fc-49cc-a790-48ef75bcbaed", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:55:08.260140Z", + "iopub.status.busy": "2021-09-01T04:55:08.259914Z", + "iopub.status.idle": "2021-09-01T04:55:09.140095Z", + "shell.execute_reply": "2021-09-01T04:55:09.139119Z", + "shell.execute_reply.started": "2021-09-01T04:55:08.260116Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "86627409 WALK_LRF\n", + "86627413 WALK_LRF\n", + "86673657 WALK_LOC\n", + "86673658 WALK\n", + "86673659 WALK\n", + " ... \n", + "2464446025 WALK\n", + "2464446029 WALK\n", + "2464449633 WALK\n", + "2464449634 WALK\n", + "2464449637 WALK\n", + "Length: 168, dtype: object" + ] + }, + "execution_count": 451, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def simple_simulate_rum(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", + "\n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices\n", + "\n", + "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" + ] + }, + { + "cell_type": "code", + "execution_count": 380, + "id": "2da81b5e-7992-4437-8e0f-905caf026804", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:33:10.886502Z", + "iopub.status.busy": "2021-09-01T02:33:10.886252Z", + "iopub.status.idle": "2021-09-01T02:33:11.092145Z", + "shell.execute_reply": "2021-09-01T02:33:11.090725Z", + "shell.execute_reply.started": "2021-09-01T02:33:10.886465Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'trip_mode_choice'" + ] + }, + "execution_count": 380, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trace_label" + ] + }, + { + "cell_type": "code", + "execution_count": 393, + "id": "f3c5c1ba-e9c3-4511-98bf-9ac98c987da7", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:39:52.539983Z", + "iopub.status.busy": "2021-09-01T02:39:52.539668Z", + "iopub.status.idle": "2021-09-01T02:39:52.796170Z", + "shell.execute_reply": "2021-09-01T02:39:52.795498Z", + "shell.execute_reply.started": "2021-09-01T02:39:52.539944Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " segment_trace_label, log_alt_losers=log_alt_losers, \n", + " trace_column_names=trace_column_names, seed=seed)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " choices_list.append(choices)\n", + "\n", + " choices_df = pd.concat(choices_list)\n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": 454, + "id": "74eeedd3-4ade-4729-8170-79fa9dcf11f8", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:58:52.825042Z", + "iopub.status.busy": "2021-09-01T04:58:52.824768Z", + "iopub.status.idle": "2021-09-01T04:58:53.042884Z", + "shell.execute_reply": "2021-09-01T04:58:53.042061Z", + "shell.execute_reply.started": "2021-09-01T04:58:52.825004Z" + } + }, + "outputs": [], + "source": [ + "## caculate probabilities with Asim methodology, should be correct\n", + "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", + " full_probs = [] # analytical probs\n", + "\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " #choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " # segment_trace_label, log_alt_losers=log_alt_losers, \n", + " # trace_column_names=trace_column_names, seed=seed)\n", + " probs = simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, \n", + " estimator, segment_trace_label, log_alt_losers, \n", + " trace_column_names)\n", + " full_probs.append(probs)\n", + " probs_df = pd.concat(full_probs)\n", + " return probs_df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 455, + "id": "295a4fd1-8e85-47e2-af82-9f1beac31d63", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:59:46.126555Z", + "iopub.status.busy": "2021-09-01T04:59:46.126262Z", + "iopub.status.idle": "2021-09-01T04:59:52.979908Z", + "shell.execute_reply": "2021-09-01T04:59:52.979043Z", + "shell.execute_reply.started": "2021-09-01T04:59:46.126522Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2021-09-01 14:59:46.353007 Start\n", + "2021-09-01 14:59:52.976736 End\n" + ] + } + ], + "source": [ + "print(f\"{datetime.now()} Start\")\n", + "pipeline.open_pipeline(resume_after)\n", + "inject.get_injectable('preload_injectables', None)\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "trips_df = trips.to_frame()\n", + "#print(\"Running with %d trips\", trips_df.shape[0])\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "\n", + "all_choices = []\n", + "for i in range(100):\n", + " if i % 10 == 0:\n", + " print(f\"{datetime.now()} iteration {i}\")\n", + " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=None)\n", + " all_choices.append(choices_df)\n", + "all_choices = pd.concat(all_choices, axis=1)\n", + "\n", + "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", + " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", + " log_alt_losers=None, trace_column_names=None)\n", + "\n", + "# update trips table with choices (and potionally logssums)\n", + "#trips_df = trips.to_frame()\n", + "#\n", + "#assign_in_place(trips_df, choices_df)\n", + "#assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(f\"{datetime.now()} End\")" + ] + }, + { + "cell_type": "code", + "execution_count": 463, + "id": "dc87e904-16d1-44d4-81b7-cafbe4e223c3", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:01:40.901982Z", + "iopub.status.busy": "2021-09-01T05:01:40.901759Z", + "iopub.status.idle": "2021-09-01T05:01:41.337758Z", + "shell.execute_reply": "2021-09-01T05:01:41.336757Z", + "shell.execute_reply.started": "2021-09-01T05:01:40.901946Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#all_choices.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", + "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts = val_counts / all_choices.shape[1]\n", + "#val_counts = val_counts.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", + "#val_counts['prob_of_asim_choice'] = val_counts.apply(lambda x: x[x.asim], axis=1) # this is what our simulation says w.r.t. to asim choice\n", + "# for 100% and many samples should mostly agree" + ] + }, + { + "cell_type": "code", + "execution_count": 464, + "id": "09bf151c-a39a-4823-acfd-9c8ae747c338", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:01:46.119117Z", + "iopub.status.busy": "2021-09-01T05:01:46.118849Z", + "iopub.status.idle": "2021-09-01T05:01:46.354114Z", + "shell.execute_reply": "2021-09-01T05:01:46.353248Z", + "shell.execute_reply.started": "2021-09-01T05:01:46.119092Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#val_counts['prob_of_asim_choice'].hist(bins=100);" + ] + }, + { + "cell_type": "code", + "execution_count": 490, + "id": "f3b6ade9-d875-4104-b438-b53ef42e342c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:10:08.653227Z", + "iopub.status.busy": "2021-09-01T05:10:08.652978Z", + "iopub.status.idle": "2021-09-01T05:10:08.915620Z", + "shell.execute_reply": "2021-09-01T05:10:08.914513Z", + "shell.execute_reply.started": "2021-09-01T05:10:08.653189Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", + "analytical 3.152 0.852 0.0 0.0 0.0 \n", + "experiment 3.243 0.880 0.0 0.0 0.0 \n", + "\n", + " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", + "analytical 0.0 0.0 0.0 0.685 0.0 \n", + "experiment 0.0 0.0 0.0 0.772 0.0 \n", + "\n", + " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK \\\n", + "analytical 0.129 0.0 0.182 0.260 1.334 63.708 \n", + "experiment 0.156 0.0 0.174 0.259 1.434 63.243 \n", + "\n", + " WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", + "analytical 0.0 0.0 0.455 18.355 10.887 \n", + "experiment 0.0 0.0 0.394 18.639 10.807 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mode_share_analytical = (probs_nl.sum(axis=0) / probs_nl.shape[0]).to_frame('analytical')\n", + "assert np.allclose(mode_share_analytical.sum(), 1)\n", + "mode_share_rum = (val_counts.sum(axis=0) / val_counts.shape[0]).to_frame('experiment')\n", + "assert np.allclose(mode_share_rum.sum(), 1)\n", + "full_share = mode_share_analytical.join(mode_share_rum, how='outer').fillna(0)\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * full_share).T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e48a56e-d613-4dfd-bef6-62bee524f12f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50914e1d-750d-440c-9617-f96bd2a46c56", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2d70e0e-520e-49a6-8024-ae31345f6ead", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "f9e8ef02-1d57-4b21-8aef-25a6e1095c02", + "metadata": {}, + "source": [ + "### try zenith normalisation of simple_simulate_rum" + ] + }, + { + "cell_type": "code", + "execution_count": 629, + "id": "db235e90-d2e5-4e90-b41c-6d90a670e41d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:24.045881Z", + "iopub.status.busy": "2021-09-01T11:37:24.045637Z", + "iopub.status.idle": "2021-09-01T11:37:24.276488Z", + "shell.execute_reply": "2021-09-01T11:37:24.274898Z", + "shell.execute_reply.started": "2021-09-01T11:37:24.045844Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation belowi s equivalent.\n", + "def compute_nested_utilities_zenith(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) # / nest.product_of_coefficients #coefficient\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", + " \n", + " # now go over all leaves and correct for scale\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] /= nest.coefficient\n", + " \n", + " return nested_utilities\n", + "\n", + "\n", + "def simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " utils_df = compute_nested_utilities_zenith(raw_utilities, nest_spec)\n", + "\n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices\n", + "\n", + "#simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" + ] + }, + { + "cell_type": "code", + "execution_count": 630, + "id": "f69bc3dc-84e4-4dd5-9a57-c053cadba4b5", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:24.828926Z", + "iopub.status.busy": "2021-09-01T11:37:24.828707Z", + "iopub.status.idle": "2021-09-01T11:37:25.046409Z", + "shell.execute_reply": "2021-09-01T11:37:25.045399Z", + "shell.execute_reply.started": "2021-09-01T11:37:24.828902Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " segment_trace_label, log_alt_losers=log_alt_losers, \n", + " trace_column_names=trace_column_names, seed=seed)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " choices_list.append(choices)\n", + "\n", + " choices_df = pd.concat(choices_list)\n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": 631, + "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:32.250808Z", + "iopub.status.busy": "2021-09-01T11:37:32.250548Z", + "iopub.status.idle": "2021-09-01T20:20:35.257463Z", + "shell.execute_reply": "2021-09-01T20:20:35.255964Z", + "shell.execute_reply.started": "2021-09-01T11:37:32.250782Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "num_samples = 10\n", + "\n", + "rng_test = default_rng(23423)\n", + "\n", + "print(f\"{datetime.now()} Start\")\n", + "pipeline.open_pipeline(resume_after)\n", + "inject.get_injectable('preload_injectables', None)\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "trips_df = trips.to_frame()\n", + "#print(\"Running with %d trips\", trips_df.shape[0])\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "all_choices_zenith = []\n", + "all_choices = []\n", + "for i in range(num_samples):\n", + " \n", + " seed = rng_test.integers(0, 100000) #int(9.3 * (i+1)**3) # why not\n", + " \n", + " if i % 50 == 0:\n", + " print(f\"{datetime.now()} iteration {i}\")\n", + " choices_df_zenith = stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", + " all_choices_zenith.append(choices_df_zenith)\n", + "\n", + " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", + " all_choices.append(choices_df)\n", + " \n", + " t_ = choices_df_zenith.merge(choices_df, left_index=True, right_index=True, suffixes=['_zenith', '_asim'])\n", + " diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", + " \n", + " #print(f\"seed {seed} leads to {diffs.shape[0]} differences. tripids {diffs.index}\")\n", + "\n", + "all_choices_zenith = pd.concat(all_choices_zenith, axis=1)\n", + "all_choices = pd.concat(all_choices, axis=1)\n", + "\n", + "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", + " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", + " log_alt_losers=None, trace_column_names=None)\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(f\"{datetime.now()} End\")" + ] + }, + { + "cell_type": "code", + "execution_count": 643, + "id": "3abd7949-dc9c-469b-845e-26c17741f70e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:33:21.533110Z", + "iopub.status.busy": "2021-09-01T20:33:21.531915Z", + "iopub.status.idle": "2021-09-01T20:33:21.740946Z", + "shell.execute_reply": "2021-09-01T20:33:21.739713Z", + "shell.execute_reply.started": "2021-09-01T20:33:21.533074Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Have 2038 samples\n" + ] + } + ], + "source": [ + "print(f\"Have {all_choices_zenith.shape[1]} samples\")" + ] + }, + { + "cell_type": "code", + "execution_count": 633, + "id": "8762bd6d-ac04-458c-869f-be86d8297351", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:21:27.778803Z", + "iopub.status.busy": "2021-09-01T20:21:27.778531Z", + "iopub.status.idle": "2021-09-01T20:21:28.675645Z", + "shell.execute_reply": "2021-09-01T20:21:28.674734Z", + "shell.execute_reply.started": "2021-09-01T20:21:27.778754Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "val_counts_zenith = all_choices_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts_zenith = val_counts_zenith / all_choices_zenith.shape[1]\n", + "\n", + "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts = val_counts / all_choices.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 634, + "id": "0f566105-9424-457b-86f5-d60c1d63aac5", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:21:30.954894Z", + "iopub.status.busy": "2021-09-01T20:21:30.954654Z", + "iopub.status.idle": "2021-09-01T20:21:31.179689Z", + "shell.execute_reply": "2021-09-01T20:21:31.178304Z", + "shell.execute_reply.started": "2021-09-01T20:21:30.954867Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
experiment_zenith3.1500.8740.00.00.00.00.00.00.6800.00.1360.00.1540.2361.28563.7670.00.00.51518.82410.379
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", + "analytical 3.152 0.852 0.0 0.0 0.0 \n", + "experiment 3.243 0.880 0.0 0.0 0.0 \n", + "experiment_zenith 3.150 0.874 0.0 0.0 0.0 \n", + "\n", + " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", + "analytical 0.0 0.0 0.0 0.685 0.0 \n", + "experiment 0.0 0.0 0.0 0.772 0.0 \n", + "experiment_zenith 0.0 0.0 0.0 0.680 0.0 \n", + "\n", + " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE \\\n", + "analytical 0.129 0.0 0.182 0.260 1.334 \n", + "experiment 0.156 0.0 0.174 0.259 1.434 \n", + "experiment_zenith 0.136 0.0 0.154 0.236 1.285 \n", + "\n", + " WALK WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", + "analytical 63.708 0.0 0.0 0.455 18.355 10.887 \n", + "experiment 63.243 0.0 0.0 0.394 18.639 10.807 \n", + "experiment_zenith 63.767 0.0 0.0 0.515 18.824 10.379 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mode_share_rum_zenith = (val_counts_zenith.sum(axis=0) / val_counts_zenith.shape[0]).to_frame('experiment_zenith')\n", + "assert np.allclose(mode_share_rum_zenith.sum(), 1)\n", + "full_share_incl_zenith = full_share.merge(mode_share_rum_zenith, left_index=True, right_index=True, how='outer').fillna(0)\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * full_share_incl_zenith).T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c0d3081-b45b-4573-9246-e4a9d39591dd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "612b2f04-45e2-4eb2-b52a-9f753427877f", + "metadata": {}, + "source": [ + "## investigate diverging seed and look at diff in formulation" + ] + }, + { + "cell_type": "code", + "execution_count": 619, + "id": "ac09dc75-a92f-43fb-a0cd-fe7de1f8b66d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:12:02.988141Z", + "iopub.status.busy": "2021-09-01T11:12:02.987900Z", + "iopub.status.idle": "2021-09-01T11:12:03.212179Z", + "shell.execute_reply": "2021-09-01T11:12:03.210993Z", + "shell.execute_reply.started": "2021-09-01T11:12:02.988102Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation based on asim probabilities is equivalent\n", + "def compute_nested_utilities_asim(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) / nest.product_of_coefficients\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", + " return nested_utilities\n", + "\n", + "def compute_nested_utilities_zenith_check(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) # scale correction is below\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", + " \n", + " # now go over all leaves and correct for scale\n", + " for nest in logit.each_nest(nest_spec):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] /= nest.coefficient\n", + " \n", + " return nested_utilities" + ] + }, + { + "cell_type": "code", + "execution_count": 688, + "id": "4b7c6368-21ca-4442-969e-f2f8ff868842", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:49:56.958862Z", + "iopub.status.busy": "2021-09-01T20:49:56.958623Z", + "iopub.status.idle": "2021-09-01T20:49:57.191884Z", + "shell.execute_reply": "2021-09-01T20:49:57.190975Z", + "shell.execute_reply.started": "2021-09-01T20:49:56.958827Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None, use_zenith=True, raw_utilities=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " \n", + " if raw_utilities is None:\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " if use_zenith:\n", + " utils_df = compute_nested_utilities_zenith_check(raw_utilities, nest_spec)\n", + " else:\n", + " utils_df = compute_nested_utilities_asim(raw_utilities, nest_spec)\n", + "\n", + " # test\n", + " #return compute_nested_utilities_zenith_check(raw_utilities, nest_spec), compute_nested_utilities_asim(raw_utilities, nest_spec)\n", + " \n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices" + ] + }, + { + "cell_type": "code", + "execution_count": 626, + "id": "bd6cadc3-79f5-4824-b55e-639c39d5ab18", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:13:12.077886Z", + "iopub.status.busy": "2021-09-01T11:13:12.077652Z", + "iopub.status.idle": "2021-09-01T11:13:12.290222Z", + "shell.execute_reply": "2021-09-01T11:13:12.288861Z", + "shell.execute_reply.started": "2021-09-01T11:13:12.077863Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# t_z, t_a = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed)\n", + "# tr_id = 86673661\n", + "# display(t_z.loc[t_z.index==tr_id])\n", + "# display(t_a.loc[t_a.index==tr_id])" + ] + }, + { + "cell_type": "code", + "execution_count": 625, + "id": "42e6da62-58e8-4b16-9169-9faa1fc8a162", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:13:06.847583Z", + "iopub.status.busy": "2021-09-01T11:13:06.847204Z", + "iopub.status.idle": "2021-09-01T11:13:08.401700Z", + "shell.execute_reply": "2021-09-01T11:13:08.400711Z", + "shell.execute_reply.started": "2021-09-01T11:13:06.847554Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seed 51006 leads to 34 differences out of 168. tripids Int64Index([ 86673657, 86673661, 106741681, 106741682, 106741685,\n", + " 106741686, 106741687, 106741688, 211327433, 211327437,\n", + " 444793574, 484173905, 484173909, 535170694, 535620053,\n", + " 708171014, 943749470, 943749471, 1060575853, 1091770617,\n", + " 1146472489, 1146472493, 1276281769, 1276281773, 1658748793,\n", + " 1658748797, 1767013726, 1767186577, 1767186578, 1768237161,\n", + " 1768237165, 1768237166, 2463663417, 2463663421],\n", + " dtype='int64', name='trip_id')\n" + ] + } + ], + "source": [ + "seed = 51006\n", + "x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True)\n", + "y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False)\n", + "t_ = x_.to_frame('trip_mode_zenith').merge(y_.to_frame('trip_mode_asim'), left_index=True, right_index=True)\n", + "diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", + "print(f\"seed {seed} leads to {diffs.shape[0]} differences out of {t_.shape[0]}. tripids {diffs.index}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f3c0506-a79d-4a31-9d56-71b9966d0a92", + "metadata": {}, + "outputs": [], + "source": [ + "seed 51006 leads to 3 differences. tripids Int64Index([86673661, 535170689, 1060575849], dtype='int64', name='trip_id')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a5f332a-b1d7-4a80-bb21-d37a1611cf9f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24c5e4a1-6142-49d4-8da8-a1e266b3cb9b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 753, + "id": "2098907f-28af-4f53-af74-6aaee8a6c53c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T23:04:23.861048Z", + "iopub.status.busy": "2021-09-01T23:04:23.860748Z", + "iopub.status.idle": "2021-09-01T23:04:24.069307Z", + "shell.execute_reply": "2021-09-01T23:04:24.068434Z", + "shell.execute_reply.started": "2021-09-01T23:04:23.861024Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "rng_ = default_rng(seed=100)\n", + "new_utils = pd.DataFrame(0.1 * -np.log(-np.log(rng_.uniform(0,1,raw_utilities.shape))), columns=raw_utilities.columns)\n", + "new_utils.index = raw_utilities.index" + ] + }, + { + "cell_type": "code", + "execution_count": 767, + "id": "da48a178-82b2-44a5-b45a-e9de3bb5e5c2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T23:17:20.369675Z", + "iopub.status.busy": "2021-09-01T23:17:20.369424Z", + "iopub.status.idle": "2021-09-01T23:17:20.666506Z", + "shell.execute_reply": "2021-09-01T23:17:20.665626Z", + "shell.execute_reply.started": "2021-09-01T23:17:20.369637Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
1372487210.0566420.0373050.0427800.0295760.0861540.0265860.0883130.1003310.0280210.0185160.0643910.0200090.0263990.0226220.0230070.0217540.0871260.0211710.0364750.0276500.135171
1372487250.1019020.0228760.0312270.0699900.0569130.0466850.0884110.0877850.0254140.0381620.0239880.0288000.0393180.0201600.0483640.0230030.0339590.0345970.0377080.0995230.041213
2113882010.0375620.0700330.0559510.0403890.0200410.0959740.0946780.0943680.0360140.0250510.0348600.0349530.0354990.0280400.0436500.0247130.0313880.0259310.0669990.0526780.051228
2113882050.0354800.0562560.0439440.0480850.0420780.0622210.1068190.0872650.0351170.0303990.0351540.0264690.0506460.0357270.0242840.0329050.0394270.0410920.0547140.0484590.063458
8063884010.0549180.0404880.0331490.0542940.0434770.0679820.1524650.0741520.0334040.0287350.0592690.0243390.0369680.0277600.0235220.0609320.0234450.0239310.0446430.0337730.058353
8063884050.0327640.0573010.0184370.0770990.0469630.0635210.0949510.0890720.0308260.0378960.0357520.0198780.0495360.0796100.0269220.0248330.0248400.0266950.0799360.0401460.043022
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n", + "trip_id \n", + "137248721 0.056642 0.037305 0.042780 0.029576 \n", + "137248725 0.101902 0.022876 0.031227 0.069990 \n", + "211388201 0.037562 0.070033 0.055951 0.040389 \n", + "211388205 0.035480 0.056256 0.043944 0.048085 \n", + "806388401 0.054918 0.040488 0.033149 0.054294 \n", + "806388405 0.032764 0.057301 0.018437 0.077099 \n", + "\n", + " SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\n", + "trip_id \n", + "137248721 0.086154 0.026586 0.088313 0.100331 0.028021 0.018516 \n", + "137248725 0.056913 0.046685 0.088411 0.087785 0.025414 0.038162 \n", + "211388201 0.020041 0.095974 0.094678 0.094368 0.036014 0.025051 \n", + "211388205 0.042078 0.062221 0.106819 0.087265 0.035117 0.030399 \n", + "806388401 0.043477 0.067982 0.152465 0.074152 0.033404 0.028735 \n", + "806388405 0.046963 0.063521 0.094951 0.089072 0.030826 0.037896 \n", + "\n", + " WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\n", + "trip_id \n", + "137248721 0.064391 0.020009 0.026399 0.022622 0.023007 0.021754 \n", + "137248725 0.023988 0.028800 0.039318 0.020160 0.048364 0.023003 \n", + "211388201 0.034860 0.034953 0.035499 0.028040 0.043650 0.024713 \n", + "211388205 0.035154 0.026469 0.050646 0.035727 0.024284 0.032905 \n", + "806388401 0.059269 0.024339 0.036968 0.027760 0.023522 0.060932 \n", + "806388405 0.035752 0.019878 0.049536 0.079610 0.026922 0.024833 \n", + "\n", + " DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n", + "trip_id \n", + "137248721 0.087126 0.021171 0.036475 0.027650 0.135171 \n", + "137248725 0.033959 0.034597 0.037708 0.099523 0.041213 \n", + "211388201 0.031388 0.025931 0.066999 0.052678 0.051228 \n", + "211388205 0.039427 0.041092 0.054714 0.048459 0.063458 \n", + "806388401 0.023445 0.023931 0.044643 0.033773 0.058353 \n", + "806388405 0.024840 0.026695 0.079936 0.040146 0.043022 " + ] + }, + "execution_count": 767, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nested_exp_utilities_cf = simulate.compute_nested_exp_utilities(new_utils, nest_spec)\n", + "nested_probabilities_cf = simulate.compute_nested_probabilities(nested_exp_utilities_cf, nest_spec, trace_label=None)\n", + "base_probabilities_cf = simulate.compute_base_probabilities(nested_probabilities_cf, nest_spec, spec)\n", + "base_probabilities_cf" + ] + }, + { + "cell_type": "code", + "execution_count": 799, + "id": "50dfd1ae-10c3-475c-94be-de783c2fa5c2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:15:29.056902Z", + "iopub.status.busy": "2021-09-02T00:15:29.056683Z", + "iopub.status.idle": "2021-09-02T00:27:21.965206Z", + "shell.execute_reply": "2021-09-02T00:27:21.964340Z", + "shell.execute_reply.started": "2021-09-02T00:15:29.056877Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 11min 49s, sys: 766 ms, total: 11min 50s\n", + "Wall time: 11min 52s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "num_samples = 10000 # 7.5s per 100\n", + "\n", + "data_zenith = []\n", + "data_asim = []\n", + "for i in range(num_samples):\n", + " seed = rng_.integers(0, 100000)\n", + " x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True, raw_utilities=new_utils)\n", + " y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False, raw_utilities=new_utils)\n", + " data_zenith.append(x_)\n", + " data_asim.append(y_)\n", + " \n", + "data_asim = pd.concat(data_asim, axis=1)\n", + "data_zenith = pd.concat(data_zenith, axis=1)\n", + "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", + "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_asim = counts_asim / data_asim.shape[1]\n", + "\n", + "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", + "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", + "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", + "# assert np.allclose(mode_share_zenith.sum(), 1)\n", + "# assert np.allclose(mode_share_asim.sum(), 1)\n", + "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", + "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 796, + "id": "bb16f9d7-44f5-4316-b981-d65216a5e217", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:14:11.772592Z", + "iopub.status.busy": "2021-09-02T00:14:11.772347Z", + "iopub.status.idle": "2021-09-02T00:14:12.001077Z", + "shell.execute_reply": "2021-09-02T00:14:12.000166Z", + "shell.execute_reply.started": "2021-09-02T00:14:11.772556Z" + } + }, + "outputs": [], + "source": [ + "#temp_z = data_zenith.copy()\n", + "#temp_a = data_asim.copy()\n", + "#data_asim = data_asim.join(temp_a, lsuffix=\"_o\", rsuffix=\"_n\")\n", + "#data_zenith = data_zenith.join(temp_z, lsuffix=\"_o\", rsuffix=\"_n\")\n", + "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", + "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_asim = counts_asim / data_asim.shape[1]\n", + "\n", + "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", + "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", + "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", + "# assert np.allclose(mode_share_zenith.sum(), 1)\n", + "# assert np.allclose(mode_share_asim.sum(), 1)\n", + "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", + "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 805, + "id": "031d8340-0af0-4608-8a83-ed2af4aaa3b2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:41:01.948669Z", + "iopub.status.busy": "2021-09-02T00:41:01.948422Z", + "iopub.status.idle": "2021-09-02T00:41:02.175972Z", + "shell.execute_reply": "2021-09-02T00:41:02.174759Z", + "shell.execute_reply.started": "2021-09-02T00:41:01.948633Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
zenith8.0555.1674.6313.5023.6064.2663.9633.4983.8955.1764.7995.8514.9996.1214.6689.4864.2284.1252.9773.6713.317
asim6.0966.6984.9773.2023.5224.3344.1293.1843.2245.1496.2838.2964.7936.5844.5327.4844.1524.2732.6293.4333.023
probs8.8835.3214.7382.8903.1364.0033.5653.1623.7585.3244.9276.0495.3416.5415.03710.4273.9734.2242.5743.1472.979
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP DRIVE_HVY \\\n", + "zenith 8.055 5.167 4.631 3.502 3.606 4.266 \n", + "asim 6.096 6.698 4.977 3.202 3.522 4.334 \n", + "probs 8.883 5.321 4.738 2.890 3.136 4.003 \n", + "\n", + " DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY SHARED3FREE \\\n", + "zenith 3.963 3.498 3.895 5.176 4.799 \n", + "asim 4.129 3.184 3.224 5.149 6.283 \n", + "probs 3.565 3.162 3.758 5.324 4.927 \n", + "\n", + " SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK WALK_COM WALK_EXP \\\n", + "zenith 5.851 4.999 6.121 4.668 9.486 4.228 4.125 \n", + "asim 8.296 4.793 6.584 4.532 7.484 4.152 4.273 \n", + "probs 6.049 5.341 6.541 5.037 10.427 3.973 4.224 \n", + "\n", + " WALK_HVY WALK_LOC WALK_LRF \n", + "zenith 2.977 3.671 3.317 \n", + "asim 2.629 3.433 3.023 \n", + "probs 2.574 3.147 2.979 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# sample of 20000\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * mode_share_comp).T)" + ] + }, + { + "cell_type": "code", + "execution_count": 806, + "id": "aece448f-4055-4d2a-b90d-8530caec9cb9", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:43:32.971972Z", + "iopub.status.busy": "2021-09-02T00:43:32.971684Z", + "iopub.status.idle": "2021-09-02T00:43:33.223345Z", + "shell.execute_reply": "2021-09-02T00:43:33.222488Z", + "shell.execute_reply.started": "2021-09-02T00:43:32.971932Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
trip_id137248721_z137248721_a137248721
BIKE0.092700.054350.100331
DRIVEALONEFREE0.053650.050050.056642
DRIVEALONEPAY0.038800.031550.037305
DRIVE_COM0.028700.024700.021171
DRIVE_EXP0.028800.026050.021754
DRIVE_HVY0.078150.099700.087126
DRIVE_LOC0.028750.027000.022622
DRIVE_LRF0.027800.024450.023007
SHARED2FREE0.043250.023000.042780
SHARED2PAY0.033600.015850.029576
SHARED3FREE0.073650.119200.086154
SHARED3PAY0.032600.039600.026586
TAXI0.035100.050300.036475
TNC_SHARED0.126200.181150.135171
TNC_SINGLE0.026250.038450.027650
WALK0.079400.048250.088313
WALK_COM0.031850.024600.026399
WALK_EXP0.057050.058950.064391
WALK_HVY0.025200.018400.020009
WALK_LOC0.034550.027400.028021
WALK_LRF0.023950.017000.018516
\n", + "
" + ], + "text/plain": [ + "trip_id 137248721_z 137248721_a 137248721\n", + "BIKE 0.09270 0.05435 0.100331\n", + "DRIVEALONEFREE 0.05365 0.05005 0.056642\n", + "DRIVEALONEPAY 0.03880 0.03155 0.037305\n", + "DRIVE_COM 0.02870 0.02470 0.021171\n", + "DRIVE_EXP 0.02880 0.02605 0.021754\n", + "DRIVE_HVY 0.07815 0.09970 0.087126\n", + "DRIVE_LOC 0.02875 0.02700 0.022622\n", + "DRIVE_LRF 0.02780 0.02445 0.023007\n", + "SHARED2FREE 0.04325 0.02300 0.042780\n", + "SHARED2PAY 0.03360 0.01585 0.029576\n", + "SHARED3FREE 0.07365 0.11920 0.086154\n", + "SHARED3PAY 0.03260 0.03960 0.026586\n", + "TAXI 0.03510 0.05030 0.036475\n", + "TNC_SHARED 0.12620 0.18115 0.135171\n", + "TNC_SINGLE 0.02625 0.03845 0.027650\n", + "WALK 0.07940 0.04825 0.088313\n", + "WALK_COM 0.03185 0.02460 0.026399\n", + "WALK_EXP 0.05705 0.05895 0.064391\n", + "WALK_HVY 0.02520 0.01840 0.020009\n", + "WALK_LOC 0.03455 0.02740 0.028021\n", + "WALK_LRF 0.02395 0.01700 0.018516" + ] + }, + "execution_count": 806, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# TODO: look at probs per trip, i.e. do not sum across trips\n", + "trip_id = 137248721\n", + "\n", + "counts_zenith.loc[counts_zenith.index == trip_id].T.merge(\n", + " counts_asim.loc[counts_asim.index == trip_id].T, suffixes=['_z', '_a'], left_index=True, right_index=True, how='outer').merge(\n", + " base_probabilities_cf.loc[base_probabilities_cf.index == trip_id].T, suffixes=['', '_probs'], left_index=True, right_index=True, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ef84f10-d616-40e4-87e5-425499027ff3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "431cee49-40a9-4dc7-93a5-75308fb302dd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fcdc0ff-c510-4878-8672-8365e046442c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/frozen_rand_mode_choice_dev.ipynb b/notebooks/frozen_rand_mode_choice_dev.ipynb new file mode 100644 index 0000000000..f4203e92a6 --- /dev/null +++ b/notebooks/frozen_rand_mode_choice_dev.ipynb @@ -0,0 +1,4199 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", + "metadata": {}, + "source": [ + "# Start work here, clean up as you go\n", + "\n", + "Problem below: scale of error term on lower levels needs to be given by nest, not the case atm" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.385153Z", + "iopub.status.busy": "2022-05-01T10:17:57.384881Z", + "iopub.status.idle": "2022-05-01T10:17:57.534433Z", + "shell.execute_reply": "2022-05-01T10:17:57.533096Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3dba451-1e10-403e-8614-35d57e6577f4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.536623Z", + "iopub.status.busy": "2022-05-01T10:17:57.536012Z", + "iopub.status.idle": "2022-05-01T10:17:57.542755Z", + "shell.execute_reply": "2022-05-01T10:17:57.541685Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:00.350944Z", + "iopub.status.busy": "2022-05-01T10:18:00.350730Z", + "iopub.status.idle": "2022-05-01T10:18:12.760977Z", + "shell.execute_reply": "2022-05-01T10:18:12.760013Z", + "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "from datetime import datetime\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from numpy.random import default_rng\n", + "\n", + "\n", + "from activitysim.cli import run\n", + "from activitysim.core import inject\n", + "from activitysim.core import tracing\n", + "from activitysim.core import config\n", + "from activitysim.core import pipeline\n", + "from activitysim.core import mem\n", + "from activitysim.core import chunk\n", + "from activitysim.core import simulate\n", + "from activitysim.core import logit\n", + "from activitysim.abm.models.util.mode import mode_choice_simulate\n", + "from activitysim.abm.models.util import estimation\n", + "from activitysim.core import expressions\n", + "from activitysim.core.util import assign_in_place" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:16.944537Z", + "iopub.status.busy": "2022-05-01T10:18:16.944291Z", + "iopub.status.idle": "2022-05-01T10:18:17.124764Z", + "shell.execute_reply": "2022-05-01T10:18:17.123725Z", + "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "pd.set_option(\"max_columns\", 500)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:17.839947Z", + "iopub.status.busy": "2022-05-01T10:18:17.839070Z", + "iopub.status.idle": "2022-05-01T10:18:18.019676Z", + "shell.execute_reply": "2022-05-01T10:18:18.018689Z", + "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_example_mtc\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:18.894533Z", + "iopub.status.busy": "2022-05-01T10:18:18.894303Z", + "iopub.status.idle": "2022-05-01T10:18:19.078807Z", + "shell.execute_reply": "2022-05-01T10:18:19.077951Z", + "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "os.chdir(example_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:22.111723Z", + "iopub.status.busy": "2022-05-01T10:18:22.111490Z", + "iopub.status.idle": "2022-05-01T10:18:22.297437Z", + "shell.execute_reply": "2022-05-01T10:18:22.296501Z", + "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "run.add_run_args(parser)\n", + "# args = parser.parse_args()\n", + "# parser.parse_args(['--sum', '7', '-1', '42'])\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", + "#run.run(args) # 2mins full example run\n", + "\n", + "\n", + "if not inject.is_injectable('preload_injectables'):\n", + " from activitysim import abm # register abm steps and other abm-specific injectables\n", + "run.handle_standard_args(args) # possibly update injectables" + ] + }, + { + "cell_type": "markdown", + "id": "6045438f-d1eb-4b5c-9737-185798b7f97f", + "metadata": {}, + "source": [ + "## trip mode choice by hand" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "600ae244-3e6c-4b66-8d39-aa9f6f60b378", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:26.399145Z", + "iopub.status.busy": "2022-05-01T10:18:26.398687Z", + "iopub.status.idle": "2022-05-01T10:18:26.583256Z", + "shell.execute_reply": "2022-05-01T10:18:26.582233Z", + "shell.execute_reply.started": "2022-05-01T10:18:26.399105Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#inject.get_table('trips').to_frame() #pipeline.orca.get_raw_table('trips').to_frame()\n", + "#inject.get_table('tours_merged').to_frame() #pipeline.orca.get_raw_table('tours_merged').to_frame()\n", + "#inject.get_injectable('network_los')" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "outputs": [], + "source": [ + "def run_trip_mode_choice(do_these_purposes=None, simulate_function=simulate.simple_simulate):\n", + " #do_these_purposes=['escort']\n", + " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", + " resume_after = \"trip_scheduling\"\n", + " model_name = \"trip_mode_choice\"\n", + " chunk_size = 0 # test_mtc means no chunking\n", + "\n", + " pipeline.open_pipeline(resume_after)\n", + " # preload any bulky injectables (e.g. skims) not in pipeline\n", + " inject.get_injectable('preload_injectables', None)\n", + " pipeline._PIPELINE.rng().begin_step(model_name)\n", + " step_name = model_name\n", + " args = {}\n", + " checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + " inject.set_step_args(args)\n", + "\n", + " trips = inject.get_table('trips')\n", + " tours_merged = inject.get_table('tours_merged')\n", + " network_los = inject.get_injectable('network_los')\n", + "\n", + " trace_label = 'trip_mode_choice'\n", + " model_settings_file_name = 'trip_mode_choice.yaml'\n", + " model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + " mode_column_name = 'trip_mode'\n", + "\n", + " trips_df = trips.to_frame()\n", + " print(\"Running with %d trips\", trips_df.shape[0])\n", + "\n", + " tours_merged = tours_merged.to_frame()\n", + " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "\n", + " # - trips_merged - merge trips and tours_merged\n", + " trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + " assert trips_merged.index.equals(trips.index)\n", + "\n", + " # setup skim keys\n", + " assert ('trip_period' not in trips_merged)\n", + " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + " orig_col = 'origin'\n", + " dest_col = 'destination'\n", + "\n", + " constants = {}\n", + " constants.update(config.get_model_constants(model_settings))\n", + " constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + " })\n", + "\n", + " skim_dict = network_los.get_default_skim_dict()\n", + "\n", + " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + " skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + " }\n", + "\n", + " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + " nest_spec = config.get_logit_model_settings(model_settings)\n", + "\n", + " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + "\n", + " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", + " continue\n", + "\n", + " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " (primary_purpose, len(trips_segment.index), ))\n", + "\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " ################ Replace wrapper function\n", + " # choices = mode_choice_simulate(...)\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simulate_function(\n", + " choosers=trips_segment,\n", + " spec=spec,\n", + " nest_spec=nest_spec,\n", + " skims=skims,\n", + " locals_d=locals_dict,\n", + " chunk_size=chunk_size,\n", + " want_logsums=logsum_column_name is not None,\n", + " trace_label=segment_trace_label,\n", + " trace_choice_name='trip_mode_choice',\n", + " estimator=estimator,\n", + " trace_column_names=None)\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " alts = spec.columns\n", + " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", + " ################\n", + " choices_list.append(choices)\n", + " choices_df_asim = pd.concat(choices_list)\n", + "\n", + " # update trips table with choices (and potionally logssums)\n", + " trips_df = trips.to_frame()\n", + "\n", + " if (do_these_purposes is not None):\n", + " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", + "\n", + " assign_in_place(trips_df, choices_df_asim)\n", + " assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + " finalise = True\n", + " if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + " print(\"Done\")\n", + "\n", + " return trips_df" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:28:53.643404Z", + "iopub.status.busy": "2021-09-01T02:28:53.643055Z", + "iopub.status.idle": "2021-09-01T02:29:01.223951Z", + "shell.execute_reply": "2021-09-01T02:29:01.222865Z", + "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'eatout' (33 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'escort' (6 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'othmaint' (46 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'school' (37 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'shopping' (77 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'social' (19 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'univ' (26 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'work' (168 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n" + ] + } + ], + "source": [ + "trips_df = run_trip_mode_choice()" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "7eae2854-38bd-4da0-a161-3a15bbd17177", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:30:57.562848Z", + "iopub.status.busy": "2021-08-31T10:30:57.562615Z", + "iopub.status.idle": "2021-08-31T10:30:57.806655Z", + "shell.execute_reply": "2021-08-31T10:30:57.805135Z", + "shell.execute_reply.started": "2021-08-31T10:30:57.562824Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n8684833 26478 26478 eatout 1 True \n8684837 26478 26478 eatout 1 False \n8685009 26478 26478 othmaint 1 True \n8685013 26478 26478 othmaint 1 False \n8753057 26686 26686 eatout 1 True \n... ... ... ... ... ... \n2472945113 7539466 2848131 shopping 1 True \n2472945117 7539466 2848131 shopping 1 False \n2472945118 7539466 2848131 shopping 2 False \n2473024473 7539708 2848373 univ 1 True \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n8684833 1 13 8 1085604 eatout \n8684837 1 8 13 1085604 home \n8685009 1 10 8 1085626 othmaint \n8685013 1 8 10 1085626 home \n8753057 1 5 8 1094132 eatout \n... ... ... ... ... ... \n2472945113 1 8 3 309118139 shopping \n2472945117 2 25 8 309118139 shopping \n2472945118 2 3 25 309118139 home \n2473024473 1 13 18 309128059 univ \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n8684833 NaN 11.0 WALK -1.171760 \n8684837 NaN 11.0 WALK -1.238719 \n8685009 NaN 12.0 BIKE 6.198626 \n8685013 NaN 13.0 BIKE 6.175681 \n8753057 NaN 19.0 WALK 4.457539 \n... ... ... ... ... \n2472945113 NaN 18.0 WALK_LOC 12.537675 \n2472945117 56.842247 21.0 WALK_LOC 11.880804 \n2472945118 NaN 22.0 WALK 13.710030 \n2473024473 NaN 16.0 WALK_LOC -0.530696 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[482 rows x 14 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86848332647826478eatout1True11381085604eatoutNaN11.0WALK-1.171760
86848372647826478eatout1False18131085604homeNaN11.0WALK-1.238719
86850092647826478othmaint1True11081085626othmaintNaN12.0BIKE6.198626
86850132647826478othmaint1False18101085626homeNaN13.0BIKE6.175681
87530572668626686eatout1True1581094132eatoutNaN19.0WALK4.457539
.............................................
247294511375394662848131shopping1True183309118139shoppingNaN18.0WALK_LOC12.537675
247294511775394662848131shopping1False2258309118139shopping56.84224721.0WALK_LOC11.880804
247294511875394662848131shopping2False2325309118139homeNaN22.0WALK13.710030
247302447375397082848373univ1True11318309128059univNaN16.0WALK_LOC-0.530696
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

482 rows × 14 columns

\n
" + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips_df" + ] + }, + { + "cell_type": "markdown", + "id": "8eebea31-499c-41ca-8411-883a88ca800a", + "metadata": {}, + "source": [ + "## nested dev" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:01.113060Z", + "iopub.status.busy": "2021-08-31T10:31:01.112829Z", + "iopub.status.idle": "2021-08-31T10:31:01.359912Z", + "shell.execute_reply": "2021-08-31T10:31:01.358948Z", + "shell.execute_reply.started": "2021-08-31T10:31:01.113034Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# see fct above - return if necessary\n", + "#spec = simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + "#nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "36c100b3-3e39-4950-a586-4d42be695eaa", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:01.775068Z", + "iopub.status.busy": "2021-08-31T10:31:01.774846Z", + "iopub.status.idle": "2021-08-31T10:31:01.975816Z", + "shell.execute_reply": "2021-08-31T10:31:01.974499Z", + "shell.execute_reply.started": "2021-08-31T10:31:01.775044Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#print(nest_spec)\n", + "#for nest in logit.each_nest(nest_spec):\n", + "# nest.print()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:59:35.109079Z", + "iopub.status.busy": "2021-09-01T01:59:35.108781Z", + "iopub.status.idle": "2021-09-01T01:59:35.319301Z", + "shell.execute_reply": "2021-09-01T01:59:35.318402Z", + "shell.execute_reply.started": "2021-09-01T01:59:35.109050Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_dev(choosers, spec, nest_spec,\n", + " skims=None, locals_d=None,\n", + " chunk_size=0, custom_chooser=None,\n", + " log_alt_losers=False,\n", + " want_logsums=False,\n", + " estimator=None,\n", + " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", + " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", + " assert len(choosers) > 0\n", + " result_list = []\n", + " # segment by person type and pick the right spec for each person type\n", + " for i, chooser_chunk, chunk_trace_label \\\n", + " in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", + " # the following replaces choices = _simple_simulate(...)\n", + " if skims is not None:\n", + " simulate.set_skim_wrapper_targets(choosers, skims)\n", + "\n", + " # only do this for nested, logit is straight forward\n", + " assert nest_spec is not None\n", + " choices = simulate.eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser,\n", + " log_alt_losers=log_alt_losers,\n", + " want_logsums=want_logsums,\n", + " estimator=estimator,\n", + " trace_label=trace_label,\n", + " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", + "\n", + "\n", + " result_list.append(choices)\n", + " chunk.log_df(trace_label, f'result_list', result_list)\n", + "\n", + " if len(result_list) > 1:\n", + " choices = pd.concat(result_list)\n", + " assert len(choices.index == len(choosers.index))\n", + " return choices\n" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n" + ] + } + ], + "source": [ + "test_trips = run_trip_mode_choice(do_these_purposes=[\"escort\"], simulate_function=simple_simulate_dev)" + ] + }, + { + "cell_type": "markdown", + "id": "c60656db-075a-4eab-96b3-fc54eb785075", + "metadata": {}, + "source": [ + "## eval_nl dev" + ] + }, + { + "cell_type": "code", + "execution_count": 372, + "id": "56c8923c-e946-48d8-b964-87695c30163d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:27:40.290389Z", + "iopub.status.busy": "2021-09-01T02:27:40.290139Z", + "iopub.status.idle": "2021-09-01T02:27:40.977526Z", + "shell.execute_reply": "2021-09-01T02:27:40.976594Z", + "shell.execute_reply.started": "2021-09-01T02:27:40.290363Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# replaces simple_simulate_dev:\n", + "\n", + "want_logsums = True\n", + "trace_choice_name='trip_mode_choice'\n", + "trace_column_names=None\n", + "log_alt_losers=False\n", + "\n", + "tr_label = tracing.extend_trace_label(segment_trace_label, 'simple_simulate')\n", + "assert len(trips_segment) > 0\n", + "result_list = []\n", + "# segment by person type and pick the right spec for each person type\n", + "for i, chooser_chunk, chunk_trace_label \\\n", + " in chunk.adaptive_chunked_choosers(trips_segment, chunk_size, tr_label):\n", + " # the following replaces choices = _simple_simulate(...)\n", + " if skims is not None:\n", + " simulate.set_skim_wrapper_targets(trips_segment, skims)\n", + "\n", + " # only do this for nested, logit is straight forward\n", + " assert nest_spec is not None\n", + " choices = simulate.eval_nl(trips_segment, spec, nest_spec, locals_dict, custom_chooser=None,\n", + " log_alt_losers=log_alt_losers,\n", + " want_logsums=want_logsums,\n", + " estimator=estimator,\n", + " trace_label=tr_label,\n", + " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", + "\n", + "\n", + " result_list.append(choices)\n", + "\n", + "if len(result_list) > 1:\n", + " choices = pd.concat(result_list)\n", + "assert len(choices.index == len(trips_segment.index))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "simple_simulate_dev" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 93, + "id": "cbc2dfde-1bd9-4b03-96d9-ca9b7222d36c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T12:52:34.544865Z", + "iopub.status.busy": "2021-08-31T12:52:34.544586Z", + "iopub.status.idle": "2021-08-31T12:52:35.159615Z", + "shell.execute_reply": "2021-08-31T12:52:35.158682Z", + "shell.execute_reply.started": "2021-08-31T12:52:34.544840Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "locals_d = locals_dict\n", + "custom_chooser=None\n", + "\n", + "trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + "logit.validate_nest_spec(nest_spec, trace_label)\n", + "\n", + "raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + "nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + "\n", + "nested_probabilities = \\\n", + " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", + "\n", + "logsums = pd.Series(np.log(nested_exp_utilities.root), index=trips_segment.index)\n", + "\n", + "# global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + "base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", + "\n", + "choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "id": "0cd9593b-b3fd-41f6-a7ca-4c5b73b63571", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T12:52:45.509057Z", + "iopub.status.busy": "2021-08-31T12:52:45.508819Z", + "iopub.status.idle": "2021-08-31T12:52:45.721956Z", + "shell.execute_reply": "2021-08-31T12:52:45.721064Z", + "shell.execute_reply.started": "2021-08-31T12:52:45.509021Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 0.990308\n", + "137248725 0.108433\n", + "211388201 0.806612\n", + "211388205 0.195994\n", + "806388401 0.854020\n", + "806388405 0.660550\n", + "dtype: float64" + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rands" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "id": "b467330b-f1cc-43bc-9325-c6dfeaa9b364", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T12:50:51.735273Z", + "iopub.status.busy": "2021-08-31T12:50:51.734962Z", + "iopub.status.idle": "2021-08-31T12:50:52.023440Z", + "shell.execute_reply": "2021-08-31T12:50:52.022582Z", + "shell.execute_reply.started": "2021-08-31T12:50:51.735231Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.798480-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.360270-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.435800
137248725-1.732971-3966.018685-0.606540-0.324578-3964.610293-0.113602-3966.009714-7930.295428-inf0.26146615.945036-1368.16910311.480426-2737.292402-5512.292402-5512.292402-5512.292402-5512.292402-inf-2737.782457-5522.136798-5522.136798-5516.076156-5525.080218-inf-inf-24.341318-25.104291-24.140831-8.40687011.480440
211388201-3966.000225-7930.285939-inf-24.120135-3988.405850-8.442047-29.480718-3993.766432-10.318251-5.975663-4.216264-1381.397292-3.03571013.295720-2761.150907-2761.150907-2761.150907-2761.1509076.647860-2761.648763-5543.048131-5543.048131-2769.692185-2773.867648-inf4.786459-15.861515-3.864497-5.068360-1.2967574.789158
211388205-3967.791945-7932.077659-inf-25.129596-3989.415310-8.795359-30.177275-3994.462989-10.562046-6.219063-4.216264-1381.397292-3.03571014.022438-2760.977562-2760.977562-2760.977562-2760.9775627.011219-2761.294814-5546.132585-5546.132585-2764.829835-2774.132982-inf5.048078-15.776424-3.845077-5.005482-1.2860945.050171
806388401-3966.492870-7930.778584-inf-12.618378-3976.904092-4.416432-14.908422-3979.194136-5.217948-2.9129776.192499-1378.4807234.45859917.513379-2755.845870-2755.208370-2755.728926-2755.7930938.756690-2757.559519-5540.685990-5540.685990-5542.838278-5546.780761-inf6.304817-19.143696-19.495908-16.361074-5.8539176.451457
806388405-3969.835821-7934.121535-inf-14.515916-3978.801631-5.080571-16.227796-3980.513510-5.679728-3.3428055.998749-1378.5194734.31909917.553369-2755.987742-2755.350242-2755.870797-2755.9349648.776685-2757.571454-5542.673358-5542.673358-5537.799228-5547.515590-inf6.319213-18.886576-19.495133-16.300395-5.8285406.446188
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", + "trip_id \n", + "137248721 -1.798480 -3966.084195 -0.629468 -0.389611 \n", + "137248725 -1.732971 -3966.018685 -0.606540 -0.324578 \n", + "211388201 -3966.000225 -7930.285939 -inf -24.120135 \n", + "211388205 -3967.791945 -7932.077659 -inf -25.129596 \n", + "806388401 -3966.492870 -7930.778584 -inf -12.618378 \n", + "806388405 -3969.835821 -7934.121535 -inf -14.515916 \n", + "\n", + " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", + "trip_id \n", + "137248721 -3964.675326 -0.136364 -3966.074556 -7930.360270 -inf \n", + "137248725 -3964.610293 -0.113602 -3966.009714 -7930.295428 -inf \n", + "211388201 -3988.405850 -8.442047 -29.480718 -3993.766432 -10.318251 \n", + "211388205 -3989.415310 -8.795359 -30.177275 -3994.462989 -10.562046 \n", + "806388401 -3976.904092 -4.416432 -14.908422 -3979.194136 -5.217948 \n", + "806388405 -3978.801631 -5.080571 -16.227796 -3980.513510 -5.679728 \n", + "\n", + " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", + "trip_id \n", + "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", + "137248725 0.261466 15.945036 -1368.169103 11.480426 -2737.292402 \n", + "211388201 -5.975663 -4.216264 -1381.397292 -3.035710 13.295720 \n", + "211388205 -6.219063 -4.216264 -1381.397292 -3.035710 14.022438 \n", + "806388401 -2.912977 6.192499 -1378.480723 4.458599 17.513379 \n", + "806388405 -3.342805 5.998749 -1378.519473 4.319099 17.553369 \n", + "\n", + " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", + "trip_id \n", + "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", + "137248725 -5512.292402 -5512.292402 -5512.292402 -5512.292402 -inf \n", + "211388201 -2761.150907 -2761.150907 -2761.150907 -2761.150907 6.647860 \n", + "211388205 -2760.977562 -2760.977562 -2760.977562 -2760.977562 7.011219 \n", + "806388401 -2755.845870 -2755.208370 -2755.728926 -2755.793093 8.756690 \n", + "806388405 -2755.987742 -2755.350242 -2755.870797 -2755.934964 8.776685 \n", + "\n", + " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", + "trip_id \n", + "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", + "137248725 -2737.782457 -5522.136798 -5522.136798 -5516.076156 -5525.080218 \n", + "211388201 -2761.648763 -5543.048131 -5543.048131 -2769.692185 -2773.867648 \n", + "211388205 -2761.294814 -5546.132585 -5546.132585 -2764.829835 -2774.132982 \n", + "806388401 -2757.559519 -5540.685990 -5540.685990 -5542.838278 -5546.780761 \n", + "806388405 -2757.571454 -5542.673358 -5542.673358 -5537.799228 -5547.515590 \n", + "\n", + " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", + "trip_id \n", + "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", + "137248725 -inf -inf -24.341318 -25.104291 -24.140831 -8.406870 \n", + "211388201 -inf 4.786459 -15.861515 -3.864497 -5.068360 -1.296757 \n", + "211388205 -inf 5.048078 -15.776424 -3.845077 -5.005482 -1.286094 \n", + "806388401 -inf 6.304817 -19.143696 -19.495908 -16.361074 -5.853917 \n", + "806388405 -inf 6.319213 -18.886576 -19.495133 -16.300395 -5.828540 \n", + "\n", + " root \n", + "trip_id \n", + "137248721 11.435800 \n", + "137248725 11.480440 \n", + "211388201 4.789158 \n", + "211388205 5.050171 \n", + "806388401 6.451457 \n", + "806388405 6.446188 " + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# root is logsum here, no need to take log again as for exponentiated utils\n", + "simulate.compute_nested_exp_utilities(raw_utilities, nest_spec, exponentiate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "5c9ba018-167d-497f-ac65-f7c49036ee58", + "metadata": {}, + "source": [ + "#### scale of utils, including emus" + ] + }, + { + "cell_type": "code", + "execution_count": 283, + "id": "9febf6ef-7c6b-4f3c-bdd9-aa3326a82e95", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:23:00.020038Z", + "iopub.status.busy": "2021-08-31T23:23:00.019377Z", + "iopub.status.idle": "2021-08-31T23:23:00.220889Z", + "shell.execute_reply": "2021-08-31T23:23:00.220154Z", + "shell.execute_reply.started": "2021-08-31T23:23:00.019997Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "nests_ = list(logit.each_nest(nest_spec))" + ] + }, + { + "cell_type": "code", + "execution_count": 284, + "id": "48ff920e-0c9b-4b69-9cee-eb3c3427bd4f", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:23:01.085713Z", + "iopub.status.busy": "2021-08-31T23:23:01.085486Z", + "iopub.status.idle": "2021-08-31T23:23:01.283509Z", + "shell.execute_reply": "2021-08-31T23:23:01.282444Z", + "shell.execute_reply.started": "2021-08-31T23:23:01.085685Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] alternatives: ['DRIVEALONE', 'SHAREDRIDE2', 'SHAREDRIDE3']\n", + "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] alternatives: ['DRIVEALONEFREE', 'DRIVEALONEPAY']\n", + "Nest name: DRIVEALONEFREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE'] alternatives: None\n", + "Nest name: DRIVEALONEPAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY'] alternatives: None\n" + ] + } + ], + "source": [ + "#list(map(lambda x: x.print(), nests_));\n", + "# have a look at \n", + "nests_[1].print()\n", + "nests_[2].print()\n", + "nests_[3].print()\n", + "nests_[4].print()" + ] + }, + { + "cell_type": "code", + "execution_count": 312, + "id": "7b7e453d-da45-4751-8889-561a349bd373", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:22.371464Z", + "iopub.status.busy": "2021-09-01T01:11:22.371192Z", + "iopub.status.idle": "2021-09-01T01:11:22.593424Z", + "shell.execute_reply": "2021-09-01T01:11:22.592472Z", + "shell.execute_reply.started": "2021-09-01T01:11:22.371415Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation belowi s equivalent.\n", + "def compute_nested_utilities(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) / nest.product_of_coefficients #coefficient\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", + " return nested_utilities\n", + "\n", + "\n", + "utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", + "\n", + "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", + "seed = 9326543345\n", + "rng = default_rng(seed=seed)\n", + "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", + "rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", + "\n", + "probs_arr = utils_df - np.log(-np.log(rands))\n", + "\n", + "# Now replace the following with recursive nested choice\n", + "# choices = np.argmax(probs_arr, axis=1)\n", + "# choices = pd.Series(choices, index=utils_df.index)\n", + "# rands = pd.Series(np.asanyarray(rands).flatten(), index=utils_df.index)" + ] + }, + { + "cell_type": "code", + "execution_count": 313, + "id": "39b71df9-12ac-4d9c-8134-43d22f7e6a6f", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:23.703299Z", + "iopub.status.busy": "2021-09-01T01:11:23.703088Z", + "iopub.status.idle": "2021-09-01T01:11:23.937077Z", + "shell.execute_reply": "2021-09-01T01:11:23.935627Z", + "shell.execute_reply.started": "2021-09-01T01:11:23.703274Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.79848-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.36027-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.4358
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", + "trip_id \n", + "137248721 -1.79848 -3966.084195 -0.629468 -0.389611 \n", + "\n", + " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", + "trip_id \n", + "137248721 -3964.675326 -0.136364 -3966.074556 -7930.36027 -inf \n", + "\n", + " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", + "trip_id \n", + "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", + "\n", + " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", + "trip_id \n", + "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", + "\n", + " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", + "trip_id \n", + "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", + "\n", + " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", + "trip_id \n", + "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", + "\n", + " root \n", + "trip_id \n", + "137248721 11.4358 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-0.717963-3964.2474012.7217010.230429-3964.716506-0.274584-3965.126773-7930.610173-inf1.73337615.473633-1368.52552614.717027-2736.517471-5510.738449-5512.038516-5512.559241-5511.584293-inf-2738.143234-5518.606969-5519.009587-5519.830735-5521.525966-inf-inf-25.519792-21.698963-25.267185-7.44465211.689443
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", + "trip_id \n", + "137248721 -0.717963 -3964.247401 2.721701 0.230429 \n", + "\n", + " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", + "trip_id \n", + "137248721 -3964.716506 -0.274584 -3965.126773 -7930.610173 -inf \n", + "\n", + " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", + "trip_id \n", + "137248721 1.733376 15.473633 -1368.525526 14.717027 -2736.517471 \n", + "\n", + " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", + "trip_id \n", + "137248721 -5510.738449 -5512.038516 -5512.559241 -5511.584293 -inf \n", + "\n", + " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", + "trip_id \n", + "137248721 -2738.143234 -5518.606969 -5519.009587 -5519.830735 -5521.525966 \n", + "\n", + " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", + "trip_id \n", + "137248721 -inf -inf -25.519792 -21.698963 -25.267185 -7.444652 \n", + "\n", + " root \n", + "trip_id \n", + "137248721 11.689443 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(utils_df.head(1), probs_arr.head(1))" + ] + }, + { + "cell_type": "markdown", + "id": "2f080150-c15a-4059-8a26-42a8c0072606", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:16:40.014024Z", + "iopub.status.busy": "2021-08-31T23:16:40.013770Z", + "iopub.status.idle": "2021-08-31T23:16:40.236326Z", + "shell.execute_reply": "2021-08-31T23:16:40.235553Z", + "shell.execute_reply.started": "2021-08-31T23:16:40.013973Z" + }, + "tags": [] + }, + "source": [ + "### make choice at each level" + ] + }, + { + "cell_type": "code", + "execution_count": 315, + "id": "2ed71b54-67e4-4087-957a-75f4fa184144", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:43.778081Z", + "iopub.status.busy": "2021-09-01T01:11:43.777757Z", + "iopub.status.idle": "2021-09-01T01:11:43.975332Z", + "shell.execute_reply": "2021-09-01T01:11:43.974588Z", + "shell.execute_reply.started": "2021-09-01T01:11:43.778055Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_alternatives(nests, name):\n", + " alts = list(filter(lambda x: x.name == name, nests))\n", + " assert len(alts) == 1, f\"{len(alts)} not one\"\n", + " alts = alts[0].alternatives\n", + " return alts\n", + "\n", + "def recursive_choice(row, columns, nest_levels, nests):\n", + " choices = row[columns].idxmax() #axis=1).values[0]\n", + " next_level_columns = get_alternatives(nests, choices)\n", + " #print(f\"{choices} leads to columns {next_level_columns}\")\n", + " if next_level_columns is None:\n", + " return choices \n", + " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", + " return new_choice" + ] + }, + { + "cell_type": "code", + "execution_count": 316, + "id": "30424562-2627-446b-abd5-7c763c52060e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:49.444289Z", + "iopub.status.busy": "2021-09-01T01:11:49.444002Z", + "iopub.status.idle": "2021-09-01T01:11:49.664214Z", + "shell.execute_reply": "2021-09-01T01:11:49.663088Z", + "shell.execute_reply.started": "2021-09-01T01:11:49.444266Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n" + ] + } + ], + "source": [ + "# group nests by level:\n", + "depth = np.max([x.level for x in nests_])\n", + "print(depth)\n", + "nest_levels = {x: [] for x in range(1, depth+1)}\n", + "for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + "assert len(nest_levels[1]) == 1\n", + "assert nest_levels[1][0] == 'root'" + ] + }, + { + "cell_type": "code", + "execution_count": 326, + "id": "bae6e7a8-d7c3-42a2-a82f-874b1e59a0b6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:56:38.377472Z", + "iopub.status.busy": "2021-09-01T01:56:38.377253Z", + "iopub.status.idle": "2021-09-01T01:56:38.607858Z", + "shell.execute_reply": "2021-09-01T01:56:38.606913Z", + "shell.execute_reply.started": "2021-09-01T01:56:38.377447Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 WALK\n", + "137248725 WALK\n", + "211388201 WALK_LOC\n", + "211388205 WALK_LOC\n", + "806388401 WALK\n", + "806388405 WALK\n", + "dtype: object" + ] + }, + "execution_count": 326, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests_), axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 321, + "id": "fc315f63-542f-46fd-8d70-b7a1fcd492c6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:53:15.500933Z", + "iopub.status.busy": "2021-09-01T01:53:15.500709Z", + "iopub.status.idle": "2021-09-01T01:53:15.703201Z", + "shell.execute_reply": "2021-09-01T01:53:15.702357Z", + "shell.execute_reply.started": "2021-09-01T01:53:15.500909Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#display(utils_df, probs_arr)" + ] + }, + { + "cell_type": "markdown", + "id": "8a45e1c0-0a32-4f7a-8803-7559d71abefd", + "metadata": {}, + "source": [ + "### some tests\n", + "seeds, average?" + ] + }, + { + "cell_type": "code", + "execution_count": 333, + "id": "f0e00814-2a14-4da1-bc6c-89b52b7fb286", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:58:26.002425Z", + "iopub.status.busy": "2021-09-01T01:58:26.002200Z", + "iopub.status.idle": "2021-09-01T01:58:26.247954Z", + "shell.execute_reply": "2021-09-01T01:58:26.247054Z", + "shell.execute_reply.started": "2021-09-01T01:58:26.002401Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 WALK\n", + "137248725 WALK\n", + "211388201 WALK_LOC\n", + "211388205 WALK_LOC\n", + "806388401 WALK\n", + "806388405 WALK\n", + "dtype: object" + ] + }, + "execution_count": 333, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", + "seed = 9326543345\n", + "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", + "\n", + "def make_choice(utils_df, nests, nest_levels, seed=None):\n", + " rng = default_rng(seed=seed)\n", + " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", + " probs_arr = utils_df - np.log(-np.log(rands))\n", + " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", + " return choices\n", + " \n", + "make_choice(utils_df, nests_, nest_levels, seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 348, + "id": "bca659db-4c88-4287-8f5e-9c9516e3ef41", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:59:00.410062Z", + "iopub.status.busy": "2021-09-01T01:59:00.409811Z", + "iopub.status.idle": "2021-09-01T01:59:00.621940Z", + "shell.execute_reply": "2021-09-01T01:59:00.621156Z", + "shell.execute_reply.started": "2021-09-01T01:59:00.410037Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "137248721 WALK\n", + "137248725 WALK\n", + "211388201 WALK_LOC\n", + "211388205 WALK_LOC\n", + "806388401 WALK_LOC\n", + "806388405 WALK_LOC\n", + "dtype: object" + ] + }, + "execution_count": 348, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "make_choice(utils_df, nests_, nest_levels)" + ] + }, + { + "cell_type": "code", + "execution_count": 453, + "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:55:59.542669Z", + "iopub.status.busy": "2021-09-01T04:55:59.542360Z", + "iopub.status.idle": "2021-09-01T04:55:59.771722Z", + "shell.execute_reply": "2021-09-01T04:55:59.770801Z", + "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + " nested_probabilities = \\\n", + " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", + " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) \n", + " return base_probabilities\n", + "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 451, + "id": "054fd9b1-72fc-49cc-a790-48ef75bcbaed", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:55:08.260140Z", + "iopub.status.busy": "2021-09-01T04:55:08.259914Z", + "iopub.status.idle": "2021-09-01T04:55:09.140095Z", + "shell.execute_reply": "2021-09-01T04:55:09.139119Z", + "shell.execute_reply.started": "2021-09-01T04:55:08.260116Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "86627409 WALK_LRF\n", + "86627413 WALK_LRF\n", + "86673657 WALK_LOC\n", + "86673658 WALK\n", + "86673659 WALK\n", + " ... \n", + "2464446025 WALK\n", + "2464446029 WALK\n", + "2464449633 WALK\n", + "2464449634 WALK\n", + "2464449637 WALK\n", + "Length: 168, dtype: object" + ] + }, + "execution_count": 451, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def simple_simulate_rum(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", + "\n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices\n", + "\n", + "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" + ] + }, + { + "cell_type": "code", + "execution_count": 380, + "id": "2da81b5e-7992-4437-8e0f-905caf026804", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:33:10.886502Z", + "iopub.status.busy": "2021-09-01T02:33:10.886252Z", + "iopub.status.idle": "2021-09-01T02:33:11.092145Z", + "shell.execute_reply": "2021-09-01T02:33:11.090725Z", + "shell.execute_reply.started": "2021-09-01T02:33:10.886465Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'trip_mode_choice'" + ] + }, + "execution_count": 380, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trace_label" + ] + }, + { + "cell_type": "code", + "execution_count": 393, + "id": "f3c5c1ba-e9c3-4511-98bf-9ac98c987da7", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:39:52.539983Z", + "iopub.status.busy": "2021-09-01T02:39:52.539668Z", + "iopub.status.idle": "2021-09-01T02:39:52.796170Z", + "shell.execute_reply": "2021-09-01T02:39:52.795498Z", + "shell.execute_reply.started": "2021-09-01T02:39:52.539944Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " segment_trace_label, log_alt_losers=log_alt_losers, \n", + " trace_column_names=trace_column_names, seed=seed)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " choices_list.append(choices)\n", + "\n", + " choices_df = pd.concat(choices_list)\n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": 454, + "id": "74eeedd3-4ade-4729-8170-79fa9dcf11f8", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:58:52.825042Z", + "iopub.status.busy": "2021-09-01T04:58:52.824768Z", + "iopub.status.idle": "2021-09-01T04:58:53.042884Z", + "shell.execute_reply": "2021-09-01T04:58:53.042061Z", + "shell.execute_reply.started": "2021-09-01T04:58:52.825004Z" + } + }, + "outputs": [], + "source": [ + "## caculate probabilities with Asim methodology, should be correct\n", + "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", + " full_probs = [] # analytical probs\n", + "\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " #choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " # segment_trace_label, log_alt_losers=log_alt_losers, \n", + " # trace_column_names=trace_column_names, seed=seed)\n", + " probs = simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, \n", + " estimator, segment_trace_label, log_alt_losers, \n", + " trace_column_names)\n", + " full_probs.append(probs)\n", + " probs_df = pd.concat(full_probs)\n", + " return probs_df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 455, + "id": "295a4fd1-8e85-47e2-af82-9f1beac31d63", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:59:46.126555Z", + "iopub.status.busy": "2021-09-01T04:59:46.126262Z", + "iopub.status.idle": "2021-09-01T04:59:52.979908Z", + "shell.execute_reply": "2021-09-01T04:59:52.979043Z", + "shell.execute_reply.started": "2021-09-01T04:59:46.126522Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2021-09-01 14:59:46.353007 Start\n", + "2021-09-01 14:59:52.976736 End\n" + ] + } + ], + "source": [ + "print(f\"{datetime.now()} Start\")\n", + "pipeline.open_pipeline(resume_after)\n", + "inject.get_injectable('preload_injectables', None)\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "trips_df = trips.to_frame()\n", + "#print(\"Running with %d trips\", trips_df.shape[0])\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "\n", + "all_choices = []\n", + "for i in range(100):\n", + " if i % 10 == 0:\n", + " print(f\"{datetime.now()} iteration {i}\")\n", + " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=None)\n", + " all_choices.append(choices_df)\n", + "all_choices = pd.concat(all_choices, axis=1)\n", + "\n", + "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", + " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", + " log_alt_losers=None, trace_column_names=None)\n", + "\n", + "# update trips table with choices (and potionally logssums)\n", + "#trips_df = trips.to_frame()\n", + "#\n", + "#assign_in_place(trips_df, choices_df)\n", + "#assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(f\"{datetime.now()} End\")" + ] + }, + { + "cell_type": "code", + "execution_count": 463, + "id": "dc87e904-16d1-44d4-81b7-cafbe4e223c3", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:01:40.901982Z", + "iopub.status.busy": "2021-09-01T05:01:40.901759Z", + "iopub.status.idle": "2021-09-01T05:01:41.337758Z", + "shell.execute_reply": "2021-09-01T05:01:41.336757Z", + "shell.execute_reply.started": "2021-09-01T05:01:40.901946Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#all_choices.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", + "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts = val_counts / all_choices.shape[1]\n", + "#val_counts = val_counts.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", + "#val_counts['prob_of_asim_choice'] = val_counts.apply(lambda x: x[x.asim], axis=1) # this is what our simulation says w.r.t. to asim choice\n", + "# for 100% and many samples should mostly agree" + ] + }, + { + "cell_type": "code", + "execution_count": 464, + "id": "09bf151c-a39a-4823-acfd-9c8ae747c338", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:01:46.119117Z", + "iopub.status.busy": "2021-09-01T05:01:46.118849Z", + "iopub.status.idle": "2021-09-01T05:01:46.354114Z", + "shell.execute_reply": "2021-09-01T05:01:46.353248Z", + "shell.execute_reply.started": "2021-09-01T05:01:46.119092Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#val_counts['prob_of_asim_choice'].hist(bins=100);" + ] + }, + { + "cell_type": "code", + "execution_count": 490, + "id": "f3b6ade9-d875-4104-b438-b53ef42e342c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:10:08.653227Z", + "iopub.status.busy": "2021-09-01T05:10:08.652978Z", + "iopub.status.idle": "2021-09-01T05:10:08.915620Z", + "shell.execute_reply": "2021-09-01T05:10:08.914513Z", + "shell.execute_reply.started": "2021-09-01T05:10:08.653189Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", + "analytical 3.152 0.852 0.0 0.0 0.0 \n", + "experiment 3.243 0.880 0.0 0.0 0.0 \n", + "\n", + " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", + "analytical 0.0 0.0 0.0 0.685 0.0 \n", + "experiment 0.0 0.0 0.0 0.772 0.0 \n", + "\n", + " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK \\\n", + "analytical 0.129 0.0 0.182 0.260 1.334 63.708 \n", + "experiment 0.156 0.0 0.174 0.259 1.434 63.243 \n", + "\n", + " WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", + "analytical 0.0 0.0 0.455 18.355 10.887 \n", + "experiment 0.0 0.0 0.394 18.639 10.807 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mode_share_analytical = (probs_nl.sum(axis=0) / probs_nl.shape[0]).to_frame('analytical')\n", + "assert np.allclose(mode_share_analytical.sum(), 1)\n", + "mode_share_rum = (val_counts.sum(axis=0) / val_counts.shape[0]).to_frame('experiment')\n", + "assert np.allclose(mode_share_rum.sum(), 1)\n", + "full_share = mode_share_analytical.join(mode_share_rum, how='outer').fillna(0)\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * full_share).T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e48a56e-d613-4dfd-bef6-62bee524f12f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50914e1d-750d-440c-9617-f96bd2a46c56", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2d70e0e-520e-49a6-8024-ae31345f6ead", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "f9e8ef02-1d57-4b21-8aef-25a6e1095c02", + "metadata": {}, + "source": [ + "### try zenith normalisation of simple_simulate_rum" + ] + }, + { + "cell_type": "code", + "execution_count": 629, + "id": "db235e90-d2e5-4e90-b41c-6d90a670e41d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:24.045881Z", + "iopub.status.busy": "2021-09-01T11:37:24.045637Z", + "iopub.status.idle": "2021-09-01T11:37:24.276488Z", + "shell.execute_reply": "2021-09-01T11:37:24.274898Z", + "shell.execute_reply.started": "2021-09-01T11:37:24.045844Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation belowi s equivalent.\n", + "def compute_nested_utilities_zenith(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) # / nest.product_of_coefficients #coefficient\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", + " \n", + " # now go over all leaves and correct for scale\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] /= nest.coefficient\n", + " \n", + " return nested_utilities\n", + "\n", + "\n", + "def simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " utils_df = compute_nested_utilities_zenith(raw_utilities, nest_spec)\n", + "\n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices\n", + "\n", + "#simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" + ] + }, + { + "cell_type": "code", + "execution_count": 630, + "id": "f69bc3dc-84e4-4dd5-9a57-c053cadba4b5", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:24.828926Z", + "iopub.status.busy": "2021-09-01T11:37:24.828707Z", + "iopub.status.idle": "2021-09-01T11:37:25.046409Z", + "shell.execute_reply": "2021-09-01T11:37:25.045399Z", + "shell.execute_reply.started": "2021-09-01T11:37:24.828902Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " segment_trace_label, log_alt_losers=log_alt_losers, \n", + " trace_column_names=trace_column_names, seed=seed)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " choices_list.append(choices)\n", + "\n", + " choices_df = pd.concat(choices_list)\n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": 631, + "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:32.250808Z", + "iopub.status.busy": "2021-09-01T11:37:32.250548Z", + "iopub.status.idle": "2021-09-01T20:20:35.257463Z", + "shell.execute_reply": "2021-09-01T20:20:35.255964Z", + "shell.execute_reply.started": "2021-09-01T11:37:32.250782Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "num_samples = 10\n", + "\n", + "rng_test = default_rng(23423)\n", + "\n", + "print(f\"{datetime.now()} Start\")\n", + "pipeline.open_pipeline(resume_after)\n", + "inject.get_injectable('preload_injectables', None)\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "trips_df = trips.to_frame()\n", + "#print(\"Running with %d trips\", trips_df.shape[0])\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "all_choices_zenith = []\n", + "all_choices = []\n", + "for i in range(num_samples):\n", + " \n", + " seed = rng_test.integers(0, 100000) #int(9.3 * (i+1)**3) # why not\n", + " \n", + " if i % 50 == 0:\n", + " print(f\"{datetime.now()} iteration {i}\")\n", + " choices_df_zenith = stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", + " all_choices_zenith.append(choices_df_zenith)\n", + "\n", + " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", + " all_choices.append(choices_df)\n", + " \n", + " t_ = choices_df_zenith.merge(choices_df, left_index=True, right_index=True, suffixes=['_zenith', '_asim'])\n", + " diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", + " \n", + " #print(f\"seed {seed} leads to {diffs.shape[0]} differences. tripids {diffs.index}\")\n", + "\n", + "all_choices_zenith = pd.concat(all_choices_zenith, axis=1)\n", + "all_choices = pd.concat(all_choices, axis=1)\n", + "\n", + "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", + " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", + " log_alt_losers=None, trace_column_names=None)\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(f\"{datetime.now()} End\")" + ] + }, + { + "cell_type": "code", + "execution_count": 643, + "id": "3abd7949-dc9c-469b-845e-26c17741f70e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:33:21.533110Z", + "iopub.status.busy": "2021-09-01T20:33:21.531915Z", + "iopub.status.idle": "2021-09-01T20:33:21.740946Z", + "shell.execute_reply": "2021-09-01T20:33:21.739713Z", + "shell.execute_reply.started": "2021-09-01T20:33:21.533074Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Have 2038 samples\n" + ] + } + ], + "source": [ + "print(f\"Have {all_choices_zenith.shape[1]} samples\")" + ] + }, + { + "cell_type": "code", + "execution_count": 633, + "id": "8762bd6d-ac04-458c-869f-be86d8297351", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:21:27.778803Z", + "iopub.status.busy": "2021-09-01T20:21:27.778531Z", + "iopub.status.idle": "2021-09-01T20:21:28.675645Z", + "shell.execute_reply": "2021-09-01T20:21:28.674734Z", + "shell.execute_reply.started": "2021-09-01T20:21:27.778754Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "val_counts_zenith = all_choices_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts_zenith = val_counts_zenith / all_choices_zenith.shape[1]\n", + "\n", + "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts = val_counts / all_choices.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 634, + "id": "0f566105-9424-457b-86f5-d60c1d63aac5", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:21:30.954894Z", + "iopub.status.busy": "2021-09-01T20:21:30.954654Z", + "iopub.status.idle": "2021-09-01T20:21:31.179689Z", + "shell.execute_reply": "2021-09-01T20:21:31.178304Z", + "shell.execute_reply.started": "2021-09-01T20:21:30.954867Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
experiment_zenith3.1500.8740.00.00.00.00.00.00.6800.00.1360.00.1540.2361.28563.7670.00.00.51518.82410.379
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", + "analytical 3.152 0.852 0.0 0.0 0.0 \n", + "experiment 3.243 0.880 0.0 0.0 0.0 \n", + "experiment_zenith 3.150 0.874 0.0 0.0 0.0 \n", + "\n", + " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", + "analytical 0.0 0.0 0.0 0.685 0.0 \n", + "experiment 0.0 0.0 0.0 0.772 0.0 \n", + "experiment_zenith 0.0 0.0 0.0 0.680 0.0 \n", + "\n", + " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE \\\n", + "analytical 0.129 0.0 0.182 0.260 1.334 \n", + "experiment 0.156 0.0 0.174 0.259 1.434 \n", + "experiment_zenith 0.136 0.0 0.154 0.236 1.285 \n", + "\n", + " WALK WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", + "analytical 63.708 0.0 0.0 0.455 18.355 10.887 \n", + "experiment 63.243 0.0 0.0 0.394 18.639 10.807 \n", + "experiment_zenith 63.767 0.0 0.0 0.515 18.824 10.379 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mode_share_rum_zenith = (val_counts_zenith.sum(axis=0) / val_counts_zenith.shape[0]).to_frame('experiment_zenith')\n", + "assert np.allclose(mode_share_rum_zenith.sum(), 1)\n", + "full_share_incl_zenith = full_share.merge(mode_share_rum_zenith, left_index=True, right_index=True, how='outer').fillna(0)\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * full_share_incl_zenith).T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c0d3081-b45b-4573-9246-e4a9d39591dd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "612b2f04-45e2-4eb2-b52a-9f753427877f", + "metadata": {}, + "source": [ + "## investigate diverging seed and look at diff in formulation" + ] + }, + { + "cell_type": "code", + "execution_count": 619, + "id": "ac09dc75-a92f-43fb-a0cd-fe7de1f8b66d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:12:02.988141Z", + "iopub.status.busy": "2021-09-01T11:12:02.987900Z", + "iopub.status.idle": "2021-09-01T11:12:03.212179Z", + "shell.execute_reply": "2021-09-01T11:12:03.210993Z", + "shell.execute_reply.started": "2021-09-01T11:12:02.988102Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation based on asim probabilities is equivalent\n", + "def compute_nested_utilities_asim(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) / nest.product_of_coefficients\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", + " return nested_utilities\n", + "\n", + "def compute_nested_utilities_zenith_check(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) # scale correction is below\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", + " \n", + " # now go over all leaves and correct for scale\n", + " for nest in logit.each_nest(nest_spec):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] /= nest.coefficient\n", + " \n", + " return nested_utilities" + ] + }, + { + "cell_type": "code", + "execution_count": 688, + "id": "4b7c6368-21ca-4442-969e-f2f8ff868842", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:49:56.958862Z", + "iopub.status.busy": "2021-09-01T20:49:56.958623Z", + "iopub.status.idle": "2021-09-01T20:49:57.191884Z", + "shell.execute_reply": "2021-09-01T20:49:57.190975Z", + "shell.execute_reply.started": "2021-09-01T20:49:56.958827Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None, use_zenith=True, raw_utilities=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " \n", + " if raw_utilities is None:\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " if use_zenith:\n", + " utils_df = compute_nested_utilities_zenith_check(raw_utilities, nest_spec)\n", + " else:\n", + " utils_df = compute_nested_utilities_asim(raw_utilities, nest_spec)\n", + "\n", + " # test\n", + " #return compute_nested_utilities_zenith_check(raw_utilities, nest_spec), compute_nested_utilities_asim(raw_utilities, nest_spec)\n", + " \n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices" + ] + }, + { + "cell_type": "code", + "execution_count": 626, + "id": "bd6cadc3-79f5-4824-b55e-639c39d5ab18", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:13:12.077886Z", + "iopub.status.busy": "2021-09-01T11:13:12.077652Z", + "iopub.status.idle": "2021-09-01T11:13:12.290222Z", + "shell.execute_reply": "2021-09-01T11:13:12.288861Z", + "shell.execute_reply.started": "2021-09-01T11:13:12.077863Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# t_z, t_a = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed)\n", + "# tr_id = 86673661\n", + "# display(t_z.loc[t_z.index==tr_id])\n", + "# display(t_a.loc[t_a.index==tr_id])" + ] + }, + { + "cell_type": "code", + "execution_count": 625, + "id": "42e6da62-58e8-4b16-9169-9faa1fc8a162", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:13:06.847583Z", + "iopub.status.busy": "2021-09-01T11:13:06.847204Z", + "iopub.status.idle": "2021-09-01T11:13:08.401700Z", + "shell.execute_reply": "2021-09-01T11:13:08.400711Z", + "shell.execute_reply.started": "2021-09-01T11:13:06.847554Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seed 51006 leads to 34 differences out of 168. tripids Int64Index([ 86673657, 86673661, 106741681, 106741682, 106741685,\n", + " 106741686, 106741687, 106741688, 211327433, 211327437,\n", + " 444793574, 484173905, 484173909, 535170694, 535620053,\n", + " 708171014, 943749470, 943749471, 1060575853, 1091770617,\n", + " 1146472489, 1146472493, 1276281769, 1276281773, 1658748793,\n", + " 1658748797, 1767013726, 1767186577, 1767186578, 1768237161,\n", + " 1768237165, 1768237166, 2463663417, 2463663421],\n", + " dtype='int64', name='trip_id')\n" + ] + } + ], + "source": [ + "seed = 51006\n", + "x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True)\n", + "y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False)\n", + "t_ = x_.to_frame('trip_mode_zenith').merge(y_.to_frame('trip_mode_asim'), left_index=True, right_index=True)\n", + "diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", + "print(f\"seed {seed} leads to {diffs.shape[0]} differences out of {t_.shape[0]}. tripids {diffs.index}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f3c0506-a79d-4a31-9d56-71b9966d0a92", + "metadata": {}, + "outputs": [], + "source": [ + "seed 51006 leads to 3 differences. tripids Int64Index([86673661, 535170689, 1060575849], dtype='int64', name='trip_id')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a5f332a-b1d7-4a80-bb21-d37a1611cf9f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24c5e4a1-6142-49d4-8da8-a1e266b3cb9b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 753, + "id": "2098907f-28af-4f53-af74-6aaee8a6c53c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T23:04:23.861048Z", + "iopub.status.busy": "2021-09-01T23:04:23.860748Z", + "iopub.status.idle": "2021-09-01T23:04:24.069307Z", + "shell.execute_reply": "2021-09-01T23:04:24.068434Z", + "shell.execute_reply.started": "2021-09-01T23:04:23.861024Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "rng_ = default_rng(seed=100)\n", + "new_utils = pd.DataFrame(0.1 * -np.log(-np.log(rng_.uniform(0,1,raw_utilities.shape))), columns=raw_utilities.columns)\n", + "new_utils.index = raw_utilities.index" + ] + }, + { + "cell_type": "code", + "execution_count": 767, + "id": "da48a178-82b2-44a5-b45a-e9de3bb5e5c2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T23:17:20.369675Z", + "iopub.status.busy": "2021-09-01T23:17:20.369424Z", + "iopub.status.idle": "2021-09-01T23:17:20.666506Z", + "shell.execute_reply": "2021-09-01T23:17:20.665626Z", + "shell.execute_reply.started": "2021-09-01T23:17:20.369637Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
1372487210.0566420.0373050.0427800.0295760.0861540.0265860.0883130.1003310.0280210.0185160.0643910.0200090.0263990.0226220.0230070.0217540.0871260.0211710.0364750.0276500.135171
1372487250.1019020.0228760.0312270.0699900.0569130.0466850.0884110.0877850.0254140.0381620.0239880.0288000.0393180.0201600.0483640.0230030.0339590.0345970.0377080.0995230.041213
2113882010.0375620.0700330.0559510.0403890.0200410.0959740.0946780.0943680.0360140.0250510.0348600.0349530.0354990.0280400.0436500.0247130.0313880.0259310.0669990.0526780.051228
2113882050.0354800.0562560.0439440.0480850.0420780.0622210.1068190.0872650.0351170.0303990.0351540.0264690.0506460.0357270.0242840.0329050.0394270.0410920.0547140.0484590.063458
8063884010.0549180.0404880.0331490.0542940.0434770.0679820.1524650.0741520.0334040.0287350.0592690.0243390.0369680.0277600.0235220.0609320.0234450.0239310.0446430.0337730.058353
8063884050.0327640.0573010.0184370.0770990.0469630.0635210.0949510.0890720.0308260.0378960.0357520.0198780.0495360.0796100.0269220.0248330.0248400.0266950.0799360.0401460.043022
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n", + "trip_id \n", + "137248721 0.056642 0.037305 0.042780 0.029576 \n", + "137248725 0.101902 0.022876 0.031227 0.069990 \n", + "211388201 0.037562 0.070033 0.055951 0.040389 \n", + "211388205 0.035480 0.056256 0.043944 0.048085 \n", + "806388401 0.054918 0.040488 0.033149 0.054294 \n", + "806388405 0.032764 0.057301 0.018437 0.077099 \n", + "\n", + " SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\n", + "trip_id \n", + "137248721 0.086154 0.026586 0.088313 0.100331 0.028021 0.018516 \n", + "137248725 0.056913 0.046685 0.088411 0.087785 0.025414 0.038162 \n", + "211388201 0.020041 0.095974 0.094678 0.094368 0.036014 0.025051 \n", + "211388205 0.042078 0.062221 0.106819 0.087265 0.035117 0.030399 \n", + "806388401 0.043477 0.067982 0.152465 0.074152 0.033404 0.028735 \n", + "806388405 0.046963 0.063521 0.094951 0.089072 0.030826 0.037896 \n", + "\n", + " WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\n", + "trip_id \n", + "137248721 0.064391 0.020009 0.026399 0.022622 0.023007 0.021754 \n", + "137248725 0.023988 0.028800 0.039318 0.020160 0.048364 0.023003 \n", + "211388201 0.034860 0.034953 0.035499 0.028040 0.043650 0.024713 \n", + "211388205 0.035154 0.026469 0.050646 0.035727 0.024284 0.032905 \n", + "806388401 0.059269 0.024339 0.036968 0.027760 0.023522 0.060932 \n", + "806388405 0.035752 0.019878 0.049536 0.079610 0.026922 0.024833 \n", + "\n", + " DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n", + "trip_id \n", + "137248721 0.087126 0.021171 0.036475 0.027650 0.135171 \n", + "137248725 0.033959 0.034597 0.037708 0.099523 0.041213 \n", + "211388201 0.031388 0.025931 0.066999 0.052678 0.051228 \n", + "211388205 0.039427 0.041092 0.054714 0.048459 0.063458 \n", + "806388401 0.023445 0.023931 0.044643 0.033773 0.058353 \n", + "806388405 0.024840 0.026695 0.079936 0.040146 0.043022 " + ] + }, + "execution_count": 767, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nested_exp_utilities_cf = simulate.compute_nested_exp_utilities(new_utils, nest_spec)\n", + "nested_probabilities_cf = simulate.compute_nested_probabilities(nested_exp_utilities_cf, nest_spec, trace_label=None)\n", + "base_probabilities_cf = simulate.compute_base_probabilities(nested_probabilities_cf, nest_spec, spec)\n", + "base_probabilities_cf" + ] + }, + { + "cell_type": "code", + "execution_count": 799, + "id": "50dfd1ae-10c3-475c-94be-de783c2fa5c2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:15:29.056902Z", + "iopub.status.busy": "2021-09-02T00:15:29.056683Z", + "iopub.status.idle": "2021-09-02T00:27:21.965206Z", + "shell.execute_reply": "2021-09-02T00:27:21.964340Z", + "shell.execute_reply.started": "2021-09-02T00:15:29.056877Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 11min 49s, sys: 766 ms, total: 11min 50s\n", + "Wall time: 11min 52s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "num_samples = 10000 # 7.5s per 100\n", + "\n", + "data_zenith = []\n", + "data_asim = []\n", + "for i in range(num_samples):\n", + " seed = rng_.integers(0, 100000)\n", + " x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True, raw_utilities=new_utils)\n", + " y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False, raw_utilities=new_utils)\n", + " data_zenith.append(x_)\n", + " data_asim.append(y_)\n", + " \n", + "data_asim = pd.concat(data_asim, axis=1)\n", + "data_zenith = pd.concat(data_zenith, axis=1)\n", + "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", + "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_asim = counts_asim / data_asim.shape[1]\n", + "\n", + "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", + "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", + "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", + "# assert np.allclose(mode_share_zenith.sum(), 1)\n", + "# assert np.allclose(mode_share_asim.sum(), 1)\n", + "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", + "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 796, + "id": "bb16f9d7-44f5-4316-b981-d65216a5e217", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:14:11.772592Z", + "iopub.status.busy": "2021-09-02T00:14:11.772347Z", + "iopub.status.idle": "2021-09-02T00:14:12.001077Z", + "shell.execute_reply": "2021-09-02T00:14:12.000166Z", + "shell.execute_reply.started": "2021-09-02T00:14:11.772556Z" + } + }, + "outputs": [], + "source": [ + "#temp_z = data_zenith.copy()\n", + "#temp_a = data_asim.copy()\n", + "#data_asim = data_asim.join(temp_a, lsuffix=\"_o\", rsuffix=\"_n\")\n", + "#data_zenith = data_zenith.join(temp_z, lsuffix=\"_o\", rsuffix=\"_n\")\n", + "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", + "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_asim = counts_asim / data_asim.shape[1]\n", + "\n", + "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", + "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", + "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", + "# assert np.allclose(mode_share_zenith.sum(), 1)\n", + "# assert np.allclose(mode_share_asim.sum(), 1)\n", + "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", + "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 805, + "id": "031d8340-0af0-4608-8a83-ed2af4aaa3b2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:41:01.948669Z", + "iopub.status.busy": "2021-09-02T00:41:01.948422Z", + "iopub.status.idle": "2021-09-02T00:41:02.175972Z", + "shell.execute_reply": "2021-09-02T00:41:02.174759Z", + "shell.execute_reply.started": "2021-09-02T00:41:01.948633Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
zenith8.0555.1674.6313.5023.6064.2663.9633.4983.8955.1764.7995.8514.9996.1214.6689.4864.2284.1252.9773.6713.317
asim6.0966.6984.9773.2023.5224.3344.1293.1843.2245.1496.2838.2964.7936.5844.5327.4844.1524.2732.6293.4333.023
probs8.8835.3214.7382.8903.1364.0033.5653.1623.7585.3244.9276.0495.3416.5415.03710.4273.9734.2242.5743.1472.979
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP DRIVE_HVY \\\n", + "zenith 8.055 5.167 4.631 3.502 3.606 4.266 \n", + "asim 6.096 6.698 4.977 3.202 3.522 4.334 \n", + "probs 8.883 5.321 4.738 2.890 3.136 4.003 \n", + "\n", + " DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY SHARED3FREE \\\n", + "zenith 3.963 3.498 3.895 5.176 4.799 \n", + "asim 4.129 3.184 3.224 5.149 6.283 \n", + "probs 3.565 3.162 3.758 5.324 4.927 \n", + "\n", + " SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK WALK_COM WALK_EXP \\\n", + "zenith 5.851 4.999 6.121 4.668 9.486 4.228 4.125 \n", + "asim 8.296 4.793 6.584 4.532 7.484 4.152 4.273 \n", + "probs 6.049 5.341 6.541 5.037 10.427 3.973 4.224 \n", + "\n", + " WALK_HVY WALK_LOC WALK_LRF \n", + "zenith 2.977 3.671 3.317 \n", + "asim 2.629 3.433 3.023 \n", + "probs 2.574 3.147 2.979 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# sample of 20000\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * mode_share_comp).T)" + ] + }, + { + "cell_type": "code", + "execution_count": 806, + "id": "aece448f-4055-4d2a-b90d-8530caec9cb9", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:43:32.971972Z", + "iopub.status.busy": "2021-09-02T00:43:32.971684Z", + "iopub.status.idle": "2021-09-02T00:43:33.223345Z", + "shell.execute_reply": "2021-09-02T00:43:33.222488Z", + "shell.execute_reply.started": "2021-09-02T00:43:32.971932Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
trip_id137248721_z137248721_a137248721
BIKE0.092700.054350.100331
DRIVEALONEFREE0.053650.050050.056642
DRIVEALONEPAY0.038800.031550.037305
DRIVE_COM0.028700.024700.021171
DRIVE_EXP0.028800.026050.021754
DRIVE_HVY0.078150.099700.087126
DRIVE_LOC0.028750.027000.022622
DRIVE_LRF0.027800.024450.023007
SHARED2FREE0.043250.023000.042780
SHARED2PAY0.033600.015850.029576
SHARED3FREE0.073650.119200.086154
SHARED3PAY0.032600.039600.026586
TAXI0.035100.050300.036475
TNC_SHARED0.126200.181150.135171
TNC_SINGLE0.026250.038450.027650
WALK0.079400.048250.088313
WALK_COM0.031850.024600.026399
WALK_EXP0.057050.058950.064391
WALK_HVY0.025200.018400.020009
WALK_LOC0.034550.027400.028021
WALK_LRF0.023950.017000.018516
\n", + "
" + ], + "text/plain": [ + "trip_id 137248721_z 137248721_a 137248721\n", + "BIKE 0.09270 0.05435 0.100331\n", + "DRIVEALONEFREE 0.05365 0.05005 0.056642\n", + "DRIVEALONEPAY 0.03880 0.03155 0.037305\n", + "DRIVE_COM 0.02870 0.02470 0.021171\n", + "DRIVE_EXP 0.02880 0.02605 0.021754\n", + "DRIVE_HVY 0.07815 0.09970 0.087126\n", + "DRIVE_LOC 0.02875 0.02700 0.022622\n", + "DRIVE_LRF 0.02780 0.02445 0.023007\n", + "SHARED2FREE 0.04325 0.02300 0.042780\n", + "SHARED2PAY 0.03360 0.01585 0.029576\n", + "SHARED3FREE 0.07365 0.11920 0.086154\n", + "SHARED3PAY 0.03260 0.03960 0.026586\n", + "TAXI 0.03510 0.05030 0.036475\n", + "TNC_SHARED 0.12620 0.18115 0.135171\n", + "TNC_SINGLE 0.02625 0.03845 0.027650\n", + "WALK 0.07940 0.04825 0.088313\n", + "WALK_COM 0.03185 0.02460 0.026399\n", + "WALK_EXP 0.05705 0.05895 0.064391\n", + "WALK_HVY 0.02520 0.01840 0.020009\n", + "WALK_LOC 0.03455 0.02740 0.028021\n", + "WALK_LRF 0.02395 0.01700 0.018516" + ] + }, + "execution_count": 806, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# TODO: look at probs per trip, i.e. do not sum across trips\n", + "trip_id = 137248721\n", + "\n", + "counts_zenith.loc[counts_zenith.index == trip_id].T.merge(\n", + " counts_asim.loc[counts_asim.index == trip_id].T, suffixes=['_z', '_a'], left_index=True, right_index=True, how='outer').merge(\n", + " base_probabilities_cf.loc[base_probabilities_cf.index == trip_id].T, suffixes=['', '_probs'], left_index=True, right_index=True, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ef84f10-d616-40e4-87e5-425499027ff3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "431cee49-40a9-4dc7-93a5-75308fb302dd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fcdc0ff-c510-4878-8672-8365e046442c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/notebooks/nested_logit_frozen_rand_individ_util.ipynb b/notebooks/nested_logit_frozen_rand_individ_util.ipynb new file mode 100644 index 0000000000..2bb10981e5 --- /dev/null +++ b/notebooks/nested_logit_frozen_rand_individ_util.ipynb @@ -0,0 +1,463 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Frozen randomness\n", + "\n", + "Let's try to draw from nested logit model by drawing error terms. For logit models, this is simple because error terms are independent and therefore we can uniquely invert the CDF and simply draw from that. For nested models, this is not the case. However, we know we can write the probabilities as nested logits and therefore we think we can draw repeatedly like for logit models, taking the nesting structure into account.\n", + "\n", + "Let's start with two levels and a model where we know the probabilities, i.e. we fix the utility functions and the nesting scales, like for red bus/blue bus. We then draw like ActivitySim does, and like we want to do.\n", + "\n", + "Next, we extend to three levels, where the additional nest error term has not been derived yet." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T05:31:19.564033Z", + "iopub.status.busy": "2022-05-01T05:31:19.563663Z", + "iopub.status.idle": "2022-05-01T05:31:19.824207Z", + "shell.execute_reply": "2022-05-01T05:31:19.823432Z", + "shell.execute_reply.started": "2022-05-01T05:31:19.563960Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "import numpy as np\n", + "from numpy.random import default_rng" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T05:31:20.023960Z", + "iopub.status.busy": "2022-05-01T05:31:20.023246Z", + "iopub.status.idle": "2022-05-01T05:31:20.029538Z", + "shell.execute_reply": "2022-05-01T05:31:20.028264Z", + "shell.execute_reply.started": "2022-05-01T05:31:20.023927Z" + } + }, + "outputs": [], + "source": [ + "def logsum(utilities, nest_scale=1.0):\n", + " scaled_utils = utilities / nest_scale\n", + " max_util = np.max(scaled_utils)\n", + " return max_util + np.log(np.sum(np.exp(scaled_utils - max_util)))\n", + "\n", + "\n", + "# total probability of alternative being chosen is product of two terms:\n", + "# conditional probability of alternative given nest is chosen: exp(util / nest_scale)\n", + "# marginal probability that any alternative in nest is chosen: exp(nest_scale * logsum)\n", + " \n", + "# If you think about a single case, the probabilities are indicator variables and we take the max of each. This is what Zenith does I think.\n", + "# Given that these expressions are those of two logits, it seems natural to draw correspondingly.\n", + "# This must be related to the strict derivation of max() distributions of Hunt." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Two-level" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T09:55:14.957470Z", + "iopub.status.busy": "2022-05-01T09:55:14.957183Z", + "iopub.status.idle": "2022-05-01T09:55:14.964046Z", + "shell.execute_reply": "2022-05-01T09:55:14.963056Z", + "shell.execute_reply.started": "2022-05-01T09:55:14.957441Z" + } + }, + "outputs": [], + "source": [ + "alternatives = {1: \"car\", 2: \"blue bus\", 3: \"red bus\"}\n", + "\n", + "utility_spec = {\n", + " 1: {\"cost\": -1.0, \"asc\": 0.0},\n", + " 2: {\"cost\": -1.0, \"asc\": 0.2},\n", + " 3: {\"cost\": -1.5, \"asc\": 0.1},\n", + "}\n", + "\n", + "# blue and red bus are nested together with scale 0.5\n", + "\n", + "def utility(x, utility_spec, alternative):\n", + " assert alternative in utility_spec.keys()\n", + " return utility_spec[alternative][\"cost\"] * x + utility_spec[alternative][\"asc\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T09:55:15.467608Z", + "iopub.status.busy": "2022-05-01T09:55:15.467366Z", + "iopub.status.idle": "2022-05-01T09:55:15.478279Z", + "shell.execute_reply": "2022-05-01T09:55:15.476833Z", + "shell.execute_reply.started": "2022-05-01T09:55:15.467571Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.4452265282367507 0.5330453677531714 0.02172810401007798\n", + "1.0\n" + ] + } + ], + "source": [ + "cost = 3.0\n", + "nest_scale = 0.5\n", + "\n", + "util_3 = utility(cost, utility_spec, 3)\n", + "util_2 = utility(cost, utility_spec, 2)\n", + "logsum_bus = logsum(np.array([util_2, util_3]), nest_scale=nest_scale)\n", + "nest_util = nest_scale * logsum_bus\n", + "\n", + "util_1 = utility(cost, utility_spec, 1)\n", + "prob_1 = np.exp(util_1) / (np.exp(util_1) + np.exp(nest_util))\n", + "\n", + "nest_cond_prob = np.exp(nest_util) / (np.exp(util_1) + np.exp(nest_util))\n", + "nest_marg_prob_2 = np.exp(util_2 / nest_scale) / (np.exp(util_2 / nest_scale) + np.exp(util_3 / nest_scale))\n", + "nest_marg_prob_3 = np.exp(util_3 / nest_scale) / (np.exp(util_2 / nest_scale) + np.exp(util_3 / nest_scale))\n", + "\n", + "prob_2 = nest_cond_prob * nest_marg_prob_2\n", + "prob_3 = nest_cond_prob * nest_marg_prob_3\n", + "\n", + "print(prob_1, prob_2, prob_3)\n", + "print(sum([prob_1, prob_2, prob_3]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The ActivitySim way\n", + "Kind of pointless here but this is how we choose a single value - draw from U and pick whichever interval it falls into" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T09:55:20.523480Z", + "iopub.status.busy": "2022-05-01T09:55:20.523247Z", + "iopub.status.idle": "2022-05-01T09:55:21.341578Z", + "shell.execute_reply": "2022-05-01T09:55:21.340682Z", + "shell.execute_reply.started": "2022-05-01T09:55:20.523453Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "closed form: [0.4452265282367507, 0.5330453677531714, 0.02172810401007798],\n", + "simulated: [0.4450967 0.5331544 0.0217489]\n", + "CPU times: user 766 ms, sys: 46.9 ms, total: 812 ms\n", + "Wall time: 814 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "probs = [prob_1, prob_2, prob_3]\n", + "cum_probs = [0] + list(np.cumsum(probs))\n", + "\n", + "num_draws = 10000000\n", + "\n", + "# now draw from U and put into arrays, then value count?\n", + "rng = default_rng(999)\n", + "rands = rng.uniform(size=num_draws)\n", + "\n", + "hits, bins = np.histogram(rands, bins=cum_probs)\n", + "print(f\"closed form: {probs},\\nsimulated: {hits / num_draws}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T09:55:24.027745Z", + "iopub.status.busy": "2022-05-01T09:55:24.027421Z", + "iopub.status.idle": "2022-05-01T09:55:24.034035Z", + "shell.execute_reply": "2022-05-01T09:55:24.031703Z", + "shell.execute_reply.started": "2022-05-01T09:55:24.027708Z" + } + }, + "outputs": [], + "source": [ + "def inverse_ev1_cdf(x, location=0.0, scale=1.0):\n", + " \"quantile function of EV1\"\n", + " #return location - (1.0 / scale) * np.log(-np.log(x))\n", + " # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution where the scale is proportional to variance (not variance^{-1})\n", + " return location - scale * np.log(-np.log(x))\n", + "\n", + "# for utilities with full set of ascs location=0. Do we always assume location=0 in estimation anyways?\n", + "# the scale of the error term is unidentified and therefore set to 1 in most applications, meaning the standard deviation is pi/sqrt(6)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "execution": { + "iopub.execute_input": "2022-04-17T03:38:03.583638Z", + "iopub.status.busy": "2022-04-17T03:38:03.583273Z", + "iopub.status.idle": "2022-04-17T03:38:03.588839Z", + "shell.execute_reply": "2022-04-17T03:38:03.586978Z", + "shell.execute_reply.started": "2022-04-17T03:38:03.583594Z" + } + }, + "source": [ + "## The Zenith way\n", + "\n", + "Basically, probabilities are now indicators - choose at the lowest level, then walk up. Choice is product of these. We draw error terms for each alternative, where nest roots are now alternatives too.\n", + "\n", + "OR: do we choose a" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:02:14.035272Z", + "iopub.status.busy": "2022-05-01T10:02:14.035014Z", + "iopub.status.idle": "2022-05-01T10:02:14.040376Z", + "shell.execute_reply": "2022-05-01T10:02:14.039207Z", + "shell.execute_reply.started": "2022-05-01T10:02:14.035235Z" + } + }, + "outputs": [], + "source": [ + "util_3 = utility(cost, utility_spec, 3)\n", + "util_2 = utility(cost, utility_spec, 2)\n", + "logsum_bus = logsum(np.array([util_2, util_3]), nest_scale=nest_scale)\n", + "util_1 = utility(cost, utility_spec, 1)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:03:23.999857Z", + "iopub.status.busy": "2022-05-01T10:03:23.999571Z", + "iopub.status.idle": "2022-05-01T10:03:30.033848Z", + "shell.execute_reply": "2022-05-01T10:03:30.032689Z", + "shell.execute_reply.started": "2022-05-01T10:03:23.999816Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "closed form probs: 0.445227, 0.533045, 0.021728\n", + " simulated probs: 0.4452013, 0.5330709, 0.0217278\n" + ] + } + ], + "source": [ + "# conditional error term are given by logit with scale given by nest scale\n", + "num_draws_dec = 10000000\n", + "#mu = 1.0 / nest_scale\n", + "\n", + "rng_dec = default_rng(9)\n", + "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", + "ev1_lower = inverse_ev1_cdf(rands_dec, scale=nest_scale)\n", + "\n", + "lower_utils_2 = util_2 + ev1_lower[num_draws_dec:] \n", + "lower_utils_3 = util_3 + ev1_lower[:num_draws_dec] \n", + "\n", + "#logsum_bus = logsum(np.array([lower_utils_2, lower_utils_3]), nest_scale=nest_scale)\n", + "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=2*num_draws_dec))\n", + "nest_util = nest_scale * logsum_bus + ev1_upper[num_draws_dec:]\n", + "\n", + "full_util_1 = util_1 + ev1_upper[:num_draws_dec]\n", + "\n", + "choices = np.array([full_util_1, nest_util]).argmax(axis=0)\n", + "nest_indexes = np.nonzero(choices == 1)[0]\n", + "nest_choices = np.array([lower_utils_2[nest_indexes], lower_utils_3[nest_indexes]]).argmax(axis=0)\n", + "nest_choices += 1\n", + "choices = np.append(choices[choices == 0], nest_choices)\n", + "\n", + "vals, counts = np.unique(choices, return_counts=True)\n", + "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", + "\n", + "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", + "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### the error term decomposition way\n", + "\n", + "We can decompose the error term into one for the nest and one within nests. For the example structure here, do we need to introduce artificial levels? In other terms, does car and bus nest have a common component?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "observation: when setting all utilities equal and nest scale to 0.999, the probs should be nearly exacly equal 1/3, but the probs here are about 0.5 to 0.25,0.25 which is what you get when the nest scale is very small (i.e. strongly correlated).\n", + "\n", + "When the 1/scale in quantile function is turned into scale, the values are slightly off but much closer (0.368 to 0.3165,0.3165)\n", + "\n", + "# => check scale and 1/scale everywhere" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T09:55:25.164028Z", + "iopub.status.busy": "2022-05-01T09:55:25.163822Z", + "iopub.status.idle": "2022-05-01T09:55:25.473704Z", + "shell.execute_reply": "2022-05-01T09:55:25.472525Z", + "shell.execute_reply.started": "2022-05-01T09:55:25.164005Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "closed form probs: 0.445227, 0.533045, 0.021728\n", + " simulated probs: 0.284564, 0.493765, 0.221671\n", + "CPU times: user 281 ms, sys: 0 ns, total: 281 ms\n", + "Wall time: 303 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "# conditional error term are given by logit with scale given by nest scale\n", + "num_draws_dec = 1000000\n", + "\n", + "mu = 1.0 / nest_scale\n", + "\n", + "rng_dec = default_rng(9)\n", + "\n", + "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", + "ev1_dec = inverse_ev1_cdf(rands_dec, scale=mu)\n", + "\n", + "lower_level_utils_2 = np.repeat(util_2, num_draws_dec) + ev1_dec[num_draws_dec:]\n", + "lower_level_utils_3 = np.repeat(util_3, num_draws_dec) + ev1_dec[:num_draws_dec]\n", + "\n", + "\n", + "location_nest = - np.log(2.0) / mu\n", + "#location_nest = (- np.log(2.0) / mu) - ((1.0 - 1.0 / (mu + 1.0)) * 0.57721 * mu / (mu**2 - 1.0))\n", + "#print(location_nest, - np.log(2.0) / mu)\n", + "scale_nest = mu**2 / np.sqrt(mu**2 - 1.0)\n", + "nest_error_terms = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec), location=location_nest, scale=scale_nest)\n", + "\n", + "full_utils_2 = lower_level_utils_2 + nest_error_terms\n", + "full_utils_3 = lower_level_utils_3 + nest_error_terms\n", + "\n", + "# what's the distribution of error term for car? it's a degenerate nest, so bottom level is 1\n", + "# this here agrees with Bhat and Koppelman's mode choice script.\n", + "full_utils_1 = util_1 + inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec))\n", + "\n", + "choices = np.array([full_utils_1, full_utils_2, full_utils_3]).argmax(axis=0)\n", + "vals, counts = np.unique(choices, return_counts=True)\n", + "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", + "\n", + "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", + "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# WHY IS THIS NOT EQUAL\n", + "\n", + "see p.12 of Hildebrandt: scale in E(nu_m) and Var(nu_m) is not equal? Does this need correction factor for location?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Three-level\n", + "\n", + "and recursive - maybe use Asim structure directly?\n", + "\n", + "\n", + "Could also use larch to apply models, would be great to add there too?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From c973be8130388a7bf061f4d58c079518bcfdcefc Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 4 May 2022 15:52:43 +1000 Subject: [PATCH 002/135] prototype work - return leaf and node (exp max) utilities, nest spec --- activitysim/core/logit.py | 1 + activitysim/core/simulate.py | 40 + notebooks/frozen_rand_mode_choice_dev.ipynb | 2291 ++++++----------- ...ested_logit_frozen_rand_individ_util.ipynb | 89 +- 4 files changed, 864 insertions(+), 1557 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index b3dace2e0b..0359baab90 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -423,6 +423,7 @@ def _each_nest(spec, parent_nest, post_order): nest.level = parent_nest.level + 1 nest.product_of_coefficients = parent_nest.product_of_coefficients nest.ancestors = parent_nest.ancestors + [name] + nest.coefficient = parent_nest.coefficient yield spec, nest diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index c14939da95..6eff3c2e9f 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -709,6 +709,46 @@ def _check_for_variability(expression_values, trace_label): logger.warning("%s: %s columns have missing values" % (trace_label, has_missing_vals)) +def compute_nested_utilities(raw_utilities, nest_spec): + """ + compute nest utilities based on nesting coefficients + + For nest nodes this is the logsum of alternatives adjusted by nesting coefficient + + leaf <- raw_utility / nest_coefficient + nest <- ln(sum of exponentiated raw_utility of leaves) * nest_coefficient) + + Parameters + ---------- + raw_utilities : pandas.DataFrame + dataframe with the raw alternative utilities of all leaves + (what in non-nested logit would be the utilities of all the alternatives) + nest_spec : dict + Nest tree dict from the model spec yaml file + + Returns + ------- + nested_utilities : pandas.DataFrame + Will have the index of `raw_utilities` and columns for leaf and node utilities + """ + nested_utilities = pd.DataFrame(index=raw_utilities.index) + + for nest in logit.each_nest(nest_spec, post_order=True): + name = nest.name + if nest.is_leaf: + nested_utilities[name] = \ + raw_utilities[name].astype(float) / nest.coefficient #nest.product_of_coefficients + else: + # the alternative nested_utilities will already have been computed due to post_order + # this will RuntimeWarning: divide by zero encountered in log + # if all nest alternative utilities are zero and produce -inf + with np.errstate(divide='ignore'): + nested_utilities[name] = \ + nest.coefficient * np.log((np.exp(nested_utilities[nest.alternatives])).sum(axis=1)) + + return nested_utilities + + def compute_nested_exp_utilities(raw_utilities, nest_spec): """ compute exponentiated nest utilities based on nesting coefficients diff --git a/notebooks/frozen_rand_mode_choice_dev.ipynb b/notebooks/frozen_rand_mode_choice_dev.ipynb index f4203e92a6..b0228ab13f 100644 --- a/notebooks/frozen_rand_mode_choice_dev.ipynb +++ b/notebooks/frozen_rand_mode_choice_dev.ipynb @@ -7,7 +7,11 @@ "source": [ "# Start work here, clean up as you go\n", "\n", - "Problem below: scale of error term on lower levels needs to be given by nest, not the case atm" + "Problem below: scale of error term on lower levels needs to be given by nest, not the case atm\n", + "\n", + "Do I remember this correctly and probabilities are calculated as products of marginal and conditional probabilities?\n", + "if so, the corresponding utilities at leaf and node levels would need to be calculated, and I would be able to use\n", + "these directly, right? CHECK, would make it much easier!" ] }, { @@ -280,7 +284,7 @@ " }\n", "\n", " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - " nest_spec = config.get_logit_model_settings(model_settings)\n", + " nest_specs = config.get_logit_model_settings(model_settings)\n", "\n", " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", "\n", @@ -313,7 +317,7 @@ " ################ Replace wrapper function\n", " # choices = mode_choice_simulate(...)\n", " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", " choices = simulate_function(\n", " choosers=trips_segment,\n", " spec=spec,\n", @@ -371,7 +375,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -381,82 +385,267 @@ "shell.execute_reply": "2021-09-01T02:29:01.222865Z", "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" }, + "tags": [], + "pycharm": { + "is_executing": true + } + }, + "outputs": [], + "source": [ + "trips_df = run_trip_mode_choice()" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "7eae2854-38bd-4da0-a161-3a15bbd17177", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:30:57.562848Z", + "iopub.status.busy": "2021-08-31T10:30:57.562615Z", + "iopub.status.idle": "2021-08-31T10:30:57.806655Z", + "shell.execute_reply": "2021-08-31T10:30:57.805135Z", + "shell.execute_reply.started": "2021-08-31T10:30:57.562824Z" + }, "tags": [] }, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'atwork' (27 trips)\n" - ] + "data": { + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n8684833 26478 26478 eatout 1 True \n8684837 26478 26478 eatout 1 False \n8685009 26478 26478 othmaint 1 True \n8685013 26478 26478 othmaint 1 False \n8753057 26686 26686 eatout 1 True \n... ... ... ... ... ... \n2472945113 7539466 2848131 shopping 1 True \n2472945117 7539466 2848131 shopping 1 False \n2472945118 7539466 2848131 shopping 2 False \n2473024473 7539708 2848373 univ 1 True \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n8684833 1 13 8 1085604 eatout \n8684837 1 8 13 1085604 home \n8685009 1 10 8 1085626 othmaint \n8685013 1 8 10 1085626 home \n8753057 1 5 8 1094132 eatout \n... ... ... ... ... ... \n2472945113 1 8 3 309118139 shopping \n2472945117 2 25 8 309118139 shopping \n2472945118 2 3 25 309118139 home \n2473024473 1 13 18 309128059 univ \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n8684833 NaN 11.0 WALK -1.171760 \n8684837 NaN 11.0 WALK -1.238719 \n8685009 NaN 12.0 BIKE 6.198626 \n8685013 NaN 13.0 BIKE 6.175681 \n8753057 NaN 19.0 WALK 4.457539 \n... ... ... ... ... \n2472945113 NaN 18.0 WALK_LOC 12.537675 \n2472945117 56.842247 21.0 WALK_LOC 11.880804 \n2472945118 NaN 22.0 WALK 13.710030 \n2473024473 NaN 16.0 WALK_LOC -0.530696 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[482 rows x 14 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86848332647826478eatout1True11381085604eatoutNaN11.0WALK-1.171760
86848372647826478eatout1False18131085604homeNaN11.0WALK-1.238719
86850092647826478othmaint1True11081085626othmaintNaN12.0BIKE6.198626
86850132647826478othmaint1False18101085626homeNaN13.0BIKE6.175681
87530572668626686eatout1True1581094132eatoutNaN19.0WALK4.457539
.............................................
247294511375394662848131shopping1True183309118139shoppingNaN18.0WALK_LOC12.537675
247294511775394662848131shopping1False2258309118139shopping56.84224721.0WALK_LOC11.880804
247294511875394662848131shopping2False2325309118139homeNaN22.0WALK13.710030
247302447375397082848373univ1True11318309128059univNaN16.0WALK_LOC-0.530696
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

482 rows × 14 columns

\n
" + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips_df" + ] + }, + { + "cell_type": "markdown", + "id": "8eebea31-499c-41ca-8411-883a88ca800a", + "metadata": {}, + "source": [ + "## nested dev" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:01.113060Z", + "iopub.status.busy": "2021-08-31T10:31:01.112829Z", + "iopub.status.idle": "2021-08-31T10:31:01.359912Z", + "shell.execute_reply": "2021-08-31T10:31:01.358948Z", + "shell.execute_reply.started": "2021-08-31T10:31:01.113034Z" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] + "tags": [] + }, + "outputs": [], + "source": [ + "# see fct above - return if necessary\n", + "#spec = simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + "#nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "36c100b3-3e39-4950-a586-4d42be695eaa", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T10:31:01.775068Z", + "iopub.status.busy": "2021-08-31T10:31:01.774846Z", + "iopub.status.idle": "2021-08-31T10:31:01.975816Z", + "shell.execute_reply": "2021-08-31T10:31:01.974499Z", + "shell.execute_reply.started": "2021-08-31T10:31:01.775044Z" }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'eatout' (33 trips)\n" - ] + "tags": [] + }, + "outputs": [], + "source": [ + "#print(nest_spec)\n", + "#for nest in logit.each_nest(nest_spec):\n", + "# nest.print()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:59:35.109079Z", + "iopub.status.busy": "2021-09-01T01:59:35.108781Z", + "iopub.status.idle": "2021-09-01T01:59:35.319301Z", + "shell.execute_reply": "2021-09-01T01:59:35.318402Z", + "shell.execute_reply.started": "2021-09-01T01:59:35.109050Z" }, + "tags": [] + }, + "outputs": [], + "source": [ + "def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator,\n", + " log_alt_losers=False,\n", + " want_logsums=False, trace_label=None,\n", + " trace_choice_name=None, trace_column_names=None):\n", + "\n", + " trace_label = tracing.extend_trace_label(trace_label, 'eval_nl')\n", + " assert trace_label\n", + " have_trace_targets = tracing.has_trace_targets(choosers)\n", + "\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + "\n", + " if have_trace_targets:\n", + " tracing.trace_df(choosers, '%s.choosers' % trace_label)\n", + "\n", + " raw_utilities = simulate.eval_utilities(spec, choosers, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=have_trace_targets,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + " chunk.log_df(trace_label, \"raw_utilities\", raw_utilities)\n", + "\n", + " if have_trace_targets:\n", + " tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label,\n", + " column_labels=['alternative', 'utility'])\n", + "\n", + " # exponentiated utilities of leaves and nests\n", + " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + " chunk.log_df(trace_label, \"nested_exp_utilities\", nested_exp_utilities)\n", + "\n", + " del raw_utilities\n", + " chunk.log_df(trace_label, 'raw_utilities', None)\n", + "\n", + " if have_trace_targets:\n", + " tracing.trace_df(nested_exp_utilities, '%s.nested_exp_utilities' % trace_label,\n", + " column_labels=['alternative', 'utility'])\n", + "\n", + " # probabilities of alternatives relative to siblings sharing the same nest\n", + " nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec,\n", + " trace_label=trace_label)\n", + " chunk.log_df(trace_label, \"nested_probabilities\", nested_probabilities)\n", + "\n", + " if want_logsums:\n", + " # logsum of nest root\n", + " logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index)\n", + " chunk.log_df(trace_label, \"logsums\", logsums)\n", + "\n", + " del nested_exp_utilities\n", + " chunk.log_df(trace_label, 'nested_exp_utilities', None)\n", + "\n", + " if have_trace_targets:\n", + " tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label,\n", + " column_labels=['alternative', 'probability'])\n", + "\n", + " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", + " chunk.log_df(trace_label, \"base_probabilities\", base_probabilities)\n", + "\n", + " del nested_probabilities\n", + " chunk.log_df(trace_label, 'nested_probabilities', None)\n", + "\n", + " if have_trace_targets:\n", + " tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label,\n", + " column_labels=['alternative', 'probability'])\n", + "\n", + " # note base_probabilities could all be zero since we allowed all probs for nests to be zero\n", + " # check here to print a clear message but make_choices will raise error if probs don't sum to 1\n", + " BAD_PROB_THRESHOLD = 0.001\n", + " no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD\n", + "\n", + " if no_choices.any():\n", + "\n", + " logit.report_bad_choices(\n", + " no_choices, base_probabilities,\n", + " trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'),\n", + " trace_choosers=choosers,\n", + " msg=\"base_probabilities do not sum to one\")\n", + "\n", + " if custom_chooser:\n", + " choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec,\n", + " trace_label=trace_label)\n", + " else:\n", + " choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)\n", + "\n", + " del base_probabilities\n", + " chunk.log_df(trace_label, 'base_probabilities', None)\n", + "\n", + " if have_trace_targets:\n", + " tracing.trace_df(choices, '%s.choices' % trace_label,\n", + " columns=[None, trace_choice_name])\n", + " tracing.trace_df(rands, '%s.rands' % trace_label,\n", + " columns=[None, 'rand'])\n", + " if want_logsums:\n", + " tracing.trace_df(logsums, '%s.logsums' % trace_label,\n", + " columns=[None, 'logsum'])\n", + "\n", + " if want_logsums:\n", + " choices = choices.to_frame('choice')\n", + " choices['logsum'] = logsums\n", + "\n", + " return choices\n", + "\n", + "\n", + "def simple_simulate_dev(choosers, spec, nest_spec,\n", + " skims=None, locals_d=None,\n", + " chunk_size=0, custom_chooser=None,\n", + " log_alt_losers=False,\n", + " want_logsums=False,\n", + " estimator=None,\n", + " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", + " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", + " assert len(choosers) > 0\n", + " result_list = []\n", + " # segment by person type and pick the right spec for each person type\n", + " for i, chooser_chunk, chunk_trace_label \\\n", + " in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", + " # the following replaces choices = _simple_simulate(...)\n", + " if skims is not None:\n", + " simulate.set_skim_wrapper_targets(choosers, skims)\n", + "\n", + " # only do this for nested, logit is straight forward\n", + " assert nest_spec is not None\n", + " choices = eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser,\n", + " log_alt_losers=log_alt_losers,\n", + " want_logsums=want_logsums,\n", + " estimator=estimator,\n", + " trace_label=trace_label,\n", + " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", + "\n", + "\n", + " result_list.append(choices)\n", + " chunk.log_df(trace_label, f'result_list', result_list)\n", + "\n", + " if len(result_list) > 1:\n", + " choices = pd.concat(result_list)\n", + " assert len(choices.index == len(choosers.index))\n", + " return choices\n" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", + "metadata": {}, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ + "Running with %d trips 482\n", "trip_mode_choice tour_type 'escort' (6 trips)\n" ] }, @@ -486,67 +675,303 @@ "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + "Done\n" ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'othmaint' (46 trips)\n" - ] - }, + "data": { + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n137248721 418441 304036 escort 1 True \n137248725 418441 304036 escort 1 False \n211388201 644476 386761 escort 1 True \n211388205 644476 386761 escort 1 False \n806388401 2458501 1173905 escort 1 True \n806388405 2458501 1173905 escort 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n137248721 1 7 10 17156090 escort \n137248725 1 10 7 17156090 home \n211388201 1 11 16 26423525 escort \n211388205 1 16 11 26423525 home \n806388401 1 16 8 100798550 escort \n806388405 1 8 16 100798550 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n137248721 NaN 7.0 WALK 11.435800 \n137248725 NaN 7.0 WALK 11.480440 \n211388201 NaN 5.0 WALK_LOC 4.789158 \n211388205 NaN 6.0 WALK_LOC 5.050171 \n806388401 NaN 15.0 WALK_LOC 6.451457 \n806388405 NaN 16.0 WALK_LOC 6.446188 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
137248721418441304036escort1True171017156090escortNaN7.0WALK11.435800
137248725418441304036escort1False110717156090homeNaN7.0WALK11.480440
211388201644476386761escort1True1111626423525escortNaN5.0WALK_LOC4.789158
211388205644476386761escort1False1161126423525homeNaN6.0WALK_LOC5.050171
80638840124585011173905escort1True1168100798550escortNaN15.0WALK_LOC6.451457
80638840524585011173905escort1False1816100798550homeNaN16.0WALK_LOC6.446188
\n
" + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_trips = run_trip_mode_choice(do_these_purposes=[\"escort\"], simulate_function=simple_simulate_dev)\n", + "test_trips" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Get raw utilities, etc" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 105, + "outputs": [], + "source": [ + "def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator,\n", + " log_alt_losers=False,\n", + " want_logsums=False, trace_label=None,\n", + " trace_choice_name=None, trace_column_names=None):\n", + "\n", + " trace_label = tracing.extend_trace_label(trace_label, 'eval_nl')\n", + " assert trace_label\n", + " have_trace_targets = tracing.has_trace_targets(choosers)\n", + "\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, choosers, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=have_trace_targets,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + " # exponentiated utilities of leaves and nests\n", + " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + " nested_utils = simulate.compute_nested_utilities(raw_utilities, nest_spec)\n", + " # probabilities of alternatives relative to siblings sharing the same nest\n", + " nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec,\n", + " trace_label=trace_label)\n", + " if want_logsums:\n", + " # logsum of nest root\n", + " logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index)\n", + " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", + " # note base_probabilities could all be zero since we allowed all probs for nests to be zero\n", + " # check here to print a clear message but make_choices will raise error if probs don't sum to 1\n", + " BAD_PROB_THRESHOLD = 0.001\n", + " no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD\n", + " if no_choices.any():\n", + " print(\"BAD\")\n", + " choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)\n", + " if want_logsums:\n", + " choices = choices.to_frame('choice')\n", + " choices['logsum'] = logsums\n", + " return choices, raw_utilities, nested_exp_utilities, nested_utils\n", + "\n", + "\n", + "def simple_simulate_dev(choosers, spec, nest_spec,\n", + " skims=None, locals_d=None,\n", + " chunk_size=0, custom_chooser=None,\n", + " log_alt_losers=False,\n", + " want_logsums=False,\n", + " estimator=None,\n", + " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", + " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", + " assert len(choosers) > 0\n", + " result_list = []\n", + " # segment by person type and pick the right spec for each person type\n", + " for i, chooser_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", + " # the following replaces choices = _simple_simulate(...)\n", + " if skims is not None:\n", + " simulate.set_skim_wrapper_targets(choosers, skims)\n", + "\n", + " # only do this for nested, logit is straight forward\n", + " assert nest_spec is not None\n", + " choices, raw_utilities, nested_exp_utilities, nested_utils = eval_nl_dev(choosers, spec, nest_spec, locals_d,\n", + " custom_chooser,\n", + " log_alt_losers=log_alt_losers,\n", + " want_logsums=want_logsums,\n", + " estimator=estimator,\n", + " trace_label=trace_label,\n", + " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", + "\n", + "\n", + " result_list.append(choices)\n", + " chunk.log_df(trace_label, f'result_list', result_list)\n", + "\n", + " if len(result_list) > 1:\n", + " choices = pd.concat(result_list)\n", + " assert len(choices.index == len(choosers.index))\n", + " return choices, raw_utilities, nested_exp_utilities, nested_utils\n", + "\n", + "\n", + "def get_stuff(do_these_purposes=None):\n", + " #do_these_purposes=['escort']\n", + " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", + " resume_after = \"trip_scheduling\"\n", + " model_name = \"trip_mode_choice\"\n", + " chunk_size = 0 # test_mtc means no chunking\n", + "\n", + " pipeline.open_pipeline(resume_after)\n", + " # preload any bulky injectables (e.g. skims) not in pipeline\n", + " inject.get_injectable('preload_injectables', None)\n", + " pipeline._PIPELINE.rng().begin_step(model_name)\n", + " step_name = model_name\n", + " args = {}\n", + " checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + " inject.set_step_args(args)\n", + "\n", + " trips = inject.get_table('trips')\n", + " tours_merged = inject.get_table('tours_merged')\n", + " network_los = inject.get_injectable('network_los')\n", + "\n", + " trace_label = 'trip_mode_choice'\n", + " model_settings_file_name = 'trip_mode_choice.yaml'\n", + " model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + " mode_column_name = 'trip_mode'\n", + "\n", + " trips_df = trips.to_frame()\n", + " print(\"Running with %d trips\", trips_df.shape[0])\n", + "\n", + " tours_merged = tours_merged.to_frame()\n", + " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "\n", + " # - trips_merged - merge trips and tours_merged\n", + " trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + " assert trips_merged.index.equals(trips.index)\n", + "\n", + " # setup skim keys\n", + " assert ('trip_period' not in trips_merged)\n", + " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + " orig_col = 'origin'\n", + " dest_col = 'destination'\n", + "\n", + " constants = {}\n", + " constants.update(config.get_model_constants(model_settings))\n", + " constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + " })\n", + "\n", + " skim_dict = network_los.get_default_skim_dict()\n", + "\n", + " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + " skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + " }\n", + "\n", + " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + " nest_specs = config.get_logit_model_settings(model_settings)\n", + "\n", + " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + " choices_list = []\n", + " raw_util_list = []\n", + " nest_list = []\n", + " nu_list = []\n", + " nest_spec_list = []\n", + "\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + "\n", + " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", + " continue\n", + "\n", + " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " (primary_purpose, len(trips_segment.index), ))\n", + "\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " ################ Replace wrapper function\n", + " # choices = mode_choice_simulate(...)\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", + " choices, raw_utilities, nested_exp_utilities, nested_utils = simple_simulate_dev(\n", + " choosers=trips_segment,\n", + " spec=spec,\n", + " nest_spec=nest_spec,\n", + " skims=skims,\n", + " locals_d=locals_dict,\n", + " chunk_size=chunk_size,\n", + " want_logsums=logsum_column_name is not None,\n", + " trace_label=segment_trace_label,\n", + " trace_choice_name='trip_mode_choice',\n", + " estimator=estimator,\n", + " trace_column_names=None)\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " alts = spec.columns\n", + " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", + " ################\n", + " choices_list.append(choices)\n", + " raw_util_list.append(raw_utilities)\n", + " nest_list.append(nested_exp_utilities)\n", + " nu_list.append(nested_utils)\n", + " nest_spec_list.append(nest_spec)\n", + "\n", + " choices_df_asim = pd.concat(choices_list)\n", + "\n", + " # update trips table with choices (and potionally logssums)\n", + " trips_df = trips.to_frame()\n", + "\n", + " if (do_these_purposes is not None):\n", + " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", + "\n", + " assign_in_place(trips_df, choices_df_asim)\n", + " assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + " finalise = True\n", + " if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + " print(\"Done\")\n", + "\n", + " return trips_df, raw_util_list, nest_list, nu_list, nest_spec_list" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 106, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'school' (37 trips)\n" + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n" ] }, { @@ -568,7 +993,6 @@ "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" ] }, @@ -576,1317 +1000,239 @@ "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'shopping' (77 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'social' (19 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'univ' (26 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'work' (168 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n" + "Done\n" ] } ], "source": [ - "trips_df = run_trip_mode_choice()" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "7eae2854-38bd-4da0-a161-3a15bbd17177", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:30:57.562848Z", - "iopub.status.busy": "2021-08-31T10:30:57.562615Z", - "iopub.status.idle": "2021-08-31T10:30:57.806655Z", - "shell.execute_reply": "2021-08-31T10:30:57.805135Z", - "shell.execute_reply.started": "2021-08-31T10:30:57.562824Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n8684833 26478 26478 eatout 1 True \n8684837 26478 26478 eatout 1 False \n8685009 26478 26478 othmaint 1 True \n8685013 26478 26478 othmaint 1 False \n8753057 26686 26686 eatout 1 True \n... ... ... ... ... ... \n2472945113 7539466 2848131 shopping 1 True \n2472945117 7539466 2848131 shopping 1 False \n2472945118 7539466 2848131 shopping 2 False \n2473024473 7539708 2848373 univ 1 True \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n8684833 1 13 8 1085604 eatout \n8684837 1 8 13 1085604 home \n8685009 1 10 8 1085626 othmaint \n8685013 1 8 10 1085626 home \n8753057 1 5 8 1094132 eatout \n... ... ... ... ... ... \n2472945113 1 8 3 309118139 shopping \n2472945117 2 25 8 309118139 shopping \n2472945118 2 3 25 309118139 home \n2473024473 1 13 18 309128059 univ \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n8684833 NaN 11.0 WALK -1.171760 \n8684837 NaN 11.0 WALK -1.238719 \n8685009 NaN 12.0 BIKE 6.198626 \n8685013 NaN 13.0 BIKE 6.175681 \n8753057 NaN 19.0 WALK 4.457539 \n... ... ... ... ... \n2472945113 NaN 18.0 WALK_LOC 12.537675 \n2472945117 56.842247 21.0 WALK_LOC 11.880804 \n2472945118 NaN 22.0 WALK 13.710030 \n2473024473 NaN 16.0 WALK_LOC -0.530696 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[482 rows x 14 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86848332647826478eatout1True11381085604eatoutNaN11.0WALK-1.171760
86848372647826478eatout1False18131085604homeNaN11.0WALK-1.238719
86850092647826478othmaint1True11081085626othmaintNaN12.0BIKE6.198626
86850132647826478othmaint1False18101085626homeNaN13.0BIKE6.175681
87530572668626686eatout1True1581094132eatoutNaN19.0WALK4.457539
.............................................
247294511375394662848131shopping1True183309118139shoppingNaN18.0WALK_LOC12.537675
247294511775394662848131shopping1False2258309118139shopping56.84224721.0WALK_LOC11.880804
247294511875394662848131shopping2False2325309118139homeNaN22.0WALK13.710030
247302447375397082848373univ1True11318309128059univNaN16.0WALK_LOC-0.530696
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

482 rows × 14 columns

\n
" - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } + "t, ru, neu, nu, ns = get_stuff(do_these_purposes=[\"escort\"])" ], - "source": [ - "trips_df" - ] - }, - { - "cell_type": "markdown", - "id": "8eebea31-499c-41ca-8411-883a88ca800a", - "metadata": {}, - "source": [ - "## nested dev" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:01.113060Z", - "iopub.status.busy": "2021-08-31T10:31:01.112829Z", - "iopub.status.idle": "2021-08-31T10:31:01.359912Z", - "shell.execute_reply": "2021-08-31T10:31:01.358948Z", - "shell.execute_reply.started": "2021-08-31T10:31:01.113034Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# see fct above - return if necessary\n", - "#spec = simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - "#nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "id": "36c100b3-3e39-4950-a586-4d42be695eaa", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:01.775068Z", - "iopub.status.busy": "2021-08-31T10:31:01.774846Z", - "iopub.status.idle": "2021-08-31T10:31:01.975816Z", - "shell.execute_reply": "2021-08-31T10:31:01.974499Z", - "shell.execute_reply.started": "2021-08-31T10:31:01.775044Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#print(nest_spec)\n", - "#for nest in logit.each_nest(nest_spec):\n", - "# nest.print()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:59:35.109079Z", - "iopub.status.busy": "2021-09-01T01:59:35.108781Z", - "iopub.status.idle": "2021-09-01T01:59:35.319301Z", - "shell.execute_reply": "2021-09-01T01:59:35.318402Z", - "shell.execute_reply.started": "2021-09-01T01:59:35.109050Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_dev(choosers, spec, nest_spec,\n", - " skims=None, locals_d=None,\n", - " chunk_size=0, custom_chooser=None,\n", - " log_alt_losers=False,\n", - " want_logsums=False,\n", - " estimator=None,\n", - " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", - " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", - " assert len(choosers) > 0\n", - " result_list = []\n", - " # segment by person type and pick the right spec for each person type\n", - " for i, chooser_chunk, chunk_trace_label \\\n", - " in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", - " # the following replaces choices = _simple_simulate(...)\n", - " if skims is not None:\n", - " simulate.set_skim_wrapper_targets(choosers, skims)\n", - "\n", - " # only do this for nested, logit is straight forward\n", - " assert nest_spec is not None\n", - " choices = simulate.eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser,\n", - " log_alt_losers=log_alt_losers,\n", - " want_logsums=want_logsums,\n", - " estimator=estimator,\n", - " trace_label=trace_label,\n", - " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", - "\n", - "\n", - " result_list.append(choices)\n", - " chunk.log_df(trace_label, f'result_list', result_list)\n", - "\n", - " if len(result_list) > 1:\n", - " choices = pd.concat(result_list)\n", - " assert len(choices.index == len(choosers.index))\n", - " return choices\n" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n" - ] - } - ], - "source": [ - "test_trips = run_trip_mode_choice(do_these_purposes=[\"escort\"], simulate_function=simple_simulate_dev)" - ] - }, - { - "cell_type": "markdown", - "id": "c60656db-075a-4eab-96b3-fc54eb785075", - "metadata": {}, - "source": [ - "## eval_nl dev" - ] - }, - { - "cell_type": "code", - "execution_count": 372, - "id": "56c8923c-e946-48d8-b964-87695c30163d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:27:40.290389Z", - "iopub.status.busy": "2021-09-01T02:27:40.290139Z", - "iopub.status.idle": "2021-09-01T02:27:40.977526Z", - "shell.execute_reply": "2021-09-01T02:27:40.976594Z", - "shell.execute_reply.started": "2021-09-01T02:27:40.290363Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# replaces simple_simulate_dev:\n", - "\n", - "want_logsums = True\n", - "trace_choice_name='trip_mode_choice'\n", - "trace_column_names=None\n", - "log_alt_losers=False\n", - "\n", - "tr_label = tracing.extend_trace_label(segment_trace_label, 'simple_simulate')\n", - "assert len(trips_segment) > 0\n", - "result_list = []\n", - "# segment by person type and pick the right spec for each person type\n", - "for i, chooser_chunk, chunk_trace_label \\\n", - " in chunk.adaptive_chunked_choosers(trips_segment, chunk_size, tr_label):\n", - " # the following replaces choices = _simple_simulate(...)\n", - " if skims is not None:\n", - " simulate.set_skim_wrapper_targets(trips_segment, skims)\n", - "\n", - " # only do this for nested, logit is straight forward\n", - " assert nest_spec is not None\n", - " choices = simulate.eval_nl(trips_segment, spec, nest_spec, locals_dict, custom_chooser=None,\n", - " log_alt_losers=log_alt_losers,\n", - " want_logsums=want_logsums,\n", - " estimator=estimator,\n", - " trace_label=tr_label,\n", - " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", - "\n", - "\n", - " result_list.append(choices)\n", - "\n", - "if len(result_list) > 1:\n", - " choices = pd.concat(result_list)\n", - "assert len(choices.index == len(trips_segment.index))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "simple_simulate_dev" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 93, - "id": "cbc2dfde-1bd9-4b03-96d9-ca9b7222d36c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T12:52:34.544865Z", - "iopub.status.busy": "2021-08-31T12:52:34.544586Z", - "iopub.status.idle": "2021-08-31T12:52:35.159615Z", - "shell.execute_reply": "2021-08-31T12:52:35.158682Z", - "shell.execute_reply.started": "2021-08-31T12:52:34.544840Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "locals_d = locals_dict\n", - "custom_chooser=None\n", - "\n", - "trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - "logit.validate_nest_spec(nest_spec, trace_label)\n", - "\n", - "raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - "nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - "\n", - "nested_probabilities = \\\n", - " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", - "\n", - "logsums = pd.Series(np.log(nested_exp_utilities.root), index=trips_segment.index)\n", - "\n", - "# global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - "base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", - "\n", - "choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "id": "0cd9593b-b3fd-41f6-a7ca-4c5b73b63571", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T12:52:45.509057Z", - "iopub.status.busy": "2021-08-31T12:52:45.508819Z", - "iopub.status.idle": "2021-08-31T12:52:45.721956Z", - "shell.execute_reply": "2021-08-31T12:52:45.721064Z", - "shell.execute_reply.started": "2021-08-31T12:52:45.509021Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 0.990308\n", - "137248725 0.108433\n", - "211388201 0.806612\n", - "211388205 0.195994\n", - "806388401 0.854020\n", - "806388405 0.660550\n", - "dtype: float64" - ] - }, - "execution_count": 95, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rands" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "id": "b467330b-f1cc-43bc-9325-c6dfeaa9b364", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T12:50:51.735273Z", - "iopub.status.busy": "2021-08-31T12:50:51.734962Z", - "iopub.status.idle": "2021-08-31T12:50:52.023440Z", - "shell.execute_reply": "2021-08-31T12:50:52.022582Z", - "shell.execute_reply.started": "2021-08-31T12:50:51.735231Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.798480-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.360270-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.435800
137248725-1.732971-3966.018685-0.606540-0.324578-3964.610293-0.113602-3966.009714-7930.295428-inf0.26146615.945036-1368.16910311.480426-2737.292402-5512.292402-5512.292402-5512.292402-5512.292402-inf-2737.782457-5522.136798-5522.136798-5516.076156-5525.080218-inf-inf-24.341318-25.104291-24.140831-8.40687011.480440
211388201-3966.000225-7930.285939-inf-24.120135-3988.405850-8.442047-29.480718-3993.766432-10.318251-5.975663-4.216264-1381.397292-3.03571013.295720-2761.150907-2761.150907-2761.150907-2761.1509076.647860-2761.648763-5543.048131-5543.048131-2769.692185-2773.867648-inf4.786459-15.861515-3.864497-5.068360-1.2967574.789158
211388205-3967.791945-7932.077659-inf-25.129596-3989.415310-8.795359-30.177275-3994.462989-10.562046-6.219063-4.216264-1381.397292-3.03571014.022438-2760.977562-2760.977562-2760.977562-2760.9775627.011219-2761.294814-5546.132585-5546.132585-2764.829835-2774.132982-inf5.048078-15.776424-3.845077-5.005482-1.2860945.050171
806388401-3966.492870-7930.778584-inf-12.618378-3976.904092-4.416432-14.908422-3979.194136-5.217948-2.9129776.192499-1378.4807234.45859917.513379-2755.845870-2755.208370-2755.728926-2755.7930938.756690-2757.559519-5540.685990-5540.685990-5542.838278-5546.780761-inf6.304817-19.143696-19.495908-16.361074-5.8539176.451457
806388405-3969.835821-7934.121535-inf-14.515916-3978.801631-5.080571-16.227796-3980.513510-5.679728-3.3428055.998749-1378.5194734.31909917.553369-2755.987742-2755.350242-2755.870797-2755.9349648.776685-2757.571454-5542.673358-5542.673358-5537.799228-5547.515590-inf6.319213-18.886576-19.495133-16.300395-5.8285406.446188
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", - "trip_id \n", - "137248721 -1.798480 -3966.084195 -0.629468 -0.389611 \n", - "137248725 -1.732971 -3966.018685 -0.606540 -0.324578 \n", - "211388201 -3966.000225 -7930.285939 -inf -24.120135 \n", - "211388205 -3967.791945 -7932.077659 -inf -25.129596 \n", - "806388401 -3966.492870 -7930.778584 -inf -12.618378 \n", - "806388405 -3969.835821 -7934.121535 -inf -14.515916 \n", - "\n", - " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", - "trip_id \n", - "137248721 -3964.675326 -0.136364 -3966.074556 -7930.360270 -inf \n", - "137248725 -3964.610293 -0.113602 -3966.009714 -7930.295428 -inf \n", - "211388201 -3988.405850 -8.442047 -29.480718 -3993.766432 -10.318251 \n", - "211388205 -3989.415310 -8.795359 -30.177275 -3994.462989 -10.562046 \n", - "806388401 -3976.904092 -4.416432 -14.908422 -3979.194136 -5.217948 \n", - "806388405 -3978.801631 -5.080571 -16.227796 -3980.513510 -5.679728 \n", - "\n", - " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", - "trip_id \n", - "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", - "137248725 0.261466 15.945036 -1368.169103 11.480426 -2737.292402 \n", - "211388201 -5.975663 -4.216264 -1381.397292 -3.035710 13.295720 \n", - "211388205 -6.219063 -4.216264 -1381.397292 -3.035710 14.022438 \n", - "806388401 -2.912977 6.192499 -1378.480723 4.458599 17.513379 \n", - "806388405 -3.342805 5.998749 -1378.519473 4.319099 17.553369 \n", - "\n", - " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", - "trip_id \n", - "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", - "137248725 -5512.292402 -5512.292402 -5512.292402 -5512.292402 -inf \n", - "211388201 -2761.150907 -2761.150907 -2761.150907 -2761.150907 6.647860 \n", - "211388205 -2760.977562 -2760.977562 -2760.977562 -2760.977562 7.011219 \n", - "806388401 -2755.845870 -2755.208370 -2755.728926 -2755.793093 8.756690 \n", - "806388405 -2755.987742 -2755.350242 -2755.870797 -2755.934964 8.776685 \n", - "\n", - " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", - "trip_id \n", - "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", - "137248725 -2737.782457 -5522.136798 -5522.136798 -5516.076156 -5525.080218 \n", - "211388201 -2761.648763 -5543.048131 -5543.048131 -2769.692185 -2773.867648 \n", - "211388205 -2761.294814 -5546.132585 -5546.132585 -2764.829835 -2774.132982 \n", - "806388401 -2757.559519 -5540.685990 -5540.685990 -5542.838278 -5546.780761 \n", - "806388405 -2757.571454 -5542.673358 -5542.673358 -5537.799228 -5547.515590 \n", - "\n", - " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", - "trip_id \n", - "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", - "137248725 -inf -inf -24.341318 -25.104291 -24.140831 -8.406870 \n", - "211388201 -inf 4.786459 -15.861515 -3.864497 -5.068360 -1.296757 \n", - "211388205 -inf 5.048078 -15.776424 -3.845077 -5.005482 -1.286094 \n", - "806388401 -inf 6.304817 -19.143696 -19.495908 -16.361074 -5.853917 \n", - "806388405 -inf 6.319213 -18.886576 -19.495133 -16.300395 -5.828540 \n", - "\n", - " root \n", - "trip_id \n", - "137248721 11.435800 \n", - "137248725 11.480440 \n", - "211388201 4.789158 \n", - "211388205 5.050171 \n", - "806388401 6.451457 \n", - "806388405 6.446188 " - ] - }, - "execution_count": 92, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# root is logsum here, no need to take log again as for exponentiated utils\n", - "simulate.compute_nested_exp_utilities(raw_utilities, nest_spec, exponentiate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "5c9ba018-167d-497f-ac65-f7c49036ee58", - "metadata": {}, - "source": [ - "#### scale of utils, including emus" - ] - }, - { - "cell_type": "code", - "execution_count": 283, - "id": "9febf6ef-7c6b-4f3c-bdd9-aa3326a82e95", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:23:00.020038Z", - "iopub.status.busy": "2021-08-31T23:23:00.019377Z", - "iopub.status.idle": "2021-08-31T23:23:00.220889Z", - "shell.execute_reply": "2021-08-31T23:23:00.220154Z", - "shell.execute_reply.started": "2021-08-31T23:23:00.019997Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "nests_ = list(logit.each_nest(nest_spec))" - ] - }, - { - "cell_type": "code", - "execution_count": 284, - "id": "48ff920e-0c9b-4b69-9cee-eb3c3427bd4f", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:23:01.085713Z", - "iopub.status.busy": "2021-08-31T23:23:01.085486Z", - "iopub.status.idle": "2021-08-31T23:23:01.283509Z", - "shell.execute_reply": "2021-08-31T23:23:01.282444Z", - "shell.execute_reply.started": "2021-08-31T23:23:01.085685Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] alternatives: ['DRIVEALONE', 'SHAREDRIDE2', 'SHAREDRIDE3']\n", - "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] alternatives: ['DRIVEALONEFREE', 'DRIVEALONEPAY']\n", - "Nest name: DRIVEALONEFREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE'] alternatives: None\n", - "Nest name: DRIVEALONEPAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY'] alternatives: None\n" - ] - } - ], - "source": [ - "#list(map(lambda x: x.print(), nests_));\n", - "# have a look at \n", - "nests_[1].print()\n", - "nests_[2].print()\n", - "nests_[3].print()\n", - "nests_[4].print()" - ] - }, - { - "cell_type": "code", - "execution_count": 312, - "id": "7b7e453d-da45-4751-8889-561a349bd373", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:22.371464Z", - "iopub.status.busy": "2021-09-01T01:11:22.371192Z", - "iopub.status.idle": "2021-09-01T01:11:22.593424Z", - "shell.execute_reply": "2021-09-01T01:11:22.592472Z", - "shell.execute_reply.started": "2021-09-01T01:11:22.371415Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation belowi s equivalent.\n", - "def compute_nested_utilities(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) / nest.product_of_coefficients #coefficient\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", - " return nested_utilities\n", - "\n", - "\n", - "utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", - "\n", - "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", - "seed = 9326543345\n", - "rng = default_rng(seed=seed)\n", - "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", - "rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", - "\n", - "probs_arr = utils_df - np.log(-np.log(rands))\n", - "\n", - "# Now replace the following with recursive nested choice\n", - "# choices = np.argmax(probs_arr, axis=1)\n", - "# choices = pd.Series(choices, index=utils_df.index)\n", - "# rands = pd.Series(np.asanyarray(rands).flatten(), index=utils_df.index)" - ] - }, - { - "cell_type": "code", - "execution_count": 313, - "id": "39b71df9-12ac-4d9c-8134-43d22f7e6a6f", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:23.703299Z", - "iopub.status.busy": "2021-09-01T01:11:23.703088Z", - "iopub.status.idle": "2021-09-01T01:11:23.937077Z", - "shell.execute_reply": "2021-09-01T01:11:23.935627Z", - "shell.execute_reply.started": "2021-09-01T01:11:23.703274Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.79848-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.36027-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.4358
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", - "trip_id \n", - "137248721 -1.79848 -3966.084195 -0.629468 -0.389611 \n", - "\n", - " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", - "trip_id \n", - "137248721 -3964.675326 -0.136364 -3966.074556 -7930.36027 -inf \n", - "\n", - " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", - "trip_id \n", - "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", - "\n", - " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", - "trip_id \n", - "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", - "\n", - " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", - "trip_id \n", - "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", - "\n", - " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", - "trip_id \n", - "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", - "\n", - " root \n", - "trip_id \n", - "137248721 11.4358 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-0.717963-3964.2474012.7217010.230429-3964.716506-0.274584-3965.126773-7930.610173-inf1.73337615.473633-1368.52552614.717027-2736.517471-5510.738449-5512.038516-5512.559241-5511.584293-inf-2738.143234-5518.606969-5519.009587-5519.830735-5521.525966-inf-inf-25.519792-21.698963-25.267185-7.44465211.689443
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", - "trip_id \n", - "137248721 -0.717963 -3964.247401 2.721701 0.230429 \n", - "\n", - " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", - "trip_id \n", - "137248721 -3964.716506 -0.274584 -3965.126773 -7930.610173 -inf \n", - "\n", - " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", - "trip_id \n", - "137248721 1.733376 15.473633 -1368.525526 14.717027 -2736.517471 \n", - "\n", - " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", - "trip_id \n", - "137248721 -5510.738449 -5512.038516 -5512.559241 -5511.584293 -inf \n", - "\n", - " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", - "trip_id \n", - "137248721 -2738.143234 -5518.606969 -5519.009587 -5519.830735 -5521.525966 \n", - "\n", - " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", - "trip_id \n", - "137248721 -inf -inf -25.519792 -21.698963 -25.267185 -7.444652 \n", - "\n", - " root \n", - "trip_id \n", - "137248721 11.689443 " - ] + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 108, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.294906 -2855.580620 -0.453217 -0.280520 \n137248725 -1.247739 -2855.533453 -0.436709 -0.233697 \n211388201 -2855.520162 -5709.805876 -inf -17.366498 \n211388205 -2856.810200 -5711.095914 -inf -18.093309 \n806388401 -2855.874866 -5710.160580 -inf -9.085232 \n806388405 -2858.281791 -5712.567505 -inf -10.451460 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.566235 -0.098182 -2855.573680 -5709.859395 -inf \n137248725 -2854.519411 -0.081794 -2855.526994 -5709.812708 -inf \n211388201 -2871.652212 -6.078274 -21.226117 -2875.511831 -7.429141 \n211388205 -2872.379023 -6.332658 -21.727638 -2876.013352 -7.604673 \n806388401 -2863.370946 -3.179831 -10.734064 -2865.019778 -3.756922 \n806388405 -2864.737174 -3.658011 -11.684013 -2865.969727 -4.089405 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.311848 15.883036 -1368.200102 11.435786 -1970.783011 \n137248725 0.323683 15.945036 -1368.169103 11.480426 -1970.850529 \n211388201 -4.210520 -4.216264 -1381.397292 -3.035710 9.572919 \n211388205 -4.381625 -4.216264 -1381.397292 -3.035710 10.096155 \n806388401 -1.968599 6.192499 -1378.480723 4.458599 12.609633 \n806388405 -2.273383 5.998749 -1378.519473 4.319099 12.638426 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.783011 -3968.783011 -3968.783011 -3968.783011 -inf \n137248725 -3968.850529 -3968.850529 -3968.850529 -3968.850529 -inf \n211388201 -1988.028653 -1988.028653 -1988.028653 -1988.028653 4.786459 \n211388205 -1987.903845 -1987.903845 -1987.903845 -1987.903845 5.048078 \n806388401 -1984.209027 -1983.750027 -1984.124827 -1984.171027 6.304817 \n806388405 -1984.311174 -1983.852174 -1984.226974 -1984.273174 6.319213 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.903118 -3973.401006 -3973.401006 -3974.669500 -3975.688637 \n137248725 -1971.203369 -3975.938495 -3975.938495 -3971.574832 -3978.057757 \n211388201 -1988.387109 -3990.994654 -3990.994654 -1994.178373 -1997.184706 \n211388205 -1988.132266 -3993.215462 -3993.215462 -1990.677481 -1997.375747 \n806388401 -1985.442854 -3989.293913 -3989.293913 -3990.843560 -3993.682148 \n806388405 -1985.451447 -3990.724818 -3990.724818 -3987.215444 -3994.211225 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n137248725 -inf -inf -24.341318 -25.104291 -24.140831 -8.406870 \n211388201 -inf 3.446251 -15.861515 -3.864497 -5.068360 -1.296757 \n211388205 -inf 3.634616 -15.776424 -3.845077 -5.005482 -1.286094 \n806388401 -inf 4.539468 -19.143696 -19.495908 -16.361074 -5.853917 \n806388405 -inf 4.549833 -18.886576 -19.495133 -16.300395 -5.828540 \n\n root \ntrip_id \n137248721 11.435801 \n137248725 11.480441 \n211388201 3.456910 \n211388205 3.643469 \n806388401 5.193789 \n806388405 5.134877 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.294906-2855.580620-0.453217-0.280520-2854.566235-0.098182-2855.573680-5709.859395-inf0.31184815.883036-1368.20010211.435786-1970.783011-3968.783011-3968.783011-3968.783011-3968.783011-inf-1970.903118-3973.401006-3973.401006-3974.669500-3975.688637-inf-inf-24.503675-25.143041-24.249471-8.44847311.435801
137248725-1.247739-2855.533453-0.436709-0.233697-2854.519411-0.081794-2855.526994-5709.812708-inf0.32368315.945036-1368.16910311.480426-1970.850529-3968.850529-3968.850529-3968.850529-3968.850529-inf-1971.203369-3975.938495-3975.938495-3971.574832-3978.057757-inf-inf-24.341318-25.104291-24.140831-8.40687011.480441
211388201-2855.520162-5709.805876-inf-17.366498-2871.652212-6.078274-21.226117-2875.511831-7.429141-4.210520-4.216264-1381.397292-3.0357109.572919-1988.028653-1988.028653-1988.028653-1988.0286534.786459-1988.387109-3990.994654-3990.994654-1994.178373-1997.184706-inf3.446251-15.861515-3.864497-5.068360-1.2967573.456910
211388205-2856.810200-5711.095914-inf-18.093309-2872.379023-6.332658-21.727638-2876.013352-7.604673-4.381625-4.216264-1381.397292-3.03571010.096155-1987.903845-1987.903845-1987.903845-1987.9038455.048078-1988.132266-3993.215462-3993.215462-1990.677481-1997.375747-inf3.634616-15.776424-3.845077-5.005482-1.2860943.643469
806388401-2855.874866-5710.160580-inf-9.085232-2863.370946-3.179831-10.734064-2865.019778-3.756922-1.9685996.192499-1378.4807234.45859912.609633-1984.209027-1983.750027-1984.124827-1984.1710276.304817-1985.442854-3989.293913-3989.293913-3990.843560-3993.682148-inf4.539468-19.143696-19.495908-16.361074-5.8539175.193789
806388405-2858.281791-5712.567505-inf-10.451460-2864.737174-3.658011-11.684013-2865.969727-4.089405-2.2733835.998749-1378.5194734.31909912.638426-1984.311174-1983.852174-1984.226974-1984.2731746.319213-1985.451447-3990.724818-3990.724818-3987.215444-3994.211225-inf4.549833-18.886576-19.495133-16.300395-5.8285405.134877
\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "{'name': 'root',\n 'coefficient': 1.0,\n 'alternatives': [{'name': 'AUTO',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'DRIVEALONE',\n 'coefficient': 0.35,\n 'alternatives': ['DRIVEALONEFREE', 'DRIVEALONEPAY']},\n {'name': 'SHAREDRIDE2',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED2FREE', 'SHARED2PAY']},\n {'name': 'SHAREDRIDE3',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED3FREE', 'SHARED3PAY']}]},\n {'name': 'NONMOTORIZED',\n 'coefficient': 0.72,\n 'alternatives': ['WALK', 'BIKE']},\n {'name': 'TRANSIT',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'WALKACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['WALK_LOC',\n 'WALK_LRF',\n 'WALK_EXP',\n 'WALK_HVY',\n 'WALK_COM']},\n {'name': 'DRIVEACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['DRIVE_LOC',\n 'DRIVE_LRF',\n 'DRIVE_EXP',\n 'DRIVE_HVY',\n 'DRIVE_COM']}]},\n {'name': 'RIDEHAIL',\n 'coefficient': 0.36,\n 'alternatives': ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']}]}" }, "metadata": {}, "output_type": "display_data" } ], "source": [ - "display(utils_df.head(1), probs_arr.head(1))" - ] + "display(nu[0], ns[0])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 111, + "outputs": [], + "source": [ + "# next: add error terms to alternatives and nodes - can be done with this right here by iterating over nests and\n", + "# adding at each level.\n", + "# will probably want to vectorise, but that's for later.\n", + "# add_random returns a random number per row - this seems like the right thing to use while iterating over nest nodes\n", + "# and leafs (which gives us single columns per alternative)\n", + "# for destination choice, we might need to rethink this pattern though, but we'll cross that bridge when we come to it\n", + "\n", + "def inverse_ev1_cdf(x, location=0.0, scale=1.0):\n", + " #quantile function of EV1\n", + " # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution where the scale is proportional to variance (not variance^{-1})\n", + " # this means nested scales are between 0 and 1\n", + " # x can be number or np array for vecops\n", + " return location - scale * np.log(-np.log(x))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 112, + "outputs": [], + "source": [ + "utils_df = nu[0]\n", + "nest_spec = ns[0]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 115, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: DRIVEALONEFREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE']\n", + "Nest name: DRIVEALONEPAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY']\n", + "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE']\n", + "Nest name: SHARED2FREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2FREE']\n", + "Nest name: SHARED2PAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2PAY']\n", + "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2']\n", + "Nest name: SHARED3FREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3FREE']\n", + "Nest name: SHARED3PAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3PAY']\n", + "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3']\n", + "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO']\n", + "Nest name: WALK level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'WALK']\n", + "Nest name: BIKE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'BIKE']\n", + "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED']\n", + "Nest name: WALK_LOC level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LOC']\n", + "Nest name: WALK_LRF level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LRF']\n", + "Nest name: WALK_EXP level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_EXP']\n", + "Nest name: WALK_HVY level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_HVY']\n", + "Nest name: WALK_COM level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_COM']\n", + "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS']\n", + "Nest name: DRIVE_LOC level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LOC']\n", + "Nest name: DRIVE_LRF level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LRF']\n", + "Nest name: DRIVE_EXP level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_EXP']\n", + "Nest name: DRIVE_HVY level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_HVY']\n", + "Nest name: DRIVE_COM level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_COM']\n", + "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS']\n", + "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT']\n", + "Nest name: TAXI level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TAXI']\n", + "Nest name: TNC_SINGLE level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SINGLE']\n", + "Nest name: TNC_SHARED level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SHARED']\n", + "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL']\n", + "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root']\n" + ] + } + ], + "source": [ + "for n in logit.each_nest(nest_spec, post_order=True):\n", + " n.print()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# OLD\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } }, { "cell_type": "markdown", @@ -1934,9 +1280,34 @@ " if next_level_columns is None:\n", " return choices \n", " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", - " return new_choice" + " return new_choice\n", + "\n", + "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", + "\n", + "def make_choice(utils_df, nests, nest_levels, seed=None):\n", + " rng = default_rng(seed=seed)\n", + " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", + " probs_arr = utils_df - np.log(-np.log(rands))\n", + " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", + " return choices" ] }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", + "seed = 9326543345\n", + "make_choice(utils_df, nests_, nest_levels, seed)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": 316, @@ -1973,7 +1344,7 @@ }, { "cell_type": "code", - "execution_count": 326, + "execution_count": 101, "id": "bae6e7a8-d7c3-42a2-a82f-874b1e59a0b6", "metadata": { "execution": { @@ -1985,55 +1356,9 @@ }, "tags": [] }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 WALK\n", - "137248725 WALK\n", - "211388201 WALK_LOC\n", - "211388205 WALK_LOC\n", - "806388401 WALK\n", - "806388405 WALK\n", - "dtype: object" - ] - }, - "execution_count": 326, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests_), axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 321, - "id": "fc315f63-542f-46fd-8d70-b7a1fcd492c6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:53:15.500933Z", - "iopub.status.busy": "2021-09-01T01:53:15.500709Z", - "iopub.status.idle": "2021-09-01T01:53:15.703201Z", - "shell.execute_reply": "2021-09-01T01:53:15.702357Z", - "shell.execute_reply.started": "2021-09-01T01:53:15.500909Z" - }, - "tags": [] - }, "outputs": [], "source": [ - "#display(utils_df, probs_arr)" - ] - }, - { - "cell_type": "markdown", - "id": "8a45e1c0-0a32-4f7a-8803-7559d71abefd", - "metadata": {}, - "source": [ - "### some tests\n", - "seeds, average?" + "#probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests_), axis=1)" ] }, { @@ -2069,20 +1394,7 @@ "output_type": "execute_result" } ], - "source": [ - "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", - "seed = 9326543345\n", - "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", - "\n", - "def make_choice(utils_df, nests, nest_levels, seed=None):\n", - " rng = default_rng(seed=seed)\n", - " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", - " probs_arr = utils_df - np.log(-np.log(rands))\n", - " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", - " return choices\n", - " \n", - "make_choice(utils_df, nests_, nest_levels, seed)" - ] + "source": [] }, { "cell_type": "code", @@ -2218,35 +1530,6 @@ "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" ] }, - { - "cell_type": "code", - "execution_count": 380, - "id": "2da81b5e-7992-4437-8e0f-905caf026804", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:33:10.886502Z", - "iopub.status.busy": "2021-09-01T02:33:10.886252Z", - "iopub.status.idle": "2021-09-01T02:33:11.092145Z", - "shell.execute_reply": "2021-09-01T02:33:11.090725Z", - "shell.execute_reply.started": "2021-09-01T02:33:10.886465Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'trip_mode_choice'" - ] - }, - "execution_count": 380, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trace_label" - ] - }, { "cell_type": "code", "execution_count": 393, diff --git a/notebooks/nested_logit_frozen_rand_individ_util.ipynb b/notebooks/nested_logit_frozen_rand_individ_util.ipynb index 2bb10981e5..2578b9068e 100644 --- a/notebooks/nested_logit_frozen_rand_individ_util.ipynb +++ b/notebooks/nested_logit_frozen_rand_individ_util.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 3, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:14.957470Z", @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:15.467608Z", @@ -154,7 +154,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:20.523480Z", @@ -171,8 +171,8 @@ "text": [ "closed form: [0.4452265282367507, 0.5330453677531714, 0.02172810401007798],\n", "simulated: [0.4450967 0.5331544 0.0217489]\n", - "CPU times: user 766 ms, sys: 46.9 ms, total: 812 ms\n", - "Wall time: 814 ms\n" + "CPU times: user 969 ms, sys: 78.1 ms, total: 1.05 s\n", + "Wall time: 1.14 s\n" ] } ], @@ -194,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:24.027745Z", @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 15, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T10:02:14.035272Z", @@ -257,7 +257,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 16, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T10:03:23.999857Z", @@ -309,28 +309,32 @@ ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### the error term decomposition way\n", - "\n", - "We can decompose the error term into one for the nest and one within nests. For the example structure here, do we need to introduce artificial levels? In other terms, does car and bus nest have a common component?" - ] + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } }, { "cell_type": "markdown", "metadata": {}, "source": [ - "observation: when setting all utilities equal and nest scale to 0.999, the probs should be nearly exacly equal 1/3, but the probs here are about 0.5 to 0.25,0.25 which is what you get when the nest scale is very small (i.e. strongly correlated).\n", + "## The error term decomposition way -> not working. just use the zenith way and write it as indicators for individuals.\n", "\n", - "When the 1/scale in quantile function is turned into scale, the values are slightly off but much closer (0.368 to 0.3165,0.3165)\n", + "We can decompose the error term into one for the nest and one within nests according to Hildebrandt. However, I\n", + "cannot seem to reproduce the exact probabilities. Why?\n", "\n", - "# => check scale and 1/scale everywhere" + "Looks like one of the location parameters is wrong; 0.125 added to nest makes it right (tested for one set of params)" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 79, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:25.164028Z", @@ -346,33 +350,34 @@ "output_type": "stream", "text": [ "closed form probs: 0.445227, 0.533045, 0.021728\n", - " simulated probs: 0.284564, 0.493765, 0.221671\n", - "CPU times: user 281 ms, sys: 0 ns, total: 281 ms\n", - "Wall time: 303 ms\n" + " simulated probs: 0.4755317, 0.5039483, 0.02052\n", + "0.9363, 1.0577, 1.0589\n", + "CPU times: user 4.12 s, sys: 2.3 s, total: 6.42 s\n", + "Wall time: 6.99 s\n" ] } ], "source": [ "%%time\n", "# conditional error term are given by logit with scale given by nest scale\n", - "num_draws_dec = 1000000\n", + "num_draws_dec = 10000000\n", "\n", "mu = 1.0 / nest_scale\n", "\n", "rng_dec = default_rng(9)\n", "\n", "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", - "ev1_dec = inverse_ev1_cdf(rands_dec, scale=mu)\n", - "\n", + "ev1_dec = inverse_ev1_cdf(rands_dec, scale=1.0/mu)\n", "lower_level_utils_2 = np.repeat(util_2, num_draws_dec) + ev1_dec[num_draws_dec:]\n", "lower_level_utils_3 = np.repeat(util_3, num_draws_dec) + ev1_dec[:num_draws_dec]\n", "\n", - "\n", + "#location_nest = - 1.0 / mu * np.log(2.0 * np.exp(mu * 0.5772))\n", "location_nest = - np.log(2.0) / mu\n", "#location_nest = (- np.log(2.0) / mu) - ((1.0 - 1.0 / (mu + 1.0)) * 0.57721 * mu / (mu**2 - 1.0))\n", "#print(location_nest, - np.log(2.0) / mu)\n", - "scale_nest = mu**2 / np.sqrt(mu**2 - 1.0)\n", - "nest_error_terms = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec), location=location_nest, scale=scale_nest)\n", + "\n", + "scale_nest = mu / np.sqrt(mu**2 - 1.0)\n", + "nest_error_terms = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec), location=location_nest, scale=1.0/scale_nest)\n", "\n", "full_utils_2 = lower_level_utils_2 + nest_error_terms\n", "full_utils_3 = lower_level_utils_3 + nest_error_terms\n", @@ -386,16 +391,8 @@ "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", "\n", "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", - "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# WHY IS THIS NOT EQUAL\n", - "\n", - "see p.12 of Hildebrandt: scale in E(nu_m) and Var(nu_m) is not equal? Does this need correction factor for location?" + "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")\n", + "print(f\"{prob_1 / probs_dec[1]:.4f}, {prob_2 / probs_dec[2]:.4f}, {prob_3 / probs_dec[3]:.4f}\")" ] }, { @@ -405,20 +402,6 @@ "outputs": [], "source": [] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": {}, @@ -460,4 +443,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file From a4a76309fdbb0c1fd0ae7647a806e1db5724cbff Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 4 May 2022 20:53:30 +1000 Subject: [PATCH 003/135] adds choice based on walking nest levels --- activitysim/core/logit.py | 8 +- notebooks/frozen_rand_mode_choice_dev.ipynb | 382 ++++++++---------- ...ested_logit_frozen_rand_individ_util.ipynb | 8 +- 3 files changed, 174 insertions(+), 224 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 0359baab90..397e35e6c3 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -330,10 +330,12 @@ def __init__(self, name=None, level=0): self.ancestors = [] self.alternatives = None self.coefficient = 0 + self.parent_nest_scale = 1.0 def print(self): - print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s" % - (self.name, self.level, self.coefficient, self.product_of_coefficients, self.ancestors)) + print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s parent nest scale: " + "%s" % + (self.name, self.level, self.coefficient, self.product_of_coefficients, self.ancestors, self.parent_nest_scale)) @property def is_leaf(self): @@ -404,6 +406,7 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients * coefficient nest.alternatives = alternatives nest.ancestors = parent_nest.ancestors + [name] + nest.parent_nest_scale = parent_nest.coefficient if pre_order: yield spec, nest @@ -424,6 +427,7 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients nest.ancestors = parent_nest.ancestors + [name] nest.coefficient = parent_nest.coefficient + nest.parent_nest_scale = parent_nest.coefficient yield spec, nest diff --git a/notebooks/frozen_rand_mode_choice_dev.ipynb b/notebooks/frozen_rand_mode_choice_dev.ipynb index b0228ab13f..7003eb7cd1 100644 --- a/notebooks/frozen_rand_mode_choice_dev.ipynb +++ b/notebooks/frozen_rand_mode_choice_dev.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "execution": { @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "execution": { @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -94,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -113,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -152,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -210,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 10, "outputs": [], "source": [ "def run_trip_mode_choice(do_these_purposes=None, simulate_function=simulate.simple_simulate):\n", @@ -375,7 +375,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -385,19 +385,43 @@ "shell.execute_reply": "2021-09-01T02:29:01.222865Z", "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" }, - "tags": [], - "pycharm": { - "is_executing": true - } + "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n", + "trip_mode_choice tour_type 'eatout' (33 trips)\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n", + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", + "trip_mode_choice tour_type 'othmaint' (46 trips)\n", + "trip_mode_choice tour_type 'school' (37 trips)\n", + "trip_mode_choice tour_type 'shopping' (77 trips)\n", + "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'univ' (26 trips)\n", + "trip_mode_choice tour_type 'work' (168 trips)\n", + "Done\n" + ] + } + ], "source": [ "trips_df = run_trip_mode_choice()" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 12, "id": "7eae2854-38bd-4da0-a161-3a15bbd17177", "metadata": { "execution": { @@ -415,7 +439,7 @@ "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n8684833 26478 26478 eatout 1 True \n8684837 26478 26478 eatout 1 False \n8685009 26478 26478 othmaint 1 True \n8685013 26478 26478 othmaint 1 False \n8753057 26686 26686 eatout 1 True \n... ... ... ... ... ... \n2472945113 7539466 2848131 shopping 1 True \n2472945117 7539466 2848131 shopping 1 False \n2472945118 7539466 2848131 shopping 2 False \n2473024473 7539708 2848373 univ 1 True \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n8684833 1 13 8 1085604 eatout \n8684837 1 8 13 1085604 home \n8685009 1 10 8 1085626 othmaint \n8685013 1 8 10 1085626 home \n8753057 1 5 8 1094132 eatout \n... ... ... ... ... ... \n2472945113 1 8 3 309118139 shopping \n2472945117 2 25 8 309118139 shopping \n2472945118 2 3 25 309118139 home \n2473024473 1 13 18 309128059 univ \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n8684833 NaN 11.0 WALK -1.171760 \n8684837 NaN 11.0 WALK -1.238719 \n8685009 NaN 12.0 BIKE 6.198626 \n8685013 NaN 13.0 BIKE 6.175681 \n8753057 NaN 19.0 WALK 4.457539 \n... ... ... ... ... \n2472945113 NaN 18.0 WALK_LOC 12.537675 \n2472945117 56.842247 21.0 WALK_LOC 11.880804 \n2472945118 NaN 22.0 WALK 13.710030 \n2473024473 NaN 16.0 WALK_LOC -0.530696 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[482 rows x 14 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86848332647826478eatout1True11381085604eatoutNaN11.0WALK-1.171760
86848372647826478eatout1False18131085604homeNaN11.0WALK-1.238719
86850092647826478othmaint1True11081085626othmaintNaN12.0BIKE6.198626
86850132647826478othmaint1False18101085626homeNaN13.0BIKE6.175681
87530572668626686eatout1True1581094132eatoutNaN19.0WALK4.457539
.............................................
247294511375394662848131shopping1True183309118139shoppingNaN18.0WALK_LOC12.537675
247294511775394662848131shopping1False2258309118139shopping56.84224721.0WALK_LOC11.880804
247294511875394662848131shopping2False2325309118139homeNaN22.0WALK13.710030
247302447375397082848373univ1True11318309128059univNaN16.0WALK_LOC-0.530696
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

482 rows × 14 columns

\n
" }, - "execution_count": 41, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -434,7 +458,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 13, "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", "metadata": { "execution": { @@ -455,7 +479,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 14, "id": "36c100b3-3e39-4950-a586-4d42be695eaa", "metadata": { "execution": { @@ -476,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 15, "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", "metadata": { "execution": { @@ -629,7 +653,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 16, "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", "metadata": {}, "outputs": [ @@ -646,35 +670,7 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "trip_mode_choice tour_type 'escort' (6 trips)\n", "Done\n" ] }, @@ -683,7 +679,7 @@ "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n137248721 418441 304036 escort 1 True \n137248725 418441 304036 escort 1 False \n211388201 644476 386761 escort 1 True \n211388205 644476 386761 escort 1 False \n806388401 2458501 1173905 escort 1 True \n806388405 2458501 1173905 escort 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n137248721 1 7 10 17156090 escort \n137248725 1 10 7 17156090 home \n211388201 1 11 16 26423525 escort \n211388205 1 16 11 26423525 home \n806388401 1 16 8 100798550 escort \n806388405 1 8 16 100798550 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n137248721 NaN 7.0 WALK 11.435800 \n137248725 NaN 7.0 WALK 11.480440 \n211388201 NaN 5.0 WALK_LOC 4.789158 \n211388205 NaN 6.0 WALK_LOC 5.050171 \n806388401 NaN 15.0 WALK_LOC 6.451457 \n806388405 NaN 16.0 WALK_LOC 6.446188 ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
137248721418441304036escort1True171017156090escortNaN7.0WALK11.435800
137248725418441304036escort1False110717156090homeNaN7.0WALK11.480440
211388201644476386761escort1True1111626423525escortNaN5.0WALK_LOC4.789158
211388205644476386761escort1False1161126423525homeNaN6.0WALK_LOC5.050171
80638840124585011173905escort1True1168100798550escortNaN15.0WALK_LOC6.451457
80638840524585011173905escort1False1816100798550homeNaN16.0WALK_LOC6.446188
\n
" }, - "execution_count": 48, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -707,7 +703,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 61, "outputs": [], "source": [ "def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator,\n", @@ -956,7 +952,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 62, "outputs": [ { "name": "stderr", @@ -971,35 +967,7 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "trip_mode_choice tour_type 'escort' (6 trips)\n", "Done\n" ] } @@ -1016,7 +984,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 63, "outputs": [ { "data": { @@ -1025,17 +993,10 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/plain": "{'name': 'root',\n 'coefficient': 1.0,\n 'alternatives': [{'name': 'AUTO',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'DRIVEALONE',\n 'coefficient': 0.35,\n 'alternatives': ['DRIVEALONEFREE', 'DRIVEALONEPAY']},\n {'name': 'SHAREDRIDE2',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED2FREE', 'SHARED2PAY']},\n {'name': 'SHAREDRIDE3',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED3FREE', 'SHARED3PAY']}]},\n {'name': 'NONMOTORIZED',\n 'coefficient': 0.72,\n 'alternatives': ['WALK', 'BIKE']},\n {'name': 'TRANSIT',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'WALKACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['WALK_LOC',\n 'WALK_LRF',\n 'WALK_EXP',\n 'WALK_HVY',\n 'WALK_COM']},\n {'name': 'DRIVEACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['DRIVE_LOC',\n 'DRIVE_LRF',\n 'DRIVE_EXP',\n 'DRIVE_HVY',\n 'DRIVE_COM']}]},\n {'name': 'RIDEHAIL',\n 'coefficient': 0.36,\n 'alternatives': ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']}]}" - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ - "display(nu[0], ns[0])" + "display(nu[0])#, ns[0])" ], "metadata": { "collapsed": false, @@ -1046,7 +1007,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 64, "outputs": [], "source": [ "# next: add error terms to alternatives and nodes - can be done with this right here by iterating over nests and\n", @@ -1060,7 +1021,7 @@ " #quantile function of EV1\n", " # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution where the scale is proportional to variance (not variance^{-1})\n", " # this means nested scales are between 0 and 1\n", - " # x can be number or np array for vecops\n", + " # x can be number or np array or pd df for vecops\n", " return location - scale * np.log(-np.log(x))" ], "metadata": { @@ -1072,7 +1033,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 65, "outputs": [], "source": [ "utils_df = nu[0]\n", @@ -1087,49 +1048,125 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 66, + "outputs": [], + "source": [ + "# fake random channel for prototyping as per Asim tests\n", + "from activitysim.core.random import Random\n", + "rng = Random()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 102, + "outputs": [], + "source": [ + "nest_utils_for_choice = utils_df.copy() # we'll add random parts to this such that we can recursively choose from\n", + "# the top level\n", + "for n in logit.each_nest(nest_spec):\n", + " if n.level == 1:\n", + " assert n.name == \"root\" # TODO get this from where ever const is defined in code\n", + " continue\n", + " #n.print()\n", + " #print(nest_utils_for_choice.loc[:,n.name])\n", + " # TODO: check parent nest level scale is what we want this is right\n", + " rands = inverse_ev1_cdf(rng.random_for_df(nest_utils_for_choice, n=1), scale=n.parent_nest_scale)\n", + " #print(rands)\n", + " # this will be cleaner wtith xarrays\n", + " nest_utils_for_choice.loc[:,n.name] += rands[:,0]\n", + " #print(nest_utils_for_choice.loc[:,n.name])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 103, + "outputs": [], + "source": [ + "# alts = [\"DRIVEALONEFREE\", \"DRIVEALONEPAY\"]\n", + "# #print(nest_utils_for_choice[alts])\n", + "# t_ = nest_utils_for_choice[alts].idxmax(1)\n", + "# t_.apply(is_alternative)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 104, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nest name: DRIVEALONEFREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE']\n", - "Nest name: DRIVEALONEPAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY']\n", - "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE']\n", - "Nest name: SHARED2FREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2FREE']\n", - "Nest name: SHARED2PAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2PAY']\n", - "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2']\n", - "Nest name: SHARED3FREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3FREE']\n", - "Nest name: SHARED3PAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3PAY']\n", - "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3']\n", - "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO']\n", - "Nest name: WALK level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'WALK']\n", - "Nest name: BIKE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'BIKE']\n", - "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED']\n", - "Nest name: WALK_LOC level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LOC']\n", - "Nest name: WALK_LRF level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LRF']\n", - "Nest name: WALK_EXP level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_EXP']\n", - "Nest name: WALK_HVY level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_HVY']\n", - "Nest name: WALK_COM level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_COM']\n", - "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS']\n", - "Nest name: DRIVE_LOC level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LOC']\n", - "Nest name: DRIVE_LRF level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LRF']\n", - "Nest name: DRIVE_EXP level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_EXP']\n", - "Nest name: DRIVE_HVY level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_HVY']\n", - "Nest name: DRIVE_COM level: 4 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_COM']\n", - "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS']\n", - "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT']\n", - "Nest name: TAXI level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TAXI']\n", - "Nest name: TNC_SINGLE level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SINGLE']\n", - "Nest name: TNC_SHARED level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SHARED']\n", - "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL']\n", - "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root']\n" + "Nesting depth is 4\n" ] } ], "source": [ - "for n in logit.each_nest(nest_spec, post_order=True):\n", - " n.print()" + "all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, logit.each_nest(nest_spec))))\n", + "def is_alternative(name):\n", + " return name in all_alternatives\n", + "\n", + "def group_nests_by_level(nest_spec):\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", + " print(f\"Nesting depth is {depth}\")\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in logit.each_nest(nest_spec):\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " return nest_levels\n", + "\n", + "nest_utils_for_choice[\"choice\"] = None\n", + "\n", + "for level, alts in group_nests_by_level(nest_spec).items():\n", + " if level == 1:\n", + " continue\n", + " no_choices_made_yet = nest_utils_for_choice[\"choice\"].isnull()\n", + " choice_this_level = nest_utils_for_choice.loc[no_choices_made_yet][alts].idxmax(1)\n", + " nest_utils_for_choice.loc[no_choices_made_yet, \"choice\"] = \\\n", + " np.where(choice_this_level.apply(is_alternative), choice_this_level, None)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 105, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.116115 -2855.401829 -0.085418 -0.101729 \n137248725 -0.865187 -2855.150902 0.350254 0.148855 \n211388201 -2855.281895 -5709.567609 -inf -17.128230 \n211388205 -2856.635577 -5710.921291 -inf -17.918686 \n806388401 -2855.821604 -5710.107318 -inf -9.031970 \n806388405 -2857.992149 -5712.277863 -inf -10.161818 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.387444 0.269616 -2855.394889 -5709.680604 -inf \n137248725 -2854.136859 0.705169 -2855.144442 -5709.430157 -inf \n211388201 -2871.413945 -5.588125 -20.987850 -2875.273564 -6.938992 \n211388205 -2872.204400 -5.973433 -21.553014 -2875.838729 -7.245448 \n806388401 -2863.317684 -3.070264 -10.680802 -2864.966516 -3.647355 \n806388405 -2864.447532 -3.062176 -11.394371 -2865.680085 -3.493570 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.822679 16.250835 -1367.832304 11.946618 -1970.527596 \n137248725 1.416687 16.732000 -1367.382139 12.573430 -1970.304027 \n211388201 -3.529757 -3.726114 -1380.907142 -2.354947 9.913300 \n211388205 -3.882701 -3.857039 -1381.038067 -2.536786 10.345617 \n806388401 -1.816422 6.302066 -1378.371156 4.610776 12.685721 \n806388405 -1.445835 6.594584 -1377.923639 5.146648 13.052200 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.527596 -3968.527596 -3968.527596 -3968.527596 -inf \n137248725 -3968.304027 -3968.304027 -3968.304027 -3968.304027 -inf \n211388201 -1987.688271 -1987.688271 -1987.688271 -1987.688271 5.276609 \n211388205 -1987.654383 -1987.654383 -1987.654383 -1987.654383 5.407303 \n806388401 -1984.132938 -1983.673938 -1984.048738 -1984.094938 6.414384 \n806388405 -1983.897400 -1983.438400 -1983.813200 -1983.859400 6.915048 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.647702 -3973.145591 -3973.145591 -3974.414085 -3975.433221 \n137248725 -1970.656867 -3975.391993 -3975.391993 -3971.028330 -3977.511255 \n211388201 -1988.046728 -3990.654273 -3990.654273 -1993.837992 -1996.844325 \n211388205 -1987.882804 -3992.966000 -3992.966000 -1990.428019 -1997.126285 \n806388401 -1985.366765 -3989.217824 -3989.217824 -3990.767471 -3993.606059 \n806388405 -1985.037673 -3990.311044 -3990.311044 -3986.801670 -3993.797451 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.319775 -24.959142 -24.065572 -7.937642 \n137248725 -inf -inf -23.947836 -24.710809 -23.747349 -7.313866 \n211388201 -inf 4.127014 -15.616441 -3.619423 -4.823285 -0.615994 \n211388205 -inf 4.133540 -15.596812 -3.665464 -4.825869 -0.787171 \n806388401 -inf 4.691645 -19.088912 -19.441124 -16.306290 -5.701741 \n806388405 -inf 5.377382 -18.588659 -19.197216 -16.002477 -5.000992 \n\n root choice \ntrip_id \n137248721 11.435801 WALK \n137248725 11.480441 WALK \n211388201 3.456910 WALK_LOC \n211388205 3.643469 WALK_LOC \n806388401 5.193789 WALK_LOC \n806388405 5.134877 WALK_LOC ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILrootchoice
trip_id
137248721-1.116115-2855.401829-0.085418-0.101729-2854.3874440.269616-2855.394889-5709.680604-inf0.82267916.250835-1367.83230411.946618-1970.527596-3968.527596-3968.527596-3968.527596-3968.527596-inf-1970.647702-3973.145591-3973.145591-3974.414085-3975.433221-inf-inf-24.319775-24.959142-24.065572-7.93764211.435801WALK
137248725-0.865187-2855.1509020.3502540.148855-2854.1368590.705169-2855.144442-5709.430157-inf1.41668716.732000-1367.38213912.573430-1970.304027-3968.304027-3968.304027-3968.304027-3968.304027-inf-1970.656867-3975.391993-3975.391993-3971.028330-3977.511255-inf-inf-23.947836-24.710809-23.747349-7.31386611.480441WALK
211388201-2855.281895-5709.567609-inf-17.128230-2871.413945-5.588125-20.987850-2875.273564-6.938992-3.529757-3.726114-1380.907142-2.3549479.913300-1987.688271-1987.688271-1987.688271-1987.6882715.276609-1988.046728-3990.654273-3990.654273-1993.837992-1996.844325-inf4.127014-15.616441-3.619423-4.823285-0.6159943.456910WALK_LOC
211388205-2856.635577-5710.921291-inf-17.918686-2872.204400-5.973433-21.553014-2875.838729-7.245448-3.882701-3.857039-1381.038067-2.53678610.345617-1987.654383-1987.654383-1987.654383-1987.6543835.407303-1987.882804-3992.966000-3992.966000-1990.428019-1997.126285-inf4.133540-15.596812-3.665464-4.825869-0.7871713.643469WALK_LOC
806388401-2855.821604-5710.107318-inf-9.031970-2863.317684-3.070264-10.680802-2864.966516-3.647355-1.8164226.302066-1378.3711564.61077612.685721-1984.132938-1983.673938-1984.048738-1984.0949386.414384-1985.366765-3989.217824-3989.217824-3990.767471-3993.606059-inf4.691645-19.088912-19.441124-16.306290-5.7017415.193789WALK_LOC
806388405-2857.992149-5712.277863-inf-10.161818-2864.447532-3.062176-11.394371-2865.680085-3.493570-1.4458356.594584-1377.9236395.14664813.052200-1983.897400-1983.438400-1983.813200-1983.8594006.915048-1985.037673-3990.311044-3990.311044-3986.801670-3993.797451-inf5.377382-18.588659-19.197216-16.002477-5.0009925.134877WALK_LOC
\n
" + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nest_utils_for_choice" ], "metadata": { "collapsed": false, @@ -1342,97 +1379,6 @@ "assert nest_levels[1][0] == 'root'" ] }, - { - "cell_type": "code", - "execution_count": 101, - "id": "bae6e7a8-d7c3-42a2-a82f-874b1e59a0b6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:56:38.377472Z", - "iopub.status.busy": "2021-09-01T01:56:38.377253Z", - "iopub.status.idle": "2021-09-01T01:56:38.607858Z", - "shell.execute_reply": "2021-09-01T01:56:38.606913Z", - "shell.execute_reply.started": "2021-09-01T01:56:38.377447Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests_), axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 333, - "id": "f0e00814-2a14-4da1-bc6c-89b52b7fb286", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:58:26.002425Z", - "iopub.status.busy": "2021-09-01T01:58:26.002200Z", - "iopub.status.idle": "2021-09-01T01:58:26.247954Z", - "shell.execute_reply": "2021-09-01T01:58:26.247054Z", - "shell.execute_reply.started": "2021-09-01T01:58:26.002401Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 WALK\n", - "137248725 WALK\n", - "211388201 WALK_LOC\n", - "211388205 WALK_LOC\n", - "806388401 WALK\n", - "806388405 WALK\n", - "dtype: object" - ] - }, - "execution_count": 333, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 348, - "id": "bca659db-4c88-4287-8f5e-9c9516e3ef41", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:59:00.410062Z", - "iopub.status.busy": "2021-09-01T01:59:00.409811Z", - "iopub.status.idle": "2021-09-01T01:59:00.621940Z", - "shell.execute_reply": "2021-09-01T01:59:00.621156Z", - "shell.execute_reply.started": "2021-09-01T01:59:00.410037Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 WALK\n", - "137248725 WALK\n", - "211388201 WALK_LOC\n", - "211388205 WALK_LOC\n", - "806388401 WALK_LOC\n", - "806388405 WALK_LOC\n", - "dtype: object" - ] - }, - "execution_count": 348, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "make_choice(utils_df, nests_, nest_levels)" - ] - }, { "cell_type": "code", "execution_count": 453, diff --git a/notebooks/nested_logit_frozen_rand_individ_util.ipynb b/notebooks/nested_logit_frozen_rand_individ_util.ipynb index 2578b9068e..dede399ca8 100644 --- a/notebooks/nested_logit_frozen_rand_individ_util.ipynb +++ b/notebooks/nested_logit_frozen_rand_individ_util.ipynb @@ -171,8 +171,8 @@ "text": [ "closed form: [0.4452265282367507, 0.5330453677531714, 0.02172810401007798],\n", "simulated: [0.4450967 0.5331544 0.0217489]\n", - "CPU times: user 969 ms, sys: 78.1 ms, total: 1.05 s\n", - "Wall time: 1.14 s\n" + "CPU times: user 750 ms, sys: 78.1 ms, total: 828 ms\n", + "Wall time: 811 ms\n" ] } ], @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 8, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T10:02:14.035272Z", @@ -257,7 +257,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T10:03:23.999857Z", From c6b609afd6ffc881964ba560d66b2a12c8248d0f Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 4 May 2022 21:43:07 +1000 Subject: [PATCH 004/135] prototype simple_simulate --- activitysim/core/logit.py | 53 +++++++++ activitysim/core/simulate.py | 116 ++++++++++++++++++++ notebooks/frozen_rand_mode_choice_dev.ipynb | 108 ++++++++---------- 3 files changed, 216 insertions(+), 61 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 397e35e6c3..18d933cbd6 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -184,6 +184,49 @@ def utils_to_probs(utils, trace_label=None, exponentiated=False, allow_zero_prob return probs +def inverse_ev1_cdf(x, location=0.0, scale=1.0): + # quantile function of EV1 + # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution (and biogeme and larch convention) where the scale + # is proportional to variance (not variance^{-1}). this means nested scales are between 0 and 1. + # x can be number or np array or pd df for vecops + return location - scale * np.log(-np.log(x)) + + +def add_ev1_random(df, nest_spec): + # TODO: generalise to logit for nest_spec==None by adding one rand (scale=1) to all columns + nest_utils_for_choice = df.copy() + for n in each_nest(nest_spec): + if n.level == 1: + continue # skip the root level + # TODO: check parent nest level scale is what we want this is right + uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice) + rands = inverse_ev1_cdf(uniform_rands, scale=n.parent_nest_scale) + nest_utils_for_choice.loc[:, n.name] += rands[:, 0] # inverse_ev1_cdf of single-row df adds dimension + return nest_utils_for_choice + + +def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_choosers=None): + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') + nest_utils_for_choice = add_ev1_random(nested_utilities, nest_spec) + all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, each_nest(nest_spec)))) + + def is_alternative(name): + return name in all_alternatives + + nest_utils_for_choice["choice"] = None + for level, alts in group_nests_by_level(nest_spec).items(): + if level == 1: + continue + no_choices_made_yet = nest_utils_for_choice["choice"].isnull() + choice_this_level = nest_utils_for_choice.loc[no_choices_made_yet][alts].idxmax(1) + nest_utils_for_choice.loc[no_choices_made_yet, "choice"] = \ + np.where(choice_this_level.apply(is_alternative), choice_this_level, None) + + assert not nest_utils_for_choice["choice"].isnull.any(), "No choice for XXX - implement reporting" + choices = pd.Series(nest_utils_for_choice["choice"], index=nested_utilities.index) + return choices + + def make_choices(probs, trace_label=None, trace_choosers=None, allow_bad_probs=False): """ Make choices for each chooser from among a set of alternatives. @@ -475,3 +518,13 @@ def count_each_nest(spec, count): return 1 return count_each_nest(nest_spec, 0) if nest_spec is not None else 0 + + +def group_nests_by_level(nest_spec): + # group nests by level, returns {level: [nest.name at that level]} + depth = np.max([x.level for x in each_nest(nest_spec)]) + nest_levels = {x: [] for x in range(1, depth+1)} + for n in each_nest(nest_spec): + nest_levels[n.level].append(n.name) + assert len(nest_levels[1]) == 1 # only one root + return nest_levels diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 6eff3c2e9f..6f659606c6 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -964,6 +964,122 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, return choices +def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, + log_alt_losers=False, + want_logsums=False, trace_label=None, + trace_choice_name=None, trace_column_names=None): + """ + Run a nested-logit simulation for when the model spec does not involve alternative + specific data, e.g. there are no interactions with alternative + properties and no need to sample from alternatives. + + Parameters + ---------- + choosers : pandas.DataFrame + spec : pandas.DataFrame + A table of variable specifications and coefficient values. + Variable expressions should be in the table index and the table + should have a column for each alternative. + nest_spec: + dictionary specifying nesting structure and nesting coefficients + (from the model spec yaml file) + locals_d : Dict or None + This is a dictionary of local variables that will be the environment + for an evaluation of an expression that begins with @ + custom_chooser : function(probs, choosers, spec, trace_label) returns choices, rands + custom alternative to logit.make_choices + estimator : Estimator object + called to report intermediate table results (used for estimation) + trace_label: str + This is the label to be used for trace log file entries and dump file names + when household tracing enabled. No tracing occurs if label is empty or None. + trace_choice_name: str + This is the column label to be used in trace file csv dump of choices + trace_column_names: str or list of str + chooser columns to include when tracing expression_values + + Returns + ------- + choices : pandas.Series + Index will be that of `choosers`, values will match the columns + of `spec`. + """ + + trace_label = tracing.extend_trace_label(trace_label, 'eval_nl_fixed_ru') + assert trace_label + have_trace_targets = tracing.has_trace_targets(choosers) + + logit.validate_nest_spec(nest_spec, trace_label) + + if have_trace_targets: + tracing.trace_df(choosers, '%s.choosers' % trace_label) + + raw_utilities = eval_utilities(spec, choosers, locals_d, + log_alt_losers=log_alt_losers, + trace_label=trace_label, have_trace_targets=have_trace_targets, + estimator=estimator, trace_column_names=trace_column_names) + chunk.log_df(trace_label, "raw_utilities", raw_utilities) + + if have_trace_targets: + tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label, + column_labels=['alternative', 'utility']) + + # utilities of leaves and nests + nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) + chunk.log_df(trace_label, "nested_utilities", nested_utilities) + + del raw_utilities + chunk.log_df(trace_label, 'raw_utilities', None) + + if have_trace_targets: + tracing.trace_df(nested_utilities, '%s.nested_utilities' % trace_label, + column_labels=['alternative', 'utility']) + + # TODO - check this is correct + if want_logsums: + # logsum of nest root + logsums = pd.Series(nested_utilities.root, index=choosers.index) + chunk.log_df(trace_label, "logsums", logsums) + + + # TODO: add checks on utilities? + # # note base_probabilities could all be zero since we allowed all probs for nests to be zero + # # check here to print a clear message but make_choices will raise error if probs don't sum to 1 + # BAD_PROB_THRESHOLD = 0.001 + # no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD + # + # if no_choices.any(): + # + # logit.report_bad_choices( + # no_choices, base_probabilities, + # trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'), + # trace_choosers=choosers, + # msg="base_probabilities do not sum to one") + + # TODO: add custom_chooser + # if custom_chooser: + # choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, + # trace_label=trace_label) + # else: + + choices = logit.make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=trace_label) + + if have_trace_targets: + tracing.trace_df(choices, '%s.choices' % trace_label, + columns=[None, trace_choice_name]) + #tracing.trace_df(rands, '%s.rands' % trace_label, + # columns=[None, 'rand']) + if want_logsums: + tracing.trace_df(logsums, '%s.logsums' % trace_label, + columns=[None, 'logsum']) + + if want_logsums: + choices = choices.to_frame('choice') + choices['logsum'] = logsums + + return choices + + def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, log_alt_losers=False, want_logsums=False, trace_label=None, diff --git a/notebooks/frozen_rand_mode_choice_dev.ipynb b/notebooks/frozen_rand_mode_choice_dev.ipynb index 7003eb7cd1..4a85f62408 100644 --- a/notebooks/frozen_rand_mode_choice_dev.ipynb +++ b/notebooks/frozen_rand_mode_choice_dev.ipynb @@ -1033,27 +1033,24 @@ }, { "cell_type": "code", - "execution_count": 65, - "outputs": [], - "source": [ - "utils_df = nu[0]\n", - "nest_spec = ns[0]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 66, + "execution_count": 112, "outputs": [], "source": [ "# fake random channel for prototyping as per Asim tests\n", "from activitysim.core.random import Random\n", - "rng = Random()" + "rng = Random()\n", + "\n", + "def add_ev1_random(df, nest_spec):\n", + " # TODO: generalise to logit for nest_spec==None by adding one rand (scale=1) to all columns\n", + " nest_utils_for_choice = df.copy() # we'll add random parts to this such that we can recursively choose from\n", + " for n in logit.each_nest(nest_spec):\n", + " if n.level == 1:\n", + " continue # skip the root level\n", + " # TODO: check parent nest level scale is what we want this is right\n", + " rands = inverse_ev1_cdf(rng.random_for_df(nest_utils_for_choice, n=1), scale=n.parent_nest_scale)\n", + " # this will be cleaner wtith xarrays\n", + " nest_utils_for_choice.loc[:,n.name] += rands[:,0]\n", + " return nest_utils_for_choice" ], "metadata": { "collapsed": false, @@ -1064,23 +1061,12 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 115, "outputs": [], "source": [ - "nest_utils_for_choice = utils_df.copy() # we'll add random parts to this such that we can recursively choose from\n", - "# the top level\n", - "for n in logit.each_nest(nest_spec):\n", - " if n.level == 1:\n", - " assert n.name == \"root\" # TODO get this from where ever const is defined in code\n", - " continue\n", - " #n.print()\n", - " #print(nest_utils_for_choice.loc[:,n.name])\n", - " # TODO: check parent nest level scale is what we want this is right\n", - " rands = inverse_ev1_cdf(rng.random_for_df(nest_utils_for_choice, n=1), scale=n.parent_nest_scale)\n", - " #print(rands)\n", - " # this will be cleaner wtith xarrays\n", - " nest_utils_for_choice.loc[:,n.name] += rands[:,0]\n", - " #print(nest_utils_for_choice.loc[:,n.name])" + "utils_df = nu[0]\n", + "nest_spec = ns[0]\n", + "nest_utils_for_choice = add_ev1_random(utils_df, nest_spec)" ], "metadata": { "collapsed": false, @@ -1091,13 +1077,20 @@ }, { "cell_type": "code", - "execution_count": 103, - "outputs": [], + "execution_count": 116, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.116115 -2855.401829 -0.085418 -0.101729 \n137248725 -0.865187 -2855.150902 0.350254 0.148855 \n211388201 -2855.281895 -5709.567609 -inf -17.128230 \n211388205 -2856.635577 -5710.921291 -inf -17.918686 \n806388401 -2855.821604 -5710.107318 -inf -9.031970 \n806388405 -2857.992149 -5712.277863 -inf -10.161818 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.387444 0.269616 -2855.394889 -5709.680604 -inf \n137248725 -2854.136859 0.705169 -2855.144442 -5709.430157 -inf \n211388201 -2871.413945 -5.588125 -20.987850 -2875.273564 -6.938992 \n211388205 -2872.204400 -5.973433 -21.553014 -2875.838729 -7.245448 \n806388401 -2863.317684 -3.070264 -10.680802 -2864.966516 -3.647355 \n806388405 -2864.447532 -3.062176 -11.394371 -2865.680085 -3.493570 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.822679 16.250835 -1367.832304 11.946618 -1970.527596 \n137248725 1.416687 16.732000 -1367.382139 12.573430 -1970.304027 \n211388201 -3.529757 -3.726114 -1380.907142 -2.354947 9.913300 \n211388205 -3.882701 -3.857039 -1381.038067 -2.536786 10.345617 \n806388401 -1.816422 6.302066 -1378.371156 4.610776 12.685721 \n806388405 -1.445835 6.594584 -1377.923639 5.146648 13.052200 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.527596 -3968.527596 -3968.527596 -3968.527596 -inf \n137248725 -3968.304027 -3968.304027 -3968.304027 -3968.304027 -inf \n211388201 -1987.688271 -1987.688271 -1987.688271 -1987.688271 5.276609 \n211388205 -1987.654383 -1987.654383 -1987.654383 -1987.654383 5.407303 \n806388401 -1984.132938 -1983.673938 -1984.048738 -1984.094938 6.414384 \n806388405 -1983.897400 -1983.438400 -1983.813200 -1983.859400 6.915048 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.647702 -3973.145591 -3973.145591 -3974.414085 -3975.433221 \n137248725 -1970.656867 -3975.391993 -3975.391993 -3971.028330 -3977.511255 \n211388201 -1988.046728 -3990.654273 -3990.654273 -1993.837992 -1996.844325 \n211388205 -1987.882804 -3992.966000 -3992.966000 -1990.428019 -1997.126285 \n806388401 -1985.366765 -3989.217824 -3989.217824 -3990.767471 -3993.606059 \n806388405 -1985.037673 -3990.311044 -3990.311044 -3986.801670 -3993.797451 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.319775 -24.959142 -24.065572 -7.937642 \n137248725 -inf -inf -23.947836 -24.710809 -23.747349 -7.313866 \n211388201 -inf 4.127014 -15.616441 -3.619423 -4.823285 -0.615994 \n211388205 -inf 4.133540 -15.596812 -3.665464 -4.825869 -0.787171 \n806388401 -inf 4.691645 -19.088912 -19.441124 -16.306290 -5.701741 \n806388405 -inf 5.377382 -18.588659 -19.197216 -16.002477 -5.000992 \n\n root \ntrip_id \n137248721 11.435801 \n137248725 11.480441 \n211388201 3.456910 \n211388205 3.643469 \n806388401 5.193789 \n806388405 5.134877 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.116115-2855.401829-0.085418-0.101729-2854.3874440.269616-2855.394889-5709.680604-inf0.82267916.250835-1367.83230411.946618-1970.527596-3968.527596-3968.527596-3968.527596-3968.527596-inf-1970.647702-3973.145591-3973.145591-3974.414085-3975.433221-inf-inf-24.319775-24.959142-24.065572-7.93764211.435801
137248725-0.865187-2855.1509020.3502540.148855-2854.1368590.705169-2855.144442-5709.430157-inf1.41668716.732000-1367.38213912.573430-1970.304027-3968.304027-3968.304027-3968.304027-3968.304027-inf-1970.656867-3975.391993-3975.391993-3971.028330-3977.511255-inf-inf-23.947836-24.710809-23.747349-7.31386611.480441
211388201-2855.281895-5709.567609-inf-17.128230-2871.413945-5.588125-20.987850-2875.273564-6.938992-3.529757-3.726114-1380.907142-2.3549479.913300-1987.688271-1987.688271-1987.688271-1987.6882715.276609-1988.046728-3990.654273-3990.654273-1993.837992-1996.844325-inf4.127014-15.616441-3.619423-4.823285-0.6159943.456910
211388205-2856.635577-5710.921291-inf-17.918686-2872.204400-5.973433-21.553014-2875.838729-7.245448-3.882701-3.857039-1381.038067-2.53678610.345617-1987.654383-1987.654383-1987.654383-1987.6543835.407303-1987.882804-3992.966000-3992.966000-1990.428019-1997.126285-inf4.133540-15.596812-3.665464-4.825869-0.7871713.643469
806388401-2855.821604-5710.107318-inf-9.031970-2863.317684-3.070264-10.680802-2864.966516-3.647355-1.8164226.302066-1378.3711564.61077612.685721-1984.132938-1983.673938-1984.048738-1984.0949386.414384-1985.366765-3989.217824-3989.217824-3990.767471-3993.606059-inf4.691645-19.088912-19.441124-16.306290-5.7017415.193789
806388405-2857.992149-5712.277863-inf-10.161818-2864.447532-3.062176-11.394371-2865.680085-3.493570-1.4458356.594584-1377.9236395.14664813.052200-1983.897400-1983.438400-1983.813200-1983.8594006.915048-1985.037673-3990.311044-3990.311044-3986.801670-3993.797451-inf5.377382-18.588659-19.197216-16.002477-5.0009925.134877
\n
" + }, + "execution_count": 116, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# alts = [\"DRIVEALONEFREE\", \"DRIVEALONEPAY\"]\n", - "# #print(nest_utils_for_choice[alts])\n", - "# t_ = nest_utils_for_choice[alts].idxmax(1)\n", - "# t_.apply(is_alternative)" + "nest_utils_for_choice" ], "metadata": { "collapsed": false, @@ -1108,35 +1101,28 @@ }, { "cell_type": "code", - "execution_count": 104, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nesting depth is 4\n" - ] - } - ], + "execution_count": 106, + "outputs": [], "source": [ "all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, logit.each_nest(nest_spec))))\n", "def is_alternative(name):\n", " return name in all_alternatives\n", "\n", - "def group_nests_by_level(nest_spec):\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", - " print(f\"Nesting depth is {depth}\")\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in logit.each_nest(nest_spec):\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " return nest_levels\n", + "# NOW IN logit.group_nests_by_level(nest_spec)\n", + "# def group_nests_by_level(nest_spec):\n", + "# # group nests by level:\n", + "# depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", + "# #print(f\"Nesting depth is {depth}\")\n", + "# nest_levels = {x: [] for x in range(1, depth+1)}\n", + "# for n in logit.each_nest(nest_spec):\n", + "# nest_levels[n.level].append(n.name)\n", + "# assert len(nest_levels[1]) == 1\n", + "# # assert nest_levels[1][0] == 'root'\n", + "# return nest_levels\n", "\n", "nest_utils_for_choice[\"choice\"] = None\n", "\n", - "for level, alts in group_nests_by_level(nest_spec).items():\n", + "for level, alts in logit.group_nests_by_level(nest_spec).items():\n", " if level == 1:\n", " continue\n", " no_choices_made_yet = nest_utils_for_choice[\"choice\"].isnull()\n", @@ -1153,14 +1139,14 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 107, "outputs": [ { "data": { "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.116115 -2855.401829 -0.085418 -0.101729 \n137248725 -0.865187 -2855.150902 0.350254 0.148855 \n211388201 -2855.281895 -5709.567609 -inf -17.128230 \n211388205 -2856.635577 -5710.921291 -inf -17.918686 \n806388401 -2855.821604 -5710.107318 -inf -9.031970 \n806388405 -2857.992149 -5712.277863 -inf -10.161818 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.387444 0.269616 -2855.394889 -5709.680604 -inf \n137248725 -2854.136859 0.705169 -2855.144442 -5709.430157 -inf \n211388201 -2871.413945 -5.588125 -20.987850 -2875.273564 -6.938992 \n211388205 -2872.204400 -5.973433 -21.553014 -2875.838729 -7.245448 \n806388401 -2863.317684 -3.070264 -10.680802 -2864.966516 -3.647355 \n806388405 -2864.447532 -3.062176 -11.394371 -2865.680085 -3.493570 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.822679 16.250835 -1367.832304 11.946618 -1970.527596 \n137248725 1.416687 16.732000 -1367.382139 12.573430 -1970.304027 \n211388201 -3.529757 -3.726114 -1380.907142 -2.354947 9.913300 \n211388205 -3.882701 -3.857039 -1381.038067 -2.536786 10.345617 \n806388401 -1.816422 6.302066 -1378.371156 4.610776 12.685721 \n806388405 -1.445835 6.594584 -1377.923639 5.146648 13.052200 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.527596 -3968.527596 -3968.527596 -3968.527596 -inf \n137248725 -3968.304027 -3968.304027 -3968.304027 -3968.304027 -inf \n211388201 -1987.688271 -1987.688271 -1987.688271 -1987.688271 5.276609 \n211388205 -1987.654383 -1987.654383 -1987.654383 -1987.654383 5.407303 \n806388401 -1984.132938 -1983.673938 -1984.048738 -1984.094938 6.414384 \n806388405 -1983.897400 -1983.438400 -1983.813200 -1983.859400 6.915048 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.647702 -3973.145591 -3973.145591 -3974.414085 -3975.433221 \n137248725 -1970.656867 -3975.391993 -3975.391993 -3971.028330 -3977.511255 \n211388201 -1988.046728 -3990.654273 -3990.654273 -1993.837992 -1996.844325 \n211388205 -1987.882804 -3992.966000 -3992.966000 -1990.428019 -1997.126285 \n806388401 -1985.366765 -3989.217824 -3989.217824 -3990.767471 -3993.606059 \n806388405 -1985.037673 -3990.311044 -3990.311044 -3986.801670 -3993.797451 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.319775 -24.959142 -24.065572 -7.937642 \n137248725 -inf -inf -23.947836 -24.710809 -23.747349 -7.313866 \n211388201 -inf 4.127014 -15.616441 -3.619423 -4.823285 -0.615994 \n211388205 -inf 4.133540 -15.596812 -3.665464 -4.825869 -0.787171 \n806388401 -inf 4.691645 -19.088912 -19.441124 -16.306290 -5.701741 \n806388405 -inf 5.377382 -18.588659 -19.197216 -16.002477 -5.000992 \n\n root choice \ntrip_id \n137248721 11.435801 WALK \n137248725 11.480441 WALK \n211388201 3.456910 WALK_LOC \n211388205 3.643469 WALK_LOC \n806388401 5.193789 WALK_LOC \n806388405 5.134877 WALK_LOC ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILrootchoice
trip_id
137248721-1.116115-2855.401829-0.085418-0.101729-2854.3874440.269616-2855.394889-5709.680604-inf0.82267916.250835-1367.83230411.946618-1970.527596-3968.527596-3968.527596-3968.527596-3968.527596-inf-1970.647702-3973.145591-3973.145591-3974.414085-3975.433221-inf-inf-24.319775-24.959142-24.065572-7.93764211.435801WALK
137248725-0.865187-2855.1509020.3502540.148855-2854.1368590.705169-2855.144442-5709.430157-inf1.41668716.732000-1367.38213912.573430-1970.304027-3968.304027-3968.304027-3968.304027-3968.304027-inf-1970.656867-3975.391993-3975.391993-3971.028330-3977.511255-inf-inf-23.947836-24.710809-23.747349-7.31386611.480441WALK
211388201-2855.281895-5709.567609-inf-17.128230-2871.413945-5.588125-20.987850-2875.273564-6.938992-3.529757-3.726114-1380.907142-2.3549479.913300-1987.688271-1987.688271-1987.688271-1987.6882715.276609-1988.046728-3990.654273-3990.654273-1993.837992-1996.844325-inf4.127014-15.616441-3.619423-4.823285-0.6159943.456910WALK_LOC
211388205-2856.635577-5710.921291-inf-17.918686-2872.204400-5.973433-21.553014-2875.838729-7.245448-3.882701-3.857039-1381.038067-2.53678610.345617-1987.654383-1987.654383-1987.654383-1987.6543835.407303-1987.882804-3992.966000-3992.966000-1990.428019-1997.126285-inf4.133540-15.596812-3.665464-4.825869-0.7871713.643469WALK_LOC
806388401-2855.821604-5710.107318-inf-9.031970-2863.317684-3.070264-10.680802-2864.966516-3.647355-1.8164226.302066-1378.3711564.61077612.685721-1984.132938-1983.673938-1984.048738-1984.0949386.414384-1985.366765-3989.217824-3989.217824-3990.767471-3993.606059-inf4.691645-19.088912-19.441124-16.306290-5.7017415.193789WALK_LOC
806388405-2857.992149-5712.277863-inf-10.161818-2864.447532-3.062176-11.394371-2865.680085-3.493570-1.4458356.594584-1377.9236395.14664813.052200-1983.897400-1983.438400-1983.813200-1983.8594006.915048-1985.037673-3990.311044-3990.311044-3986.801670-3993.797451-inf5.377382-18.588659-19.197216-16.002477-5.0009925.134877WALK_LOC
\n
" }, - "execution_count": 105, + "execution_count": 107, "metadata": {}, "output_type": "execute_result" } From 5dc1a44ed9303e3269207f128a08842954ae49e1 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 4 May 2022 22:11:12 +1000 Subject: [PATCH 005/135] baby steps towards validation --- activitysim/core/logit.py | 5 +- activitysim/core/simulate.py | 29 +- notebooks/frozen_rand_mode_choice_dev.ipynb | 6 +- notebooks/validate_frozen_impl.ipynb | 2632 +++++++++++++++++++ 4 files changed, 2661 insertions(+), 11 deletions(-) create mode 100644 notebooks/validate_frozen_impl.ipynb diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 18d933cbd6..e3f87d4248 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -222,8 +222,11 @@ def is_alternative(name): nest_utils_for_choice.loc[no_choices_made_yet, "choice"] = \ np.where(choice_this_level.apply(is_alternative), choice_this_level, None) - assert not nest_utils_for_choice["choice"].isnull.any(), "No choice for XXX - implement reporting" + assert not nest_utils_for_choice["choice"].isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(nest_utils_for_choice["choice"], index=nested_utilities.index) + + assert not choices.isnull().any(), "No choice for XXX - implement reporting" + return choices diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 6f659606c6..4f9b7d9a2a 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1041,7 +1041,6 @@ def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estima logsums = pd.Series(nested_utilities.root, index=choosers.index) chunk.log_df(trace_label, "logsums", logsums) - # TODO: add checks on utilities? # # note base_probabilities could all be zero since we allowed all probs for nests to be zero # # check here to print a clear message but make_choices will raise error if probs don't sum to 1 @@ -1223,6 +1222,7 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, want_logsums=False, estimator=None, trace_label=None, trace_choice_name=None, trace_column_names=None, + choose_individual_max_utility=False ): """ Run an MNL or NL simulation for when the model spec does not involve alternative @@ -1273,6 +1273,7 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, set_skim_wrapper_targets(choosers, skims) if nest_spec is None: + # TODO: add frozen individual ru for mnl choices = eval_mnl(choosers, spec, locals_d, custom_chooser, log_alt_losers=log_alt_losers, want_logsums=want_logsums, @@ -1280,12 +1281,20 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, trace_label=trace_label, trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) else: - choices = eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, - log_alt_losers=log_alt_losers, - want_logsums=want_logsums, - estimator=estimator, - trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) + if choose_individual_max_utility: + choices = eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, + log_alt_losers=log_alt_losers, + want_logsums=want_logsums, + estimator=estimator, + trace_label=trace_label, + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) + else: + choices = eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, + log_alt_losers=log_alt_losers, + want_logsums=want_logsums, + estimator=estimator, + trace_label=trace_label, + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) return choices @@ -1308,7 +1317,8 @@ def simple_simulate(choosers, spec, nest_spec, log_alt_losers=False, want_logsums=False, estimator=None, - trace_label=None, trace_choice_name=None, trace_column_names=None): + trace_label=None, trace_choice_name=None, trace_column_names=None, + choose_individual_max_utility=False): """ Run an MNL or NL simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -1334,7 +1344,8 @@ def simple_simulate(choosers, spec, nest_spec, estimator=estimator, trace_label=chunk_trace_label, trace_choice_name=trace_choice_name, - trace_column_names=trace_column_names) + trace_column_names=trace_column_names, + choose_individual_max_utility=choose_individual_max_utility) result_list.append(choices) diff --git a/notebooks/frozen_rand_mode_choice_dev.ipynb b/notebooks/frozen_rand_mode_choice_dev.ipynb index 4a85f62408..101953e516 100644 --- a/notebooks/frozen_rand_mode_choice_dev.ipynb +++ b/notebooks/frozen_rand_mode_choice_dev.ipynb @@ -1189,7 +1189,11 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [], + "source": [ + "nest_utils_for_choice = utils_df.copy()\n", + "\n", + "logit.make_choices_ru_frozen(nest_utils_for_choice, nest_spec, trace_label=\"blah\")" + ], "metadata": { "collapsed": false, "pycharm": { diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb new file mode 100644 index 0000000000..1fd45d784c --- /dev/null +++ b/notebooks/validate_frozen_impl.ipynb @@ -0,0 +1,2632 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", + "metadata": {}, + "source": [ + "# validate results" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.385153Z", + "iopub.status.busy": "2022-05-01T10:17:57.384881Z", + "iopub.status.idle": "2022-05-01T10:17:57.534433Z", + "shell.execute_reply": "2022-05-01T10:17:57.533096Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3dba451-1e10-403e-8614-35d57e6577f4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.536623Z", + "iopub.status.busy": "2022-05-01T10:17:57.536012Z", + "iopub.status.idle": "2022-05-01T10:17:57.542755Z", + "shell.execute_reply": "2022-05-01T10:17:57.541685Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:00.350944Z", + "iopub.status.busy": "2022-05-01T10:18:00.350730Z", + "iopub.status.idle": "2022-05-01T10:18:12.760977Z", + "shell.execute_reply": "2022-05-01T10:18:12.760013Z", + "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "from datetime import datetime\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from numpy.random import default_rng\n", + "\n", + "from activitysim.cli import run\n", + "from activitysim.core import inject\n", + "from activitysim.core import tracing\n", + "from activitysim.core import config\n", + "from activitysim.core import pipeline\n", + "from activitysim.core import chunk\n", + "from activitysim.core import simulate\n", + "from activitysim.core import logit\n", + "from activitysim.abm.models.util import estimation\n", + "from activitysim.core import expressions\n", + "from activitysim.core.util import assign_in_place" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:16.944537Z", + "iopub.status.busy": "2022-05-01T10:18:16.944291Z", + "iopub.status.idle": "2022-05-01T10:18:17.124764Z", + "shell.execute_reply": "2022-05-01T10:18:17.123725Z", + "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "pd.set_option(\"max_columns\", 500)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:17.839947Z", + "iopub.status.busy": "2022-05-01T10:18:17.839070Z", + "iopub.status.idle": "2022-05-01T10:18:18.019676Z", + "shell.execute_reply": "2022-05-01T10:18:18.018689Z", + "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_example_mtc\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:18.894533Z", + "iopub.status.busy": "2022-05-01T10:18:18.894303Z", + "iopub.status.idle": "2022-05-01T10:18:19.078807Z", + "shell.execute_reply": "2022-05-01T10:18:19.077951Z", + "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "os.chdir(example_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:22.111723Z", + "iopub.status.busy": "2022-05-01T10:18:22.111490Z", + "iopub.status.idle": "2022-05-01T10:18:22.297437Z", + "shell.execute_reply": "2022-05-01T10:18:22.296501Z", + "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "run.add_run_args(parser)\n", + "# args = parser.parse_args()\n", + "# parser.parse_args(['--sum', '7', '-1', '42'])\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", + "#run.run(args) # 2mins full example run\n", + "\n", + "\n", + "if not inject.is_injectable('preload_injectables'):\n", + " from activitysim import abm # register abm steps and other abm-specific injectables\n", + "run.handle_standard_args(args) # possibly update injectables" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "outputs": [], + "source": [ + "def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True):\n", + "\n", + " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", + " resume_after = \"trip_scheduling\"\n", + " model_name = \"trip_mode_choice\"\n", + " chunk_size = 0 # test_mtc means no chunking\n", + "\n", + " pipeline.open_pipeline(resume_after)\n", + " # preload any bulky injectables (e.g. skims) not in pipeline\n", + " inject.get_injectable('preload_injectables', None)\n", + " pipeline._PIPELINE.rng().begin_step(model_name)\n", + " step_name = model_name\n", + " args = {}\n", + " checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + " inject.set_step_args(args)\n", + "\n", + " trips = inject.get_table('trips')\n", + " tours_merged = inject.get_table('tours_merged')\n", + " network_los = inject.get_injectable('network_los')\n", + "\n", + " trace_label = 'trip_mode_choice'\n", + " model_settings_file_name = 'trip_mode_choice.yaml'\n", + " model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + " mode_column_name = 'trip_mode'\n", + "\n", + " trips_df = trips.to_frame()\n", + " print(\"Running with %d trips\", trips_df.shape[0])\n", + "\n", + " tours_merged = tours_merged.to_frame()\n", + " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "\n", + " # - trips_merged - merge trips and tours_merged\n", + " trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + " assert trips_merged.index.equals(trips.index)\n", + "\n", + " # setup skim keys\n", + " assert ('trip_period' not in trips_merged)\n", + " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + " orig_col = 'origin'\n", + " dest_col = 'destination'\n", + "\n", + " constants = {}\n", + " constants.update(config.get_model_constants(model_settings))\n", + " constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + " })\n", + "\n", + " skim_dict = network_los.get_default_skim_dict()\n", + "\n", + " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + " skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + " }\n", + "\n", + " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + " nest_specs = config.get_logit_model_settings(model_settings)\n", + "\n", + " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + "\n", + " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", + " continue\n", + "\n", + " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " (primary_purpose, len(trips_segment.index), ))\n", + "\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " ################ Replace wrapper function\n", + " # choices = mode_choice_simulate(...)\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", + " choices = simulate.simple_simulate(\n", + " choosers=trips_segment,\n", + " spec=spec,\n", + " nest_spec=nest_spec,\n", + " skims=skims,\n", + " locals_d=locals_dict,\n", + " chunk_size=chunk_size,\n", + " want_logsums=logsum_column_name is not None,\n", + " trace_label=segment_trace_label,\n", + " trace_choice_name='trip_mode_choice',\n", + " estimator=estimator,\n", + " trace_column_names=None,\n", + " choose_individual_max_utility=choose_individual_max_utility)\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " if not choose_individual_max_utility:\n", + " alts = spec.columns\n", + " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", + " ################\n", + " choices_list.append(choices)\n", + " choices_df_asim = pd.concat(choices_list)\n", + "\n", + " # update trips table with choices (and potionally logssums)\n", + " trips_df = trips.to_frame()\n", + "\n", + " if (do_these_purposes is not None):\n", + " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", + "\n", + " assign_in_place(trips_df, choices_df_asim)\n", + " assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + " finalise = True\n", + " if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + " print(\"Done\")\n", + "\n", + " return trips_df" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:28:53.643404Z", + "iopub.status.busy": "2021-09-01T02:28:53.643055Z", + "iopub.status.idle": "2021-09-01T02:29:01.223951Z", + "shell.execute_reply": "2021-09-01T02:29:01.222865Z", + "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" + }, + "tags": [], + "pycharm": { + "is_executing": true + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n", + "trip_mode_choice tour_type 'eatout' (33 trips)\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n", + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n" + ] + } + ], + "source": [ + "trips_df_ru = run_trip_mode_choice()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# OLD\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "id": "2f080150-c15a-4059-8a26-42a8c0072606", + "metadata": { + "execution": { + "iopub.execute_input": "2021-08-31T23:16:40.014024Z", + "iopub.status.busy": "2021-08-31T23:16:40.013770Z", + "iopub.status.idle": "2021-08-31T23:16:40.236326Z", + "shell.execute_reply": "2021-08-31T23:16:40.235553Z", + "shell.execute_reply.started": "2021-08-31T23:16:40.013973Z" + }, + "tags": [] + }, + "source": [ + "### make choice at each level" + ] + }, + { + "cell_type": "code", + "execution_count": 315, + "id": "2ed71b54-67e4-4087-957a-75f4fa184144", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:43.778081Z", + "iopub.status.busy": "2021-09-01T01:11:43.777757Z", + "iopub.status.idle": "2021-09-01T01:11:43.975332Z", + "shell.execute_reply": "2021-09-01T01:11:43.974588Z", + "shell.execute_reply.started": "2021-09-01T01:11:43.778055Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_alternatives(nests, name):\n", + " alts = list(filter(lambda x: x.name == name, nests))\n", + " assert len(alts) == 1, f\"{len(alts)} not one\"\n", + " alts = alts[0].alternatives\n", + " return alts\n", + "\n", + "def recursive_choice(row, columns, nest_levels, nests):\n", + " choices = row[columns].idxmax() #axis=1).values[0]\n", + " next_level_columns = get_alternatives(nests, choices)\n", + " #print(f\"{choices} leads to columns {next_level_columns}\")\n", + " if next_level_columns is None:\n", + " return choices \n", + " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", + " return new_choice\n", + "\n", + "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", + "\n", + "def make_choice(utils_df, nests, nest_levels, seed=None):\n", + " rng = default_rng(seed=seed)\n", + " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", + " probs_arr = utils_df - np.log(-np.log(rands))\n", + " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", + " return choices" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", + "seed = 9326543345\n", + "make_choice(utils_df, nests_, nest_levels, seed)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 316, + "id": "30424562-2627-446b-abd5-7c763c52060e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T01:11:49.444289Z", + "iopub.status.busy": "2021-09-01T01:11:49.444002Z", + "iopub.status.idle": "2021-09-01T01:11:49.664214Z", + "shell.execute_reply": "2021-09-01T01:11:49.663088Z", + "shell.execute_reply.started": "2021-09-01T01:11:49.444266Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n" + ] + } + ], + "source": [ + "# group nests by level:\n", + "depth = np.max([x.level for x in nests_])\n", + "print(depth)\n", + "nest_levels = {x: [] for x in range(1, depth+1)}\n", + "for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + "assert len(nest_levels[1]) == 1\n", + "assert nest_levels[1][0] == 'root'" + ] + }, + { + "cell_type": "code", + "execution_count": 453, + "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:55:59.542669Z", + "iopub.status.busy": "2021-09-01T04:55:59.542360Z", + "iopub.status.idle": "2021-09-01T04:55:59.771722Z", + "shell.execute_reply": "2021-09-01T04:55:59.770801Z", + "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + " nested_probabilities = \\\n", + " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", + " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) \n", + " return base_probabilities\n", + "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 451, + "id": "054fd9b1-72fc-49cc-a790-48ef75bcbaed", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:55:08.260140Z", + "iopub.status.busy": "2021-09-01T04:55:08.259914Z", + "iopub.status.idle": "2021-09-01T04:55:09.140095Z", + "shell.execute_reply": "2021-09-01T04:55:09.139119Z", + "shell.execute_reply.started": "2021-09-01T04:55:08.260116Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "trip_id\n", + "86627409 WALK_LRF\n", + "86627413 WALK_LRF\n", + "86673657 WALK_LOC\n", + "86673658 WALK\n", + "86673659 WALK\n", + " ... \n", + "2464446025 WALK\n", + "2464446029 WALK\n", + "2464449633 WALK\n", + "2464449634 WALK\n", + "2464449637 WALK\n", + "Length: 168, dtype: object" + ] + }, + "execution_count": 451, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def simple_simulate_rum(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", + "\n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices\n", + "\n", + "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" + ] + }, + { + "cell_type": "code", + "execution_count": 393, + "id": "f3c5c1ba-e9c3-4511-98bf-9ac98c987da7", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T02:39:52.539983Z", + "iopub.status.busy": "2021-09-01T02:39:52.539668Z", + "iopub.status.idle": "2021-09-01T02:39:52.796170Z", + "shell.execute_reply": "2021-09-01T02:39:52.795498Z", + "shell.execute_reply.started": "2021-09-01T02:39:52.539944Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " segment_trace_label, log_alt_losers=log_alt_losers, \n", + " trace_column_names=trace_column_names, seed=seed)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " choices_list.append(choices)\n", + "\n", + " choices_df = pd.concat(choices_list)\n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": 454, + "id": "74eeedd3-4ade-4729-8170-79fa9dcf11f8", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:58:52.825042Z", + "iopub.status.busy": "2021-09-01T04:58:52.824768Z", + "iopub.status.idle": "2021-09-01T04:58:53.042884Z", + "shell.execute_reply": "2021-09-01T04:58:53.042061Z", + "shell.execute_reply.started": "2021-09-01T04:58:52.825004Z" + } + }, + "outputs": [], + "source": [ + "## caculate probabilities with Asim methodology, should be correct\n", + "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", + " full_probs = [] # analytical probs\n", + "\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " #choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " # segment_trace_label, log_alt_losers=log_alt_losers, \n", + " # trace_column_names=trace_column_names, seed=seed)\n", + " probs = simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, \n", + " estimator, segment_trace_label, log_alt_losers, \n", + " trace_column_names)\n", + " full_probs.append(probs)\n", + " probs_df = pd.concat(full_probs)\n", + " return probs_df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 455, + "id": "295a4fd1-8e85-47e2-af82-9f1beac31d63", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T04:59:46.126555Z", + "iopub.status.busy": "2021-09-01T04:59:46.126262Z", + "iopub.status.idle": "2021-09-01T04:59:52.979908Z", + "shell.execute_reply": "2021-09-01T04:59:52.979043Z", + "shell.execute_reply.started": "2021-09-01T04:59:46.126522Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2021-09-01 14:59:46.353007 Start\n", + "2021-09-01 14:59:52.976736 End\n" + ] + } + ], + "source": [ + "print(f\"{datetime.now()} Start\")\n", + "pipeline.open_pipeline(resume_after)\n", + "inject.get_injectable('preload_injectables', None)\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "trips_df = trips.to_frame()\n", + "#print(\"Running with %d trips\", trips_df.shape[0])\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "\n", + "all_choices = []\n", + "for i in range(100):\n", + " if i % 10 == 0:\n", + " print(f\"{datetime.now()} iteration {i}\")\n", + " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=None)\n", + " all_choices.append(choices_df)\n", + "all_choices = pd.concat(all_choices, axis=1)\n", + "\n", + "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", + " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", + " log_alt_losers=None, trace_column_names=None)\n", + "\n", + "# update trips table with choices (and potionally logssums)\n", + "#trips_df = trips.to_frame()\n", + "#\n", + "#assign_in_place(trips_df, choices_df)\n", + "#assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(f\"{datetime.now()} End\")" + ] + }, + { + "cell_type": "code", + "execution_count": 463, + "id": "dc87e904-16d1-44d4-81b7-cafbe4e223c3", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:01:40.901982Z", + "iopub.status.busy": "2021-09-01T05:01:40.901759Z", + "iopub.status.idle": "2021-09-01T05:01:41.337758Z", + "shell.execute_reply": "2021-09-01T05:01:41.336757Z", + "shell.execute_reply.started": "2021-09-01T05:01:40.901946Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#all_choices.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", + "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts = val_counts / all_choices.shape[1]\n", + "#val_counts = val_counts.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", + "#val_counts['prob_of_asim_choice'] = val_counts.apply(lambda x: x[x.asim], axis=1) # this is what our simulation says w.r.t. to asim choice\n", + "# for 100% and many samples should mostly agree" + ] + }, + { + "cell_type": "code", + "execution_count": 464, + "id": "09bf151c-a39a-4823-acfd-9c8ae747c338", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:01:46.119117Z", + "iopub.status.busy": "2021-09-01T05:01:46.118849Z", + "iopub.status.idle": "2021-09-01T05:01:46.354114Z", + "shell.execute_reply": "2021-09-01T05:01:46.353248Z", + "shell.execute_reply.started": "2021-09-01T05:01:46.119092Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "#val_counts['prob_of_asim_choice'].hist(bins=100);" + ] + }, + { + "cell_type": "code", + "execution_count": 490, + "id": "f3b6ade9-d875-4104-b438-b53ef42e342c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T05:10:08.653227Z", + "iopub.status.busy": "2021-09-01T05:10:08.652978Z", + "iopub.status.idle": "2021-09-01T05:10:08.915620Z", + "shell.execute_reply": "2021-09-01T05:10:08.914513Z", + "shell.execute_reply.started": "2021-09-01T05:10:08.653189Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", + "analytical 3.152 0.852 0.0 0.0 0.0 \n", + "experiment 3.243 0.880 0.0 0.0 0.0 \n", + "\n", + " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", + "analytical 0.0 0.0 0.0 0.685 0.0 \n", + "experiment 0.0 0.0 0.0 0.772 0.0 \n", + "\n", + " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK \\\n", + "analytical 0.129 0.0 0.182 0.260 1.334 63.708 \n", + "experiment 0.156 0.0 0.174 0.259 1.434 63.243 \n", + "\n", + " WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", + "analytical 0.0 0.0 0.455 18.355 10.887 \n", + "experiment 0.0 0.0 0.394 18.639 10.807 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mode_share_analytical = (probs_nl.sum(axis=0) / probs_nl.shape[0]).to_frame('analytical')\n", + "assert np.allclose(mode_share_analytical.sum(), 1)\n", + "mode_share_rum = (val_counts.sum(axis=0) / val_counts.shape[0]).to_frame('experiment')\n", + "assert np.allclose(mode_share_rum.sum(), 1)\n", + "full_share = mode_share_analytical.join(mode_share_rum, how='outer').fillna(0)\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * full_share).T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e48a56e-d613-4dfd-bef6-62bee524f12f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50914e1d-750d-440c-9617-f96bd2a46c56", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2d70e0e-520e-49a6-8024-ae31345f6ead", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "f9e8ef02-1d57-4b21-8aef-25a6e1095c02", + "metadata": {}, + "source": [ + "### try zenith normalisation of simple_simulate_rum" + ] + }, + { + "cell_type": "code", + "execution_count": 629, + "id": "db235e90-d2e5-4e90-b41c-6d90a670e41d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:24.045881Z", + "iopub.status.busy": "2021-09-01T11:37:24.045637Z", + "iopub.status.idle": "2021-09-01T11:37:24.276488Z", + "shell.execute_reply": "2021-09-01T11:37:24.274898Z", + "shell.execute_reply.started": "2021-09-01T11:37:24.045844Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation belowi s equivalent.\n", + "def compute_nested_utilities_zenith(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) # / nest.product_of_coefficients #coefficient\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", + " \n", + " # now go over all leaves and correct for scale\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] /= nest.coefficient\n", + " \n", + " return nested_utilities\n", + "\n", + "\n", + "def simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " utils_df = compute_nested_utilities_zenith(raw_utilities, nest_spec)\n", + "\n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices\n", + "\n", + "#simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" + ] + }, + { + "cell_type": "code", + "execution_count": 630, + "id": "f69bc3dc-84e4-4dd5-9a57-c053cadba4b5", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:24.828926Z", + "iopub.status.busy": "2021-09-01T11:37:24.828707Z", + "iopub.status.idle": "2021-09-01T11:37:25.046409Z", + "shell.execute_reply": "2021-09-01T11:37:25.045399Z", + "shell.execute_reply.started": "2021-09-01T11:37:24.828902Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", + " choices_list = []\n", + " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " # (primary_purpose, len(trips_segment.index), ))\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", + " choices = simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, \n", + " segment_trace_label, log_alt_losers=log_alt_losers, \n", + " trace_column_names=trace_column_names, seed=seed)\n", + "\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " choices_list.append(choices)\n", + "\n", + " choices_df = pd.concat(choices_list)\n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": 631, + "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:37:32.250808Z", + "iopub.status.busy": "2021-09-01T11:37:32.250548Z", + "iopub.status.idle": "2021-09-01T20:20:35.257463Z", + "shell.execute_reply": "2021-09-01T20:20:35.255964Z", + "shell.execute_reply.started": "2021-09-01T11:37:32.250782Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "num_samples = 10\n", + "\n", + "rng_test = default_rng(23423)\n", + "\n", + "print(f\"{datetime.now()} Start\")\n", + "pipeline.open_pipeline(resume_after)\n", + "inject.get_injectable('preload_injectables', None)\n", + "model_name = \"trip_mode_choice\"\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "\n", + "step_name = model_name\n", + "args = {}\n", + "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "chunk_size = 0\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "trips_df = trips.to_frame()\n", + "#print(\"Running with %d trips\", trips_df.shape[0])\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + "orig_col = 'origin'\n", + "dest_col = 'destination'\n", + "\n", + "constants = {}\n", + "constants.update(config.get_model_constants(model_settings))\n", + "constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + "})\n", + "\n", + "skim_dict = network_los.get_default_skim_dict()\n", + "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + "}\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + "all_choices_zenith = []\n", + "all_choices = []\n", + "for i in range(num_samples):\n", + " \n", + " seed = rng_test.integers(0, 100000) #int(9.3 * (i+1)**3) # why not\n", + " \n", + " if i % 50 == 0:\n", + " print(f\"{datetime.now()} iteration {i}\")\n", + " choices_df_zenith = stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", + " all_choices_zenith.append(choices_df_zenith)\n", + "\n", + " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", + " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", + " all_choices.append(choices_df)\n", + " \n", + " t_ = choices_df_zenith.merge(choices_df, left_index=True, right_index=True, suffixes=['_zenith', '_asim'])\n", + " diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", + " \n", + " #print(f\"seed {seed} leads to {diffs.shape[0]} differences. tripids {diffs.index}\")\n", + "\n", + "all_choices_zenith = pd.concat(all_choices_zenith, axis=1)\n", + "all_choices = pd.concat(all_choices, axis=1)\n", + "\n", + "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", + " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", + " log_alt_losers=None, trace_column_names=None)\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + "print(f\"{datetime.now()} End\")" + ] + }, + { + "cell_type": "code", + "execution_count": 643, + "id": "3abd7949-dc9c-469b-845e-26c17741f70e", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:33:21.533110Z", + "iopub.status.busy": "2021-09-01T20:33:21.531915Z", + "iopub.status.idle": "2021-09-01T20:33:21.740946Z", + "shell.execute_reply": "2021-09-01T20:33:21.739713Z", + "shell.execute_reply.started": "2021-09-01T20:33:21.533074Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Have 2038 samples\n" + ] + } + ], + "source": [ + "print(f\"Have {all_choices_zenith.shape[1]} samples\")" + ] + }, + { + "cell_type": "code", + "execution_count": 633, + "id": "8762bd6d-ac04-458c-869f-be86d8297351", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:21:27.778803Z", + "iopub.status.busy": "2021-09-01T20:21:27.778531Z", + "iopub.status.idle": "2021-09-01T20:21:28.675645Z", + "shell.execute_reply": "2021-09-01T20:21:28.674734Z", + "shell.execute_reply.started": "2021-09-01T20:21:27.778754Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "val_counts_zenith = all_choices_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts_zenith = val_counts_zenith / all_choices_zenith.shape[1]\n", + "\n", + "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "val_counts = val_counts / all_choices.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 634, + "id": "0f566105-9424-457b-86f5-d60c1d63aac5", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:21:30.954894Z", + "iopub.status.busy": "2021-09-01T20:21:30.954654Z", + "iopub.status.idle": "2021-09-01T20:21:31.179689Z", + "shell.execute_reply": "2021-09-01T20:21:31.178304Z", + "shell.execute_reply.started": "2021-09-01T20:21:30.954867Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
experiment_zenith3.1500.8740.00.00.00.00.00.00.6800.00.1360.00.1540.2361.28563.7670.00.00.51518.82410.379
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", + "analytical 3.152 0.852 0.0 0.0 0.0 \n", + "experiment 3.243 0.880 0.0 0.0 0.0 \n", + "experiment_zenith 3.150 0.874 0.0 0.0 0.0 \n", + "\n", + " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", + "analytical 0.0 0.0 0.0 0.685 0.0 \n", + "experiment 0.0 0.0 0.0 0.772 0.0 \n", + "experiment_zenith 0.0 0.0 0.0 0.680 0.0 \n", + "\n", + " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE \\\n", + "analytical 0.129 0.0 0.182 0.260 1.334 \n", + "experiment 0.156 0.0 0.174 0.259 1.434 \n", + "experiment_zenith 0.136 0.0 0.154 0.236 1.285 \n", + "\n", + " WALK WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", + "analytical 63.708 0.0 0.0 0.455 18.355 10.887 \n", + "experiment 63.243 0.0 0.0 0.394 18.639 10.807 \n", + "experiment_zenith 63.767 0.0 0.0 0.515 18.824 10.379 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mode_share_rum_zenith = (val_counts_zenith.sum(axis=0) / val_counts_zenith.shape[0]).to_frame('experiment_zenith')\n", + "assert np.allclose(mode_share_rum_zenith.sum(), 1)\n", + "full_share_incl_zenith = full_share.merge(mode_share_rum_zenith, left_index=True, right_index=True, how='outer').fillna(0)\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * full_share_incl_zenith).T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c0d3081-b45b-4573-9246-e4a9d39591dd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "612b2f04-45e2-4eb2-b52a-9f753427877f", + "metadata": {}, + "source": [ + "## investigate diverging seed and look at diff in formulation" + ] + }, + { + "cell_type": "code", + "execution_count": 619, + "id": "ac09dc75-a92f-43fb-a0cd-fe7de1f8b66d", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:12:02.988141Z", + "iopub.status.busy": "2021-09-01T11:12:02.987900Z", + "iopub.status.idle": "2021-09-01T11:12:03.212179Z", + "shell.execute_reply": "2021-09-01T11:12:03.210993Z", + "shell.execute_reply.started": "2021-09-01T11:12:02.988102Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# TODO: work out if our formulation and formulation based on asim probabilities is equivalent\n", + "def compute_nested_utilities_asim(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) / nest.product_of_coefficients\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", + " return nested_utilities\n", + "\n", + "def compute_nested_utilities_zenith_check(raw_utilities, nest_spec):\n", + " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] = \\\n", + " raw_utilities[name].astype(float) # scale correction is below\n", + " else:\n", + " with np.errstate(divide='ignore'):\n", + " nested_utilities[name] = \\\n", + " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", + " \n", + " # now go over all leaves and correct for scale\n", + " for nest in logit.each_nest(nest_spec):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " nested_utilities[name] /= nest.coefficient\n", + " \n", + " return nested_utilities" + ] + }, + { + "cell_type": "code", + "execution_count": 688, + "id": "4b7c6368-21ca-4442-969e-f2f8ff868842", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T20:49:56.958862Z", + "iopub.status.busy": "2021-09-01T20:49:56.958623Z", + "iopub.status.idle": "2021-09-01T20:49:57.191884Z", + "shell.execute_reply": "2021-09-01T20:49:57.190975Z", + "shell.execute_reply.started": "2021-09-01T20:49:56.958827Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None, use_zenith=True, raw_utilities=None):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " \n", + " if raw_utilities is None:\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + "\n", + " if use_zenith:\n", + " utils_df = compute_nested_utilities_zenith_check(raw_utilities, nest_spec)\n", + " else:\n", + " utils_df = compute_nested_utilities_asim(raw_utilities, nest_spec)\n", + "\n", + " # test\n", + " #return compute_nested_utilities_zenith_check(raw_utilities, nest_spec), compute_nested_utilities_asim(raw_utilities, nest_spec)\n", + " \n", + " nests_ = list(logit.each_nest(nest_spec))\n", + " # group nests by level:\n", + " depth = np.max([x.level for x in nests_])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in nests_:\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1\n", + " assert nest_levels[1][0] == 'root'\n", + " # make choices\n", + " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", + "\n", + " return choices" + ] + }, + { + "cell_type": "code", + "execution_count": 626, + "id": "bd6cadc3-79f5-4824-b55e-639c39d5ab18", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:13:12.077886Z", + "iopub.status.busy": "2021-09-01T11:13:12.077652Z", + "iopub.status.idle": "2021-09-01T11:13:12.290222Z", + "shell.execute_reply": "2021-09-01T11:13:12.288861Z", + "shell.execute_reply.started": "2021-09-01T11:13:12.077863Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# t_z, t_a = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed)\n", + "# tr_id = 86673661\n", + "# display(t_z.loc[t_z.index==tr_id])\n", + "# display(t_a.loc[t_a.index==tr_id])" + ] + }, + { + "cell_type": "code", + "execution_count": 625, + "id": "42e6da62-58e8-4b16-9169-9faa1fc8a162", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T11:13:06.847583Z", + "iopub.status.busy": "2021-09-01T11:13:06.847204Z", + "iopub.status.idle": "2021-09-01T11:13:08.401700Z", + "shell.execute_reply": "2021-09-01T11:13:08.400711Z", + "shell.execute_reply.started": "2021-09-01T11:13:06.847554Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seed 51006 leads to 34 differences out of 168. tripids Int64Index([ 86673657, 86673661, 106741681, 106741682, 106741685,\n", + " 106741686, 106741687, 106741688, 211327433, 211327437,\n", + " 444793574, 484173905, 484173909, 535170694, 535620053,\n", + " 708171014, 943749470, 943749471, 1060575853, 1091770617,\n", + " 1146472489, 1146472493, 1276281769, 1276281773, 1658748793,\n", + " 1658748797, 1767013726, 1767186577, 1767186578, 1768237161,\n", + " 1768237165, 1768237166, 2463663417, 2463663421],\n", + " dtype='int64', name='trip_id')\n" + ] + } + ], + "source": [ + "seed = 51006\n", + "x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True)\n", + "y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False)\n", + "t_ = x_.to_frame('trip_mode_zenith').merge(y_.to_frame('trip_mode_asim'), left_index=True, right_index=True)\n", + "diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", + "print(f\"seed {seed} leads to {diffs.shape[0]} differences out of {t_.shape[0]}. tripids {diffs.index}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f3c0506-a79d-4a31-9d56-71b9966d0a92", + "metadata": {}, + "outputs": [], + "source": [ + "seed 51006 leads to 3 differences. tripids Int64Index([86673661, 535170689, 1060575849], dtype='int64', name='trip_id')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a5f332a-b1d7-4a80-bb21-d37a1611cf9f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24c5e4a1-6142-49d4-8da8-a1e266b3cb9b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 753, + "id": "2098907f-28af-4f53-af74-6aaee8a6c53c", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T23:04:23.861048Z", + "iopub.status.busy": "2021-09-01T23:04:23.860748Z", + "iopub.status.idle": "2021-09-01T23:04:24.069307Z", + "shell.execute_reply": "2021-09-01T23:04:24.068434Z", + "shell.execute_reply.started": "2021-09-01T23:04:23.861024Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "rng_ = default_rng(seed=100)\n", + "new_utils = pd.DataFrame(0.1 * -np.log(-np.log(rng_.uniform(0,1,raw_utilities.shape))), columns=raw_utilities.columns)\n", + "new_utils.index = raw_utilities.index" + ] + }, + { + "cell_type": "code", + "execution_count": 767, + "id": "da48a178-82b2-44a5-b45a-e9de3bb5e5c2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-01T23:17:20.369675Z", + "iopub.status.busy": "2021-09-01T23:17:20.369424Z", + "iopub.status.idle": "2021-09-01T23:17:20.666506Z", + "shell.execute_reply": "2021-09-01T23:17:20.665626Z", + "shell.execute_reply.started": "2021-09-01T23:17:20.369637Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
1372487210.0566420.0373050.0427800.0295760.0861540.0265860.0883130.1003310.0280210.0185160.0643910.0200090.0263990.0226220.0230070.0217540.0871260.0211710.0364750.0276500.135171
1372487250.1019020.0228760.0312270.0699900.0569130.0466850.0884110.0877850.0254140.0381620.0239880.0288000.0393180.0201600.0483640.0230030.0339590.0345970.0377080.0995230.041213
2113882010.0375620.0700330.0559510.0403890.0200410.0959740.0946780.0943680.0360140.0250510.0348600.0349530.0354990.0280400.0436500.0247130.0313880.0259310.0669990.0526780.051228
2113882050.0354800.0562560.0439440.0480850.0420780.0622210.1068190.0872650.0351170.0303990.0351540.0264690.0506460.0357270.0242840.0329050.0394270.0410920.0547140.0484590.063458
8063884010.0549180.0404880.0331490.0542940.0434770.0679820.1524650.0741520.0334040.0287350.0592690.0243390.0369680.0277600.0235220.0609320.0234450.0239310.0446430.0337730.058353
8063884050.0327640.0573010.0184370.0770990.0469630.0635210.0949510.0890720.0308260.0378960.0357520.0198780.0495360.0796100.0269220.0248330.0248400.0266950.0799360.0401460.043022
\n", + "
" + ], + "text/plain": [ + " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n", + "trip_id \n", + "137248721 0.056642 0.037305 0.042780 0.029576 \n", + "137248725 0.101902 0.022876 0.031227 0.069990 \n", + "211388201 0.037562 0.070033 0.055951 0.040389 \n", + "211388205 0.035480 0.056256 0.043944 0.048085 \n", + "806388401 0.054918 0.040488 0.033149 0.054294 \n", + "806388405 0.032764 0.057301 0.018437 0.077099 \n", + "\n", + " SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\n", + "trip_id \n", + "137248721 0.086154 0.026586 0.088313 0.100331 0.028021 0.018516 \n", + "137248725 0.056913 0.046685 0.088411 0.087785 0.025414 0.038162 \n", + "211388201 0.020041 0.095974 0.094678 0.094368 0.036014 0.025051 \n", + "211388205 0.042078 0.062221 0.106819 0.087265 0.035117 0.030399 \n", + "806388401 0.043477 0.067982 0.152465 0.074152 0.033404 0.028735 \n", + "806388405 0.046963 0.063521 0.094951 0.089072 0.030826 0.037896 \n", + "\n", + " WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\n", + "trip_id \n", + "137248721 0.064391 0.020009 0.026399 0.022622 0.023007 0.021754 \n", + "137248725 0.023988 0.028800 0.039318 0.020160 0.048364 0.023003 \n", + "211388201 0.034860 0.034953 0.035499 0.028040 0.043650 0.024713 \n", + "211388205 0.035154 0.026469 0.050646 0.035727 0.024284 0.032905 \n", + "806388401 0.059269 0.024339 0.036968 0.027760 0.023522 0.060932 \n", + "806388405 0.035752 0.019878 0.049536 0.079610 0.026922 0.024833 \n", + "\n", + " DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n", + "trip_id \n", + "137248721 0.087126 0.021171 0.036475 0.027650 0.135171 \n", + "137248725 0.033959 0.034597 0.037708 0.099523 0.041213 \n", + "211388201 0.031388 0.025931 0.066999 0.052678 0.051228 \n", + "211388205 0.039427 0.041092 0.054714 0.048459 0.063458 \n", + "806388401 0.023445 0.023931 0.044643 0.033773 0.058353 \n", + "806388405 0.024840 0.026695 0.079936 0.040146 0.043022 " + ] + }, + "execution_count": 767, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nested_exp_utilities_cf = simulate.compute_nested_exp_utilities(new_utils, nest_spec)\n", + "nested_probabilities_cf = simulate.compute_nested_probabilities(nested_exp_utilities_cf, nest_spec, trace_label=None)\n", + "base_probabilities_cf = simulate.compute_base_probabilities(nested_probabilities_cf, nest_spec, spec)\n", + "base_probabilities_cf" + ] + }, + { + "cell_type": "code", + "execution_count": 799, + "id": "50dfd1ae-10c3-475c-94be-de783c2fa5c2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:15:29.056902Z", + "iopub.status.busy": "2021-09-02T00:15:29.056683Z", + "iopub.status.idle": "2021-09-02T00:27:21.965206Z", + "shell.execute_reply": "2021-09-02T00:27:21.964340Z", + "shell.execute_reply.started": "2021-09-02T00:15:29.056877Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 11min 49s, sys: 766 ms, total: 11min 50s\n", + "Wall time: 11min 52s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "num_samples = 10000 # 7.5s per 100\n", + "\n", + "data_zenith = []\n", + "data_asim = []\n", + "for i in range(num_samples):\n", + " seed = rng_.integers(0, 100000)\n", + " x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True, raw_utilities=new_utils)\n", + " y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False, raw_utilities=new_utils)\n", + " data_zenith.append(x_)\n", + " data_asim.append(y_)\n", + " \n", + "data_asim = pd.concat(data_asim, axis=1)\n", + "data_zenith = pd.concat(data_zenith, axis=1)\n", + "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", + "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_asim = counts_asim / data_asim.shape[1]\n", + "\n", + "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", + "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", + "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", + "# assert np.allclose(mode_share_zenith.sum(), 1)\n", + "# assert np.allclose(mode_share_asim.sum(), 1)\n", + "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", + "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 796, + "id": "bb16f9d7-44f5-4316-b981-d65216a5e217", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:14:11.772592Z", + "iopub.status.busy": "2021-09-02T00:14:11.772347Z", + "iopub.status.idle": "2021-09-02T00:14:12.001077Z", + "shell.execute_reply": "2021-09-02T00:14:12.000166Z", + "shell.execute_reply.started": "2021-09-02T00:14:11.772556Z" + } + }, + "outputs": [], + "source": [ + "#temp_z = data_zenith.copy()\n", + "#temp_a = data_asim.copy()\n", + "#data_asim = data_asim.join(temp_a, lsuffix=\"_o\", rsuffix=\"_n\")\n", + "#data_zenith = data_zenith.join(temp_z, lsuffix=\"_o\", rsuffix=\"_n\")\n", + "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", + "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", + "# counts_asim = counts_asim / data_asim.shape[1]\n", + "\n", + "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", + "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", + "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", + "# assert np.allclose(mode_share_zenith.sum(), 1)\n", + "# assert np.allclose(mode_share_asim.sum(), 1)\n", + "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", + "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 805, + "id": "031d8340-0af0-4608-8a83-ed2af4aaa3b2", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:41:01.948669Z", + "iopub.status.busy": "2021-09-02T00:41:01.948422Z", + "iopub.status.idle": "2021-09-02T00:41:02.175972Z", + "shell.execute_reply": "2021-09-02T00:41:02.174759Z", + "shell.execute_reply.started": "2021-09-02T00:41:01.948633Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
zenith8.0555.1674.6313.5023.6064.2663.9633.4983.8955.1764.7995.8514.9996.1214.6689.4864.2284.1252.9773.6713.317
asim6.0966.6984.9773.2023.5224.3344.1293.1843.2245.1496.2838.2964.7936.5844.5327.4844.1524.2732.6293.4333.023
probs8.8835.3214.7382.8903.1364.0033.5653.1623.7585.3244.9276.0495.3416.5415.03710.4273.9734.2242.5743.1472.979
\n", + "
" + ], + "text/plain": [ + " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP DRIVE_HVY \\\n", + "zenith 8.055 5.167 4.631 3.502 3.606 4.266 \n", + "asim 6.096 6.698 4.977 3.202 3.522 4.334 \n", + "probs 8.883 5.321 4.738 2.890 3.136 4.003 \n", + "\n", + " DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY SHARED3FREE \\\n", + "zenith 3.963 3.498 3.895 5.176 4.799 \n", + "asim 4.129 3.184 3.224 5.149 6.283 \n", + "probs 3.565 3.162 3.758 5.324 4.927 \n", + "\n", + " SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK WALK_COM WALK_EXP \\\n", + "zenith 5.851 4.999 6.121 4.668 9.486 4.228 4.125 \n", + "asim 8.296 4.793 6.584 4.532 7.484 4.152 4.273 \n", + "probs 6.049 5.341 6.541 5.037 10.427 3.973 4.224 \n", + "\n", + " WALK_HVY WALK_LOC WALK_LRF \n", + "zenith 2.977 3.671 3.317 \n", + "asim 2.629 3.433 3.023 \n", + "probs 2.574 3.147 2.979 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# sample of 20000\n", + "with pd.option_context(\"precision\", 3):\n", + " display((100.0 * mode_share_comp).T)" + ] + }, + { + "cell_type": "code", + "execution_count": 806, + "id": "aece448f-4055-4d2a-b90d-8530caec9cb9", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-02T00:43:32.971972Z", + "iopub.status.busy": "2021-09-02T00:43:32.971684Z", + "iopub.status.idle": "2021-09-02T00:43:33.223345Z", + "shell.execute_reply": "2021-09-02T00:43:33.222488Z", + "shell.execute_reply.started": "2021-09-02T00:43:32.971932Z" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
trip_id137248721_z137248721_a137248721
BIKE0.092700.054350.100331
DRIVEALONEFREE0.053650.050050.056642
DRIVEALONEPAY0.038800.031550.037305
DRIVE_COM0.028700.024700.021171
DRIVE_EXP0.028800.026050.021754
DRIVE_HVY0.078150.099700.087126
DRIVE_LOC0.028750.027000.022622
DRIVE_LRF0.027800.024450.023007
SHARED2FREE0.043250.023000.042780
SHARED2PAY0.033600.015850.029576
SHARED3FREE0.073650.119200.086154
SHARED3PAY0.032600.039600.026586
TAXI0.035100.050300.036475
TNC_SHARED0.126200.181150.135171
TNC_SINGLE0.026250.038450.027650
WALK0.079400.048250.088313
WALK_COM0.031850.024600.026399
WALK_EXP0.057050.058950.064391
WALK_HVY0.025200.018400.020009
WALK_LOC0.034550.027400.028021
WALK_LRF0.023950.017000.018516
\n", + "
" + ], + "text/plain": [ + "trip_id 137248721_z 137248721_a 137248721\n", + "BIKE 0.09270 0.05435 0.100331\n", + "DRIVEALONEFREE 0.05365 0.05005 0.056642\n", + "DRIVEALONEPAY 0.03880 0.03155 0.037305\n", + "DRIVE_COM 0.02870 0.02470 0.021171\n", + "DRIVE_EXP 0.02880 0.02605 0.021754\n", + "DRIVE_HVY 0.07815 0.09970 0.087126\n", + "DRIVE_LOC 0.02875 0.02700 0.022622\n", + "DRIVE_LRF 0.02780 0.02445 0.023007\n", + "SHARED2FREE 0.04325 0.02300 0.042780\n", + "SHARED2PAY 0.03360 0.01585 0.029576\n", + "SHARED3FREE 0.07365 0.11920 0.086154\n", + "SHARED3PAY 0.03260 0.03960 0.026586\n", + "TAXI 0.03510 0.05030 0.036475\n", + "TNC_SHARED 0.12620 0.18115 0.135171\n", + "TNC_SINGLE 0.02625 0.03845 0.027650\n", + "WALK 0.07940 0.04825 0.088313\n", + "WALK_COM 0.03185 0.02460 0.026399\n", + "WALK_EXP 0.05705 0.05895 0.064391\n", + "WALK_HVY 0.02520 0.01840 0.020009\n", + "WALK_LOC 0.03455 0.02740 0.028021\n", + "WALK_LRF 0.02395 0.01700 0.018516" + ] + }, + "execution_count": 806, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# TODO: look at probs per trip, i.e. do not sum across trips\n", + "trip_id = 137248721\n", + "\n", + "counts_zenith.loc[counts_zenith.index == trip_id].T.merge(\n", + " counts_asim.loc[counts_asim.index == trip_id].T, suffixes=['_z', '_a'], left_index=True, right_index=True, how='outer').merge(\n", + " base_probabilities_cf.loc[base_probabilities_cf.index == trip_id].T, suffixes=['', '_probs'], left_index=True, right_index=True, how='outer').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ef84f10-d616-40e4-87e5-425499027ff3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "431cee49-40a9-4dc7-93a5-75308fb302dd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fcdc0ff-c510-4878-8672-8365e046442c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 83a86f5575f79922b9fafea6715fc0d342277846 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 4 May 2022 23:01:29 +1000 Subject: [PATCH 006/135] scale of leaf utility --- activitysim/core/logit.py | 5 +- activitysim/core/simulate.py | 9 +- notebooks/validate_frozen_impl.ipynb | 139 ++++++++++++++++++++++++--- 3 files changed, 133 insertions(+), 20 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index e3f87d4248..a2b732c52a 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -473,7 +473,10 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients nest.ancestors = parent_nest.ancestors + [name] nest.coefficient = parent_nest.coefficient - nest.parent_nest_scale = parent_nest.coefficient + + # this is used for frozen individual ru - see discussion in simulate.py l.740 + nest.parent_nest_scale = parent_nest.product_of_coefficients + #nest.parent_nest_scale = parent_nest.coefficient yield spec, nest diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 4f9b7d9a2a..27ed38e5fd 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -736,15 +736,18 @@ def compute_nested_utilities(raw_utilities, nest_spec): for nest in logit.each_nest(nest_spec, post_order=True): name = nest.name if nest.is_leaf: - nested_utilities[name] = \ - raw_utilities[name].astype(float) / nest.coefficient #nest.product_of_coefficients + nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients + #nested_utilities[name] = raw_utilities[name].astype(float) / nest.coefficient + # WHY does asim use product of coeffs here? the above gives us identical root logsums + # but then we need to dscale ev1 accordingly for alternatives, right? + # can be achieved by else: # the alternative nested_utilities will already have been computed due to post_order # this will RuntimeWarning: divide by zero encountered in log # if all nest alternative utilities are zero and produce -inf with np.errstate(divide='ignore'): nested_utilities[name] = \ - nest.coefficient * np.log((np.exp(nested_utilities[nest.alternatives])).sum(axis=1)) + nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) return nested_utilities diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 1fd45d784c..daeb7dab35 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -172,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 54, "outputs": [], "source": [ "def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True):\n", @@ -339,7 +339,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -349,10 +349,7 @@ "shell.execute_reply": "2021-09-01T02:29:01.222865Z", "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" }, - "tags": [], - "pycharm": { - "is_executing": true - } + "tags": [] }, "outputs": [ { @@ -371,19 +368,129 @@ "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n" + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", + "trip_mode_choice tour_type 'othmaint' (46 trips)\n", + "trip_mode_choice tour_type 'school' (37 trips)\n", + "trip_mode_choice tour_type 'shopping' (77 trips)\n", + "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'univ' (26 trips)\n", + "trip_mode_choice tour_type 'work' (168 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n", + "trip_mode_choice tour_type 'eatout' (33 trips)\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n", + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", + "trip_mode_choice tour_type 'othmaint' (46 trips)\n", + "trip_mode_choice tour_type 'school' (37 trips)\n", + "trip_mode_choice tour_type 'shopping' (77 trips)\n", + "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'univ' (26 trips)\n", + "trip_mode_choice tour_type 'work' (168 trips)\n", + "Done\n" ] } ], "source": [ - "trips_df_ru = run_trip_mode_choice()" + "trips_df_ru = run_trip_mode_choice()\n", + "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" ] }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 56, + "outputs": [ + { + "data": { + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 299 304\nWALK_LOC 92 85\nWALK_LRF 61 53\nBIKE 17 17\nDRIVEALONEFREE 6 4\nSHARED2FREE 3 4\nTNC_SINGLE 2 4\nWALK_HVY 2 4", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK299304
WALK_LOC9285
WALK_LRF6153
BIKE1717
DRIVEALONEFREE64
SHARED2FREE34
TNC_SINGLE24
WALK_HVY24
\n
" + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c_ = [\"trip_mode\", \"mode_choice_logsum\"]\n", + "trips = trips_df_ru[c_].merge(trips_df[c_], left_index=True, right_index=True, how=\"outer\", suffixes=[\"_fru\", \"_asim\"])\n", + "pd.merge(trips_df_ru.trip_mode.value_counts(), trips_df.trip_mode.value_counts(), left_index=True, right_index=True,\n", + " suffixes=[\"_fru\", \"_asim\"])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 57, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logsums agree\n" + ] + } + ], + "source": [ + "if np.allclose(trips.mode_choice_logsum_fru, trips.mode_choice_logsum_asim):\n", + " print(\"Logsums agree\")\n", + "else:\n", + " ((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru).hist(bins=50)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 58, + "outputs": [ + { + "data": { + "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n8684833 WALK -1.171760 WALK \n8684837 WALK -1.238719 WALK \n8685009 BIKE 6.198626 BIKE \n8685013 WALK 6.175681 BIKE \n8753057 WALK 4.457539 WALK \n... ... ... ... \n2472945113 WALK_LOC 12.537675 WALK_LOC \n2472945117 WALK 11.880804 WALK_LOC \n2472945118 WALK_LOC 13.710030 WALK \n2473024473 WALK_LOC -0.530696 WALK_LOC \n2473024477 WALK_LRF 0.624304 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n8684833 -1.171760 \n8684837 -1.238719 \n8685009 6.198626 \n8685013 6.175681 \n8753057 4.457539 \n... ... \n2472945113 12.537675 \n2472945117 11.880804 \n2472945118 13.710030 \n2473024473 -0.530696 \n2473024477 0.624304 \n\n[482 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
8684833WALK-1.171760WALK-1.171760
8684837WALK-1.238719WALK-1.238719
8685009BIKE6.198626BIKE6.198626
8685013WALK6.175681BIKE6.175681
8753057WALK4.457539WALK4.457539
...............
2472945113WALK_LOC12.537675WALK_LOC12.537675
2472945117WALK11.880804WALK_LOC11.880804
2472945118WALK_LOC13.710030WALK13.710030
2473024473WALK_LOC-0.530696WALK_LOC-0.530696
2473024477WALK_LRF0.624304WALK_LRF0.624304
\n

482 rows × 4 columns

\n
" + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips" + ], "metadata": { "collapsed": false, "pycharm": { @@ -452,14 +559,14 @@ } }, { - "cell_type": "markdown", - "source": [ - "# OLD\n" - ], + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { - "name": "#%% md\n" + "name": "#%%\n" } } }, From 5a7ad3c76604846057a897ff78c3b7983738776d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 4 May 2022 23:41:58 +1000 Subject: [PATCH 007/135] pull out cruft --- notebooks/fru_utils.py | 416 +++++++++++++++++++++ notebooks/validate_frozen_impl.ipynb | 524 +++++---------------------- 2 files changed, 509 insertions(+), 431 deletions(-) create mode 100644 notebooks/fru_utils.py diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py new file mode 100644 index 0000000000..4ead449bb9 --- /dev/null +++ b/notebooks/fru_utils.py @@ -0,0 +1,416 @@ +import os +import argparse +from datetime import datetime + +import numpy as np +import pandas as pd + +from activitysim.cli import run +from activitysim.core import inject +from activitysim.core import tracing +from activitysim.core import config +from activitysim.core import pipeline +from activitysim.core import chunk +from activitysim.core import simulate +from activitysim.core import logit +from activitysim.abm.models.util import estimation +from activitysim.core import expressions +from activitysim.core.util import assign_in_place + + +def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True): + + """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" + resume_after = "trip_scheduling" + model_name = "trip_mode_choice" + chunk_size = 0 # test_mtc means no chunking + + pipeline.open_pipeline(resume_after) + # preload any bulky injectables (e.g. skims) not in pipeline + inject.get_injectable('preload_injectables', None) + pipeline._PIPELINE.rng().begin_step(model_name) + step_name = model_name + args = {} + checkpoint = pipeline.intermediate_checkpoint(model_name) + inject.set_step_args(args) + + trips = inject.get_table('trips') + tours_merged = inject.get_table('tours_merged') + network_los = inject.get_injectable('network_los') + + trace_label = 'trip_mode_choice' + model_settings_file_name = 'trip_mode_choice.yaml' + model_settings = config.read_model_settings(model_settings_file_name) + + logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') + mode_column_name = 'trip_mode' + + trips_df = trips.to_frame() + print("Running with %d trips", trips_df.shape[0]) + + tours_merged = tours_merged.to_frame() + tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] + + # - trips_merged - merge trips and tours_merged + trips_merged = pd.merge( + trips_df, + tours_merged, + left_on='tour_id', + right_index=True, + how="left") + assert trips_merged.index.equals(trips.index) + + # setup skim keys + assert ('trip_period' not in trips_merged) + trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) + + orig_col = 'origin' + dest_col = 'destination' + + constants = {} + constants.update(config.get_model_constants(model_settings)) + constants.update({ + 'ORIGIN': orig_col, + 'DESTINATION': dest_col + }) + + skim_dict = network_los.get_default_skim_dict() + + odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col, + dim3_key='trip_period') + dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col, + dim3_key='trip_period') + od_skim_wrapper = skim_dict.wrap('origin', 'destination') + + skims = { + "odt_skims": odt_skim_stack_wrapper, + "dot_skims": dot_skim_stack_wrapper, + "od_skims": od_skim_wrapper, + } + + model_spec = simulate.read_model_spec(file_name=model_settings['SPEC']) + nest_specs = config.get_logit_model_settings(model_settings) + + estimator = estimation.manager.begin_estimation('trip_mode_choice') + + choices_list = [] + for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'): + + if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes): + continue + + print("trip_mode_choice tour_type '%s' (%s trips)" % + (primary_purpose, len(trips_segment.index), )) + + # name index so tracing knows how to slice + assert trips_segment.index.name == 'trip_id' + + coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose) + + locals_dict = {} + locals_dict.update(constants) + locals_dict.update(coefficients) + + segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose) + + expressions.annotate_preprocessors( + trips_segment, locals_dict, skims, + model_settings, segment_trace_label) + + locals_dict.update(skims) + + ################ Replace wrapper function + # choices = mode_choice_simulate(...) + spec=simulate.eval_coefficients(model_spec, coefficients, estimator) + nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label) + choices = simulate.simple_simulate( + choosers=trips_segment, + spec=spec, + nest_spec=nest_spec, + skims=skims, + locals_d=locals_dict, + chunk_size=chunk_size, + want_logsums=logsum_column_name is not None, + trace_label=segment_trace_label, + trace_choice_name='trip_mode_choice', + estimator=estimator, + trace_column_names=None, + choose_individual_max_utility=choose_individual_max_utility) + # for consistency, always return dataframe, whether or not logsums were requested + if isinstance(choices, pd.Series): + choices = choices.to_frame('choice') + choices.rename(columns={'logsum': logsum_column_name, + 'choice': mode_column_name}, + inplace=True) + if not choose_individual_max_utility: + alts = spec.columns + choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) + ################ + choices_list.append(choices) + choices_df_asim = pd.concat(choices_list) + + # update trips table with choices (and potionally logssums) + trips_df = trips.to_frame() + + if (do_these_purposes is not None): + trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)] + + assign_in_place(trips_df, choices_df_asim) + assert not trips_df[mode_column_name].isnull().any() + + finalise = True + if finalise: + inject.set_step_args(None) + # + pipeline._PIPELINE.rng().end_step(model_name) + pipeline.add_checkpoint(model_name) + if not pipeline.intermediate_checkpoint(): + pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME) + + pipeline.close_pipeline() + + print("Done") + + return trips_df + + + + +def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, + log_alt_losers=False, + want_logsums=False, trace_label=None, + trace_choice_name=None, trace_column_names=None): + + trace_label = tracing.extend_trace_label(trace_label, 'eval_nl') + assert trace_label + have_trace_targets = tracing.has_trace_targets(choosers) + + logit.validate_nest_spec(nest_spec, trace_label) + raw_utilities = simulate.eval_utilities(spec, choosers, locals_d, + log_alt_losers=log_alt_losers, + trace_label=trace_label, have_trace_targets=have_trace_targets, + estimator=estimator, trace_column_names=trace_column_names) + # exponentiated utilities of leaves and nests + nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec) + nested_utils = simulate.compute_nested_utilities(raw_utilities, nest_spec) + # probabilities of alternatives relative to siblings sharing the same nest + nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, + trace_label=trace_label) + if want_logsums: + # logsum of nest root + logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) + # global (flattened) leaf probabilities based on relative nest coefficients (in spec order) + base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) + # note base_probabilities could all be zero since we allowed all probs for nests to be zero + # check here to print a clear message but make_choices will raise error if probs don't sum to 1 + BAD_PROB_THRESHOLD = 0.001 + no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD + if no_choices.any(): + print("BAD") + choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label) + if want_logsums: + choices = choices.to_frame('choice') + choices['logsum'] = logsums + return choices, raw_utilities, nested_exp_utilities, nested_utils, nested_probabilities, base_probabilities + + +def simple_simulate_dev(choosers, spec, nest_spec, + skims=None, locals_d=None, + chunk_size=0, custom_chooser=None, + log_alt_losers=False, + want_logsums=False, + estimator=None, + trace_label=None, trace_choice_name=None, trace_column_names=None): + trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate') + assert len(choosers) > 0 + result_list = [] + # segment by person type and pick the right spec for each person type + for i, chooser_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label): + # the following replaces choices = _simple_simulate(...) + if skims is not None: + simulate.set_skim_wrapper_targets(choosers, skims) + + # only do this for nested, logit is straight forward + assert nest_spec is not None + choices, raw_utilities, nested_exp_utilities, nested_utils, \ + nested_probs, base_probs = eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, + log_alt_losers=log_alt_losers, + want_logsums=want_logsums, estimator=estimator, trace_label=trace_label, + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) + + result_list.append(choices) + chunk.log_df(trace_label, f'result_list', result_list) + + if len(result_list) > 1: + choices = pd.concat(result_list) + assert len(choices.index == len(choosers.index)) + return choices, raw_utilities, nested_exp_utilities, nested_utils, nested_probs, base_probs + + +def get_stuff(do_these_purposes=None): + #do_these_purposes=['escort'] + """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" + resume_after = "trip_scheduling" + model_name = "trip_mode_choice" + chunk_size = 0 # test_mtc means no chunking + + pipeline.open_pipeline(resume_after) + # preload any bulky injectables (e.g. skims) not in pipeline + inject.get_injectable('preload_injectables', None) + pipeline._PIPELINE.rng().begin_step(model_name) + step_name = model_name + args = {} + checkpoint = pipeline.intermediate_checkpoint(model_name) + inject.set_step_args(args) + + trips = inject.get_table('trips') + tours_merged = inject.get_table('tours_merged') + network_los = inject.get_injectable('network_los') + + trace_label = 'trip_mode_choice' + model_settings_file_name = 'trip_mode_choice.yaml' + model_settings = config.read_model_settings(model_settings_file_name) + + logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') + mode_column_name = 'trip_mode' + + trips_df = trips.to_frame() + print("Running with %d trips", trips_df.shape[0]) + + tours_merged = tours_merged.to_frame() + tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] + + # - trips_merged - merge trips and tours_merged + trips_merged = pd.merge( + trips_df, + tours_merged, + left_on='tour_id', + right_index=True, + how="left") + assert trips_merged.index.equals(trips.index) + + # setup skim keys + assert ('trip_period' not in trips_merged) + trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) + + orig_col = 'origin' + dest_col = 'destination' + + constants = {} + constants.update(config.get_model_constants(model_settings)) + constants.update({ + 'ORIGIN': orig_col, + 'DESTINATION': dest_col + }) + + skim_dict = network_los.get_default_skim_dict() + + odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col, + dim3_key='trip_period') + dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col, + dim3_key='trip_period') + od_skim_wrapper = skim_dict.wrap('origin', 'destination') + + skims = { + "odt_skims": odt_skim_stack_wrapper, + "dot_skims": dot_skim_stack_wrapper, + "od_skims": od_skim_wrapper, + } + + model_spec = simulate.read_model_spec(file_name=model_settings['SPEC']) + nest_specs = config.get_logit_model_settings(model_settings) + + estimator = estimation.manager.begin_estimation('trip_mode_choice') + + choices_list = [] + raw_util_list = [] + nest_list = [] + nu_list = [] + nest_spec_list = [] + nested_probs_list = [] + base_probs_list = [] + + for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'): + + if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes): + continue + + print("trip_mode_choice tour_type '%s' (%s trips)" % + (primary_purpose, len(trips_segment.index), )) + + # name index so tracing knows how to slice + assert trips_segment.index.name == 'trip_id' + + coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose) + + locals_dict = {} + locals_dict.update(constants) + locals_dict.update(coefficients) + + segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose) + + expressions.annotate_preprocessors( + trips_segment, locals_dict, skims, + model_settings, segment_trace_label) + + locals_dict.update(skims) + + ################ Replace wrapper function + # choices = mode_choice_simulate(...) + spec=simulate.eval_coefficients(model_spec, coefficients, estimator) + nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label) + choices, raw_utilities, nested_exp_utilities, nested_utils, nested_probs, base_probs = simple_simulate_dev( + choosers=trips_segment, + spec=spec, + nest_spec=nest_spec, + skims=skims, + locals_d=locals_dict, + chunk_size=chunk_size, + want_logsums=logsum_column_name is not None, + trace_label=segment_trace_label, + trace_choice_name='trip_mode_choice', + estimator=estimator, + trace_column_names=None) + # for consistency, always return dataframe, whether or not logsums were requested + if isinstance(choices, pd.Series): + choices = choices.to_frame('choice') + choices.rename(columns={'logsum': logsum_column_name, + 'choice': mode_column_name}, + inplace=True) + alts = spec.columns + choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) + ################ + choices_list.append(choices) + raw_util_list.append(raw_utilities) + nest_list.append(nested_exp_utilities) + nu_list.append(nested_utils) + nest_spec_list.append(nest_spec) + nested_probs_list.append(nested_probs) + base_probs_list.append(base_probs) + + + choices_df_asim = pd.concat(choices_list) + + # update trips table with choices (and potionally logssums) + trips_df = trips.to_frame() + + if (do_these_purposes is not None): + trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)] + + assign_in_place(trips_df, choices_df_asim) + assert not trips_df[mode_column_name].isnull().any() + + finalise = True + if finalise: + inject.set_step_args(None) + # + pipeline._PIPELINE.rng().end_step(model_name) + pipeline.add_checkpoint(model_name) + if not pipeline.intermediate_checkpoint(): + pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME) + + pipeline.close_pipeline() + + print("Done") + + return trips_df, raw_util_list, nest_list, nu_list, nest_spec_list, nested_probs_list, base_probs_list \ No newline at end of file diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index daeb7dab35..c4262ba02b 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -104,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -124,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -143,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -163,8 +163,6 @@ "# parser.parse_args(['--sum', '7', '-1', '42'])\n", "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", "#run.run(args) # 2mins full example run\n", - "\n", - "\n", "if not inject.is_injectable('preload_injectables'):\n", " from activitysim import abm # register abm steps and other abm-specific injectables\n", "run.handle_standard_args(args) # possibly update injectables" @@ -172,174 +170,7 @@ }, { "cell_type": "code", - "execution_count": 54, - "outputs": [], - "source": [ - "def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True):\n", - "\n", - " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", - " resume_after = \"trip_scheduling\"\n", - " model_name = \"trip_mode_choice\"\n", - " chunk_size = 0 # test_mtc means no chunking\n", - "\n", - " pipeline.open_pipeline(resume_after)\n", - " # preload any bulky injectables (e.g. skims) not in pipeline\n", - " inject.get_injectable('preload_injectables', None)\n", - " pipeline._PIPELINE.rng().begin_step(model_name)\n", - " step_name = model_name\n", - " args = {}\n", - " checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - " inject.set_step_args(args)\n", - "\n", - " trips = inject.get_table('trips')\n", - " tours_merged = inject.get_table('tours_merged')\n", - " network_los = inject.get_injectable('network_los')\n", - "\n", - " trace_label = 'trip_mode_choice'\n", - " model_settings_file_name = 'trip_mode_choice.yaml'\n", - " model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - " mode_column_name = 'trip_mode'\n", - "\n", - " trips_df = trips.to_frame()\n", - " print(\"Running with %d trips\", trips_df.shape[0])\n", - "\n", - " tours_merged = tours_merged.to_frame()\n", - " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "\n", - " # - trips_merged - merge trips and tours_merged\n", - " trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - " assert trips_merged.index.equals(trips.index)\n", - "\n", - " # setup skim keys\n", - " assert ('trip_period' not in trips_merged)\n", - " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - " orig_col = 'origin'\n", - " dest_col = 'destination'\n", - "\n", - " constants = {}\n", - " constants.update(config.get_model_constants(model_settings))\n", - " constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - " })\n", - "\n", - " skim_dict = network_los.get_default_skim_dict()\n", - "\n", - " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - " skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - " }\n", - "\n", - " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - " nest_specs = config.get_logit_model_settings(model_settings)\n", - "\n", - " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - "\n", - " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", - " continue\n", - "\n", - " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " (primary_purpose, len(trips_segment.index), ))\n", - "\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " ################ Replace wrapper function\n", - " # choices = mode_choice_simulate(...)\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", - " choices = simulate.simple_simulate(\n", - " choosers=trips_segment,\n", - " spec=spec,\n", - " nest_spec=nest_spec,\n", - " skims=skims,\n", - " locals_d=locals_dict,\n", - " chunk_size=chunk_size,\n", - " want_logsums=logsum_column_name is not None,\n", - " trace_label=segment_trace_label,\n", - " trace_choice_name='trip_mode_choice',\n", - " estimator=estimator,\n", - " trace_column_names=None,\n", - " choose_individual_max_utility=choose_individual_max_utility)\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " if not choose_individual_max_utility:\n", - " alts = spec.columns\n", - " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", - " ################\n", - " choices_list.append(choices)\n", - " choices_df_asim = pd.concat(choices_list)\n", - "\n", - " # update trips table with choices (and potionally logssums)\n", - " trips_df = trips.to_frame()\n", - "\n", - " if (do_these_purposes is not None):\n", - " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", - "\n", - " assign_in_place(trips_df, choices_df_asim)\n", - " assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - " finalise = True\n", - " if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - " print(\"Done\")\n", - "\n", - " return trips_df" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 55, + "execution_count": 8, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -374,20 +205,7 @@ "trip_mode_choice tour_type 'shopping' (77 trips)\n", "trip_mode_choice tour_type 'social' (19 trips)\n", "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n" ] }, @@ -395,6 +213,7 @@ "name": "stderr", "output_type": "stream", "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -418,20 +237,21 @@ } ], "source": [ + "from fru_utils import run_trip_mode_choice\n", "trips_df_ru = run_trip_mode_choice()\n", "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 9, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 299 304\nWALK_LOC 92 85\nWALK_LRF 61 53\nBIKE 17 17\nDRIVEALONEFREE 6 4\nSHARED2FREE 3 4\nTNC_SINGLE 2 4\nWALK_HVY 2 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK299304
WALK_LOC9285
WALK_LRF6153
BIKE1717
DRIVEALONEFREE64
SHARED2FREE34
TNC_SINGLE24
WALK_HVY24
\n
" }, - "execution_count": 56, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -451,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 62, "outputs": [ { "name": "stdout", @@ -475,26 +295,19 @@ } }, { - "cell_type": "code", - "execution_count": 58, - "outputs": [ - { - "data": { - "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n8684833 WALK -1.171760 WALK \n8684837 WALK -1.238719 WALK \n8685009 BIKE 6.198626 BIKE \n8685013 WALK 6.175681 BIKE \n8753057 WALK 4.457539 WALK \n... ... ... ... \n2472945113 WALK_LOC 12.537675 WALK_LOC \n2472945117 WALK 11.880804 WALK_LOC \n2472945118 WALK_LOC 13.710030 WALK \n2473024473 WALK_LOC -0.530696 WALK_LOC \n2473024477 WALK_LRF 0.624304 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n8684833 -1.171760 \n8684837 -1.238719 \n8685009 6.198626 \n8685013 6.175681 \n8753057 4.457539 \n... ... \n2472945113 12.537675 \n2472945117 11.880804 \n2472945118 13.710030 \n2473024473 -0.530696 \n2473024477 0.624304 \n\n[482 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
8684833WALK-1.171760WALK-1.171760
8684837WALK-1.238719WALK-1.238719
8685009BIKE6.198626BIKE6.198626
8685013WALK6.175681BIKE6.175681
8753057WALK4.457539WALK4.457539
...............
2472945113WALK_LOC12.537675WALK_LOC12.537675
2472945117WALK11.880804WALK_LOC11.880804
2472945118WALK_LOC13.710030WALK13.710030
2473024473WALK_LOC-0.530696WALK_LOC-0.530696
2473024477WALK_LRF0.624304WALK_LRF0.624304
\n

482 rows × 4 columns

\n
" - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], + "cell_type": "markdown", "source": [ - "trips" + "# Run MC validation\n", + "\n", + "see logit._each_nest parent_nest_scale for leaf node, and simulate.compute_nested_utilities for discussion of scales\n", + "of alternatives\n", + "\n", + "easiest way is to run a simulation" ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } } }, @@ -512,9 +325,30 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 14, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'social' (19 trips)\n", + "Done\n" + ] + } + ], + "source": [ + "from fru_utils import get_stuff\n", + "t, ru, neu, nu, ns, nested_probs, base_probs = get_stuff(do_these_purposes=[\"social\"])" + ], "metadata": { "collapsed": false, "pycharm": { @@ -524,9 +358,21 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 15, + "outputs": [ + { + "data": { + "text/plain": " AUTO NONMOTORIZED TRANSIT RIDEHAIL DRIVEALONE \\\ntrip_id \n8805121 0.000000e+00 1.000000 0.000000 1.418229e-07 0.0 \n8805125 0.000000e+00 1.000000 0.000000 1.333383e-07 0.0 \n8805126 0.000000e+00 1.000000 0.000000 1.666869e-07 0.0 \n468119921 0.000000e+00 1.000000 0.000000 5.175690e-08 0.0 \n468119922 0.000000e+00 0.999874 0.000000 1.255768e-04 0.0 \n468119925 0.000000e+00 0.999875 0.000000 1.254853e-04 0.0 \n468119926 0.000000e+00 1.000000 0.000000 5.207466e-08 0.0 \n468119927 0.000000e+00 1.000000 0.000000 7.363066e-08 0.0 \n468119928 0.000000e+00 1.000000 0.000000 2.191280e-08 0.0 \n642446345 4.882838e-03 0.044905 0.949699 5.136180e-04 0.0 \n642446349 3.622165e-03 0.043359 0.952506 5.131357e-04 0.0 \n1179608009 7.537428e-05 0.969538 0.000000 3.038711e-02 0.0 \n1179608013 1.855034e-05 0.969598 0.000000 3.038345e-02 0.0 \n1658807153 5.751672e-07 0.048044 0.951445 5.107302e-04 0.0 \n1658807157 6.945596e-07 0.065847 0.933430 7.226819e-04 0.0 \n1767182945 1.610090e-02 0.046835 0.931353 5.711349e-03 0.0 \n1767182949 1.540381e-02 0.041767 0.937130 5.699151e-03 0.0 \n2464104881 2.228876e-05 0.646947 0.353030 4.902454e-07 0.0 \n2464104885 2.163505e-05 0.627973 0.372005 4.758670e-07 0.0 \n\n SHAREDRIDE2 SHAREDRIDE3 DRIVEALONEFREE DRIVEALONEPAY \\\ntrip_id \n8805121 0.000000 0.000000e+00 0.0 0.0 \n8805125 0.000000 0.000000e+00 0.0 0.0 \n8805126 0.000000 0.000000e+00 0.0 0.0 \n468119921 0.000000 0.000000e+00 0.0 0.0 \n468119922 0.000000 0.000000e+00 0.0 0.0 \n468119925 0.000000 0.000000e+00 0.0 0.0 \n468119926 0.000000 0.000000e+00 0.0 0.0 \n468119927 0.000000 0.000000e+00 0.0 0.0 \n468119928 0.000000 0.000000e+00 0.0 0.0 \n642446345 1.000000 9.628424e-12 0.0 0.0 \n642446349 1.000000 1.090669e-11 0.0 0.0 \n1179608009 0.739343 2.606568e-01 0.0 0.0 \n1179608013 0.597359 4.026407e-01 0.0 0.0 \n1658807153 0.739335 2.606646e-01 0.0 0.0 \n1658807157 0.725355 2.746454e-01 0.0 0.0 \n1767182945 1.000000 9.618359e-12 0.0 0.0 \n1767182949 1.000000 9.798594e-12 0.0 0.0 \n2464104881 1.000000 9.634468e-12 0.0 0.0 \n2464104885 1.000000 9.634468e-12 0.0 0.0 \n\n SHARED2FREE SHARED2PAY SHARED3FREE SHARED3PAY WALK BIKE \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 1.0 0.0 \n8805125 0.0 0.0 0.0 0.0 1.0 0.0 \n8805126 0.0 0.0 0.0 0.0 1.0 0.0 \n468119921 0.0 0.0 0.0 0.0 1.0 0.0 \n468119922 0.0 0.0 0.0 0.0 1.0 0.0 \n468119925 0.0 0.0 0.0 0.0 1.0 0.0 \n468119926 0.0 0.0 0.0 0.0 1.0 0.0 \n468119927 0.0 0.0 0.0 0.0 1.0 0.0 \n468119928 0.0 0.0 0.0 0.0 1.0 0.0 \n642446345 1.0 0.0 1.0 0.0 1.0 0.0 \n642446349 1.0 0.0 1.0 0.0 1.0 0.0 \n1179608009 1.0 0.0 1.0 0.0 1.0 0.0 \n1179608013 1.0 0.0 1.0 0.0 1.0 0.0 \n1658807153 1.0 0.0 1.0 0.0 1.0 0.0 \n1658807157 1.0 0.0 1.0 0.0 1.0 0.0 \n1767182945 1.0 0.0 1.0 0.0 1.0 0.0 \n1767182949 1.0 0.0 1.0 0.0 1.0 0.0 \n2464104881 1.0 0.0 1.0 0.0 1.0 0.0 \n2464104885 1.0 0.0 1.0 0.0 1.0 0.0 \n\n WALKACCESS DRIVEACCESS WALK_LOC WALK_LRF WALK_EXP WALK_HVY \\\ntrip_id \n8805121 0.0 0.0 0.000000 0.000000 0.0 0.0 \n8805125 0.0 0.0 0.000000 0.000000 0.0 0.0 \n8805126 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119921 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119922 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119925 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119926 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119927 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119928 0.0 0.0 0.000000 0.000000 0.0 0.0 \n642446345 1.0 0.0 0.063138 0.936862 0.0 0.0 \n642446349 1.0 0.0 0.071770 0.928230 0.0 0.0 \n1179608009 0.0 0.0 0.000000 0.000000 0.0 0.0 \n1179608013 0.0 0.0 0.000000 0.000000 0.0 0.0 \n1658807153 1.0 0.0 1.000000 0.000000 0.0 0.0 \n1658807157 1.0 0.0 1.000000 0.000000 0.0 0.0 \n1767182945 1.0 0.0 0.154889 0.845111 0.0 0.0 \n1767182949 1.0 0.0 0.150707 0.849293 0.0 0.0 \n2464104881 1.0 0.0 1.000000 0.000000 0.0 0.0 \n2464104885 1.0 0.0 1.000000 0.000000 0.0 0.0 \n\n WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.300854 0.553048 0.146098 \n8805125 0.304288 0.549407 0.146304 \n8805126 0.291815 0.562518 0.145667 \n468119921 0.211883 0.005323 0.782795 \n468119922 0.211720 0.005255 0.783024 \n468119925 0.211271 0.005261 0.783468 \n468119926 0.301114 0.004624 0.694262 \n468119927 0.720964 0.001787 0.277249 \n468119928 0.108040 0.006816 0.885144 \n642446345 0.187119 0.254576 0.558305 \n642446349 0.181108 0.256588 0.562305 \n1179608009 0.000172 0.910435 0.089393 \n1179608013 0.000172 0.910454 0.089374 \n1658807153 0.000139 0.919273 0.080588 \n1658807157 0.000132 0.919585 0.080283 \n1767182945 0.262451 0.588087 0.149462 \n1767182949 0.259083 0.592884 0.148034 \n2464104881 0.278497 0.000018 0.721485 \n2464104885 0.278497 0.000018 0.721485 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AUTONONMOTORIZEDTRANSITRIDEHAILDRIVEALONESHAREDRIDE2SHAREDRIDE3DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALKACCESSDRIVEACCESSWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.000000e+001.0000000.0000001.418229e-070.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.3008540.5530480.146098
88051250.000000e+001.0000000.0000001.333383e-070.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.3042880.5494070.146304
88051260.000000e+001.0000000.0000001.666869e-070.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2918150.5625180.145667
4681199210.000000e+001.0000000.0000005.175690e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2118830.0053230.782795
4681199220.000000e+000.9998740.0000001.255768e-040.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2117200.0052550.783024
4681199250.000000e+000.9998750.0000001.254853e-040.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2112710.0052610.783468
4681199260.000000e+001.0000000.0000005.207466e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.3011140.0046240.694262
4681199270.000000e+001.0000000.0000007.363066e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.7209640.0017870.277249
4681199280.000000e+001.0000000.0000002.191280e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.1080400.0068160.885144
6424463454.882838e-030.0449050.9496995.136180e-040.01.0000009.628424e-120.00.01.00.01.00.01.00.01.00.00.0631380.9368620.00.00.00.00.00.00.00.00.1871190.2545760.558305
6424463493.622165e-030.0433590.9525065.131357e-040.01.0000001.090669e-110.00.01.00.01.00.01.00.01.00.00.0717700.9282300.00.00.00.00.00.00.00.00.1811080.2565880.562305
11796080097.537428e-050.9695380.0000003.038711e-020.00.7393432.606568e-010.00.01.00.01.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.0001720.9104350.089393
11796080131.855034e-050.9695980.0000003.038345e-020.00.5973594.026407e-010.00.01.00.01.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.0001720.9104540.089374
16588071535.751672e-070.0480440.9514455.107302e-040.00.7393352.606646e-010.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.0001390.9192730.080588
16588071576.945596e-070.0658470.9334307.226819e-040.00.7253552.746454e-010.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.0001320.9195850.080283
17671829451.610090e-020.0468350.9313535.711349e-030.01.0000009.618359e-120.00.01.00.01.00.01.00.01.00.00.1548890.8451110.00.00.00.00.00.00.00.00.2624510.5880870.149462
17671829491.540381e-020.0417670.9371305.699151e-030.01.0000009.798594e-120.00.01.00.01.00.01.00.01.00.00.1507070.8492930.00.00.00.00.00.00.00.00.2590830.5928840.148034
24641048812.228876e-050.6469470.3530304.902454e-070.01.0000009.634468e-120.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.2784970.0000180.721485
24641048852.163505e-050.6279730.3720054.758670e-070.01.0000009.634468e-120.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.2784970.0000180.721485
\n
" + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nested_probs[0]" + ], "metadata": { "collapsed": false, "pycharm": { @@ -536,9 +382,21 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 16, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "base_probs[0]" + ], "metadata": { "collapsed": false, "pycharm": { @@ -550,7 +408,10 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [], + "source": [ + "# for later: this seems to be a non-driving segment - grab workers or something later?\n", + "# for now: implement a run with seeding so I can reproduce these counts" + ], "metadata": { "collapsed": false, "pycharm": { @@ -558,6 +419,13 @@ } } }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "collapsed": false + } + }, { "cell_type": "code", "execution_count": null, @@ -582,73 +450,11 @@ } } }, - { - "cell_type": "markdown", - "id": "2f080150-c15a-4059-8a26-42a8c0072606", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:16:40.014024Z", - "iopub.status.busy": "2021-08-31T23:16:40.013770Z", - "iopub.status.idle": "2021-08-31T23:16:40.236326Z", - "shell.execute_reply": "2021-08-31T23:16:40.235553Z", - "shell.execute_reply.started": "2021-08-31T23:16:40.013973Z" - }, - "tags": [] - }, - "source": [ - "### make choice at each level" - ] - }, - { - "cell_type": "code", - "execution_count": 315, - "id": "2ed71b54-67e4-4087-957a-75f4fa184144", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:43.778081Z", - "iopub.status.busy": "2021-09-01T01:11:43.777757Z", - "iopub.status.idle": "2021-09-01T01:11:43.975332Z", - "shell.execute_reply": "2021-09-01T01:11:43.974588Z", - "shell.execute_reply.started": "2021-09-01T01:11:43.778055Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def get_alternatives(nests, name):\n", - " alts = list(filter(lambda x: x.name == name, nests))\n", - " assert len(alts) == 1, f\"{len(alts)} not one\"\n", - " alts = alts[0].alternatives\n", - " return alts\n", - "\n", - "def recursive_choice(row, columns, nest_levels, nests):\n", - " choices = row[columns].idxmax() #axis=1).values[0]\n", - " next_level_columns = get_alternatives(nests, choices)\n", - " #print(f\"{choices} leads to columns {next_level_columns}\")\n", - " if next_level_columns is None:\n", - " return choices \n", - " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", - " return new_choice\n", - "\n", - "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", - "\n", - "def make_choice(utils_df, nests, nest_levels, seed=None):\n", - " rng = default_rng(seed=seed)\n", - " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", - " probs_arr = utils_df - np.log(-np.log(rands))\n", - " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", - " return choices" - ] - }, { "cell_type": "code", "execution_count": null, "outputs": [], - "source": [ - "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", - "seed = 9326543345\n", - "make_choice(utils_df, nests_, nest_levels, seed)" - ], + "source": [], "metadata": { "collapsed": false, "pycharm": { @@ -658,37 +464,15 @@ }, { "cell_type": "code", - "execution_count": 316, - "id": "30424562-2627-446b-abd5-7c763c52060e", + "execution_count": null, + "outputs": [], + "source": [], "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:49.444289Z", - "iopub.status.busy": "2021-09-01T01:11:49.444002Z", - "iopub.status.idle": "2021-09-01T01:11:49.664214Z", - "shell.execute_reply": "2021-09-01T01:11:49.663088Z", - "shell.execute_reply.started": "2021-09-01T01:11:49.444266Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4\n" - ] + "collapsed": false, + "pycharm": { + "name": "#%%\n" } - ], - "source": [ - "# group nests by level:\n", - "depth = np.max([x.level for x in nests_])\n", - "print(depth)\n", - "nest_levels = {x: [] for x in range(1, depth+1)}\n", - "for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - "assert len(nest_levels[1]) == 1\n", - "assert nest_levels[1][0] == 'root'" - ] + } }, { "cell_type": "code", @@ -722,128 +506,6 @@ "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" ] }, - { - "cell_type": "code", - "execution_count": 451, - "id": "054fd9b1-72fc-49cc-a790-48ef75bcbaed", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:55:08.260140Z", - "iopub.status.busy": "2021-09-01T04:55:08.259914Z", - "iopub.status.idle": "2021-09-01T04:55:09.140095Z", - "shell.execute_reply": "2021-09-01T04:55:09.139119Z", - "shell.execute_reply.started": "2021-09-01T04:55:08.260116Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "86627409 WALK_LRF\n", - "86627413 WALK_LRF\n", - "86673657 WALK_LOC\n", - "86673658 WALK\n", - "86673659 WALK\n", - " ... \n", - "2464446025 WALK\n", - "2464446029 WALK\n", - "2464449633 WALK\n", - "2464449634 WALK\n", - "2464449637 WALK\n", - "Length: 168, dtype: object" - ] - }, - "execution_count": 451, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def simple_simulate_rum(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", - "\n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices\n", - "\n", - "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" - ] - }, - { - "cell_type": "code", - "execution_count": 393, - "id": "f3c5c1ba-e9c3-4511-98bf-9ac98c987da7", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:39:52.539983Z", - "iopub.status.busy": "2021-09-01T02:39:52.539668Z", - "iopub.status.idle": "2021-09-01T02:39:52.796170Z", - "shell.execute_reply": "2021-09-01T02:39:52.795498Z", - "shell.execute_reply.started": "2021-09-01T02:39:52.539944Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " segment_trace_label, log_alt_losers=log_alt_losers, \n", - " trace_column_names=trace_column_names, seed=seed)\n", - "\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " choices_list.append(choices)\n", - "\n", - " choices_df = pd.concat(choices_list)\n", - " return choices_df" - ] - }, { "cell_type": "code", "execution_count": 454, From 5e8bf1648342458fcb903c9f0192cec1855543f5 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 5 May 2022 12:56:34 +1000 Subject: [PATCH 008/135] towards validation of probs for single trip --- notebooks/validate_frozen_impl.ipynb | 1600 ++++++++------------------ 1 file changed, 450 insertions(+), 1150 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index c4262ba02b..dbc4ea7990 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -205,15 +205,28 @@ "trip_mode_choice tour_type 'shopping' (77 trips)\n", "trip_mode_choice tour_type 'social' (19 trips)\n", "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n" + "trip_mode_choice tour_type 'work' (168 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "Running with %d trips 482\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -221,7 +234,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", @@ -271,7 +283,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 10, "outputs": [ { "name": "stdout", @@ -295,25 +307,20 @@ } }, { - "cell_type": "markdown", - "source": [ - "# Run MC validation\n", - "\n", - "see logit._each_nest parent_nest_scale for leaf node, and simulate.compute_nested_utilities for discussion of scales\n", - "of alternatives\n", - "\n", - "easiest way is to run a simulation" - ], + "cell_type": "code", + "execution_count": 10, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { - "name": "#%% md\n" + "name": "#%%\n" } } }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "outputs": [], "source": [], "metadata": { @@ -323,9 +330,26 @@ } } }, + { + "cell_type": "markdown", + "source": [ + "# Run MC validation\n", + "\n", + "see logit._each_nest parent_nest_scale for leaf node, and simulate.compute_nested_utilities for discussion of scales\n", + "of alternatives\n", + "\n", + "easiest way is to run a simulation" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 11, "outputs": [ { "name": "stderr", @@ -358,20 +382,41 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 12, "outputs": [ { "data": { - "text/plain": " AUTO NONMOTORIZED TRANSIT RIDEHAIL DRIVEALONE \\\ntrip_id \n8805121 0.000000e+00 1.000000 0.000000 1.418229e-07 0.0 \n8805125 0.000000e+00 1.000000 0.000000 1.333383e-07 0.0 \n8805126 0.000000e+00 1.000000 0.000000 1.666869e-07 0.0 \n468119921 0.000000e+00 1.000000 0.000000 5.175690e-08 0.0 \n468119922 0.000000e+00 0.999874 0.000000 1.255768e-04 0.0 \n468119925 0.000000e+00 0.999875 0.000000 1.254853e-04 0.0 \n468119926 0.000000e+00 1.000000 0.000000 5.207466e-08 0.0 \n468119927 0.000000e+00 1.000000 0.000000 7.363066e-08 0.0 \n468119928 0.000000e+00 1.000000 0.000000 2.191280e-08 0.0 \n642446345 4.882838e-03 0.044905 0.949699 5.136180e-04 0.0 \n642446349 3.622165e-03 0.043359 0.952506 5.131357e-04 0.0 \n1179608009 7.537428e-05 0.969538 0.000000 3.038711e-02 0.0 \n1179608013 1.855034e-05 0.969598 0.000000 3.038345e-02 0.0 \n1658807153 5.751672e-07 0.048044 0.951445 5.107302e-04 0.0 \n1658807157 6.945596e-07 0.065847 0.933430 7.226819e-04 0.0 \n1767182945 1.610090e-02 0.046835 0.931353 5.711349e-03 0.0 \n1767182949 1.540381e-02 0.041767 0.937130 5.699151e-03 0.0 \n2464104881 2.228876e-05 0.646947 0.353030 4.902454e-07 0.0 \n2464104885 2.163505e-05 0.627973 0.372005 4.758670e-07 0.0 \n\n SHAREDRIDE2 SHAREDRIDE3 DRIVEALONEFREE DRIVEALONEPAY \\\ntrip_id \n8805121 0.000000 0.000000e+00 0.0 0.0 \n8805125 0.000000 0.000000e+00 0.0 0.0 \n8805126 0.000000 0.000000e+00 0.0 0.0 \n468119921 0.000000 0.000000e+00 0.0 0.0 \n468119922 0.000000 0.000000e+00 0.0 0.0 \n468119925 0.000000 0.000000e+00 0.0 0.0 \n468119926 0.000000 0.000000e+00 0.0 0.0 \n468119927 0.000000 0.000000e+00 0.0 0.0 \n468119928 0.000000 0.000000e+00 0.0 0.0 \n642446345 1.000000 9.628424e-12 0.0 0.0 \n642446349 1.000000 1.090669e-11 0.0 0.0 \n1179608009 0.739343 2.606568e-01 0.0 0.0 \n1179608013 0.597359 4.026407e-01 0.0 0.0 \n1658807153 0.739335 2.606646e-01 0.0 0.0 \n1658807157 0.725355 2.746454e-01 0.0 0.0 \n1767182945 1.000000 9.618359e-12 0.0 0.0 \n1767182949 1.000000 9.798594e-12 0.0 0.0 \n2464104881 1.000000 9.634468e-12 0.0 0.0 \n2464104885 1.000000 9.634468e-12 0.0 0.0 \n\n SHARED2FREE SHARED2PAY SHARED3FREE SHARED3PAY WALK BIKE \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 1.0 0.0 \n8805125 0.0 0.0 0.0 0.0 1.0 0.0 \n8805126 0.0 0.0 0.0 0.0 1.0 0.0 \n468119921 0.0 0.0 0.0 0.0 1.0 0.0 \n468119922 0.0 0.0 0.0 0.0 1.0 0.0 \n468119925 0.0 0.0 0.0 0.0 1.0 0.0 \n468119926 0.0 0.0 0.0 0.0 1.0 0.0 \n468119927 0.0 0.0 0.0 0.0 1.0 0.0 \n468119928 0.0 0.0 0.0 0.0 1.0 0.0 \n642446345 1.0 0.0 1.0 0.0 1.0 0.0 \n642446349 1.0 0.0 1.0 0.0 1.0 0.0 \n1179608009 1.0 0.0 1.0 0.0 1.0 0.0 \n1179608013 1.0 0.0 1.0 0.0 1.0 0.0 \n1658807153 1.0 0.0 1.0 0.0 1.0 0.0 \n1658807157 1.0 0.0 1.0 0.0 1.0 0.0 \n1767182945 1.0 0.0 1.0 0.0 1.0 0.0 \n1767182949 1.0 0.0 1.0 0.0 1.0 0.0 \n2464104881 1.0 0.0 1.0 0.0 1.0 0.0 \n2464104885 1.0 0.0 1.0 0.0 1.0 0.0 \n\n WALKACCESS DRIVEACCESS WALK_LOC WALK_LRF WALK_EXP WALK_HVY \\\ntrip_id \n8805121 0.0 0.0 0.000000 0.000000 0.0 0.0 \n8805125 0.0 0.0 0.000000 0.000000 0.0 0.0 \n8805126 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119921 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119922 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119925 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119926 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119927 0.0 0.0 0.000000 0.000000 0.0 0.0 \n468119928 0.0 0.0 0.000000 0.000000 0.0 0.0 \n642446345 1.0 0.0 0.063138 0.936862 0.0 0.0 \n642446349 1.0 0.0 0.071770 0.928230 0.0 0.0 \n1179608009 0.0 0.0 0.000000 0.000000 0.0 0.0 \n1179608013 0.0 0.0 0.000000 0.000000 0.0 0.0 \n1658807153 1.0 0.0 1.000000 0.000000 0.0 0.0 \n1658807157 1.0 0.0 1.000000 0.000000 0.0 0.0 \n1767182945 1.0 0.0 0.154889 0.845111 0.0 0.0 \n1767182949 1.0 0.0 0.150707 0.849293 0.0 0.0 \n2464104881 1.0 0.0 1.000000 0.000000 0.0 0.0 \n2464104885 1.0 0.0 1.000000 0.000000 0.0 0.0 \n\n WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.300854 0.553048 0.146098 \n8805125 0.304288 0.549407 0.146304 \n8805126 0.291815 0.562518 0.145667 \n468119921 0.211883 0.005323 0.782795 \n468119922 0.211720 0.005255 0.783024 \n468119925 0.211271 0.005261 0.783468 \n468119926 0.301114 0.004624 0.694262 \n468119927 0.720964 0.001787 0.277249 \n468119928 0.108040 0.006816 0.885144 \n642446345 0.187119 0.254576 0.558305 \n642446349 0.181108 0.256588 0.562305 \n1179608009 0.000172 0.910435 0.089393 \n1179608013 0.000172 0.910454 0.089374 \n1658807153 0.000139 0.919273 0.080588 \n1658807157 0.000132 0.919585 0.080283 \n1767182945 0.262451 0.588087 0.149462 \n1767182949 0.259083 0.592884 0.148034 \n2464104881 0.278497 0.000018 0.721485 \n2464104885 0.278497 0.000018 0.721485 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
AUTONONMOTORIZEDTRANSITRIDEHAILDRIVEALONESHAREDRIDE2SHAREDRIDE3DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALKACCESSDRIVEACCESSWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.000000e+001.0000000.0000001.418229e-070.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.3008540.5530480.146098
88051250.000000e+001.0000000.0000001.333383e-070.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.3042880.5494070.146304
88051260.000000e+001.0000000.0000001.666869e-070.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2918150.5625180.145667
4681199210.000000e+001.0000000.0000005.175690e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2118830.0053230.782795
4681199220.000000e+000.9998740.0000001.255768e-040.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2117200.0052550.783024
4681199250.000000e+000.9998750.0000001.254853e-040.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.2112710.0052610.783468
4681199260.000000e+001.0000000.0000005.207466e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.3011140.0046240.694262
4681199270.000000e+001.0000000.0000007.363066e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.7209640.0017870.277249
4681199280.000000e+001.0000000.0000002.191280e-080.00.0000000.000000e+000.00.00.00.00.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.1080400.0068160.885144
6424463454.882838e-030.0449050.9496995.136180e-040.01.0000009.628424e-120.00.01.00.01.00.01.00.01.00.00.0631380.9368620.00.00.00.00.00.00.00.00.1871190.2545760.558305
6424463493.622165e-030.0433590.9525065.131357e-040.01.0000001.090669e-110.00.01.00.01.00.01.00.01.00.00.0717700.9282300.00.00.00.00.00.00.00.00.1811080.2565880.562305
11796080097.537428e-050.9695380.0000003.038711e-020.00.7393432.606568e-010.00.01.00.01.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.0001720.9104350.089393
11796080131.855034e-050.9695980.0000003.038345e-020.00.5973594.026407e-010.00.01.00.01.00.01.00.00.00.00.0000000.0000000.00.00.00.00.00.00.00.00.0001720.9104540.089374
16588071535.751672e-070.0480440.9514455.107302e-040.00.7393352.606646e-010.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.0001390.9192730.080588
16588071576.945596e-070.0658470.9334307.226819e-040.00.7253552.746454e-010.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.0001320.9195850.080283
17671829451.610090e-020.0468350.9313535.711349e-030.01.0000009.618359e-120.00.01.00.01.00.01.00.01.00.00.1548890.8451110.00.00.00.00.00.00.00.00.2624510.5880870.149462
17671829491.540381e-020.0417670.9371305.699151e-030.01.0000009.798594e-120.00.01.00.01.00.01.00.01.00.00.1507070.8492930.00.00.00.00.00.00.00.00.2590830.5928840.148034
24641048812.228876e-050.6469470.3530304.902454e-070.01.0000009.634468e-120.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.2784970.0000180.721485
24641048852.163505e-050.6279730.3720054.758670e-070.01.0000009.634468e-120.00.01.00.01.00.01.00.01.00.01.0000000.0000000.00.00.00.00.00.00.00.00.2784970.0000180.721485
\n
" + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" }, - "execution_count": 15, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "nested_probs[0]" + "# nested_probs[0] - these are relative in each nest, so the full probs are potentially just the product of these\n", + "base_probs[0]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 13, + "outputs": [], + "source": [ + "# for later: this seems to be a non-driving segment - grab workers or something later?\n", + "# for now: implement a run with seeding so I can reproduce these counts\n", + "\n", + "# actually, can I manipulate pipeline OR inject table dynamically? If I reproduce a single observation many times we\n", + "# can check mode shares\n", + "# 1) check tests\n", + "# 2) check code in mode choice, maybe duplicate here, see old code below" ], "metadata": { "collapsed": false, @@ -380,22 +425,28 @@ } } }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "collapsed": false + } + }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 70, "outputs": [ { "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" + "text/plain": "array([ 8684833, 8684837, 8685009, 8685013, 8753057,\n 8753061, 8753233, 8753237, 8805121, 8805125,\n 8805126, 9094329, 9094333, 9717265, 9717269,\n 35302033, 35302037, 36757193, 36757197, 36757257,\n 36757261, 86627409, 86627413, 86627625, 86627629,\n 86673657, 86673658, 86673659, 86673660, 86673661,\n 106170305, 106170309, 106289041, 106289045, 106289089,\n 106289090, 106289091, 106289092, 106289093, 106289369,\n 106289373, 106570377, 106570381, 106570382, 106570383,\n 106570465, 106570469, 106741681, 106741682, 106741685,\n 106741686, 106741687, 106741688, 106741961, 106741965,\n 106804593, 106804597, 106804598, 106804657, 106804661,\n 106804662, 106804663, 106804985, 106804989, 126610345,\n 126610349, 126610561, 126610565, 126610566, 126610673,\n 126610677, 126610873, 126610877, 126628233, 126628237,\n 126628585, 126628589, 137248721, 137248725, 137248873,\n 137248877, 137248961, 137248965, 137249225, 137249229,\n 195385257, 195385261, 211327433, 211327437, 211327673,\n 211327677, 211327761, 211327765, 211328025, 211328029,\n 211328030, 211388201, 211388205, 211388329, 211388333,\n 211388353, 211388357, 211388441, 211388445, 211388721,\n 211388725, 211389033, 211389037, 414508737, 414508741,\n 415214745, 415214749, 415556913, 415556917, 444793569,\n 444793573, 444793574, 444853217, 444853221, 444853265,\n 444853269, 444928617, 444928621, 467399281, 467399285,\n 468119569, 468119573, 468119857, 468119861, 468119921,\n 468119922, 468119925, 468119926, 468119927, 468119928,\n 484173817, 484173821, 484173905, 484173909, 484174185,\n 484174189, 515832417, 515832418, 515832419, 515832421,\n 515832422, 515832423, 515921265, 515921266, 515921269,\n 515921305, 515921309, 535170409, 535170413, 535170689,\n 535170693, 535170694, 535377657, 535377661, 535388201,\n 535388202, 535388205, 535388481, 535388485, 535620049,\n 535620053, 535672921, 535672925, 535694417, 535694421,\n 535694457, 535694461, 535694465, 535694466, 535694469,\n 581959169, 581959170, 581959171, 581959172, 581959173,\n 581959174, 581959561, 581959565, 615236801, 615236805,\n 615236865, 615236866, 615236869, 615350289, 615350293,\n 615350553, 615350557, 615350558, 615350617, 615350621,\n 642446345, 642446349, 642446633, 642446637, 708171009,\n 708171013, 708171014, 708171273, 708171277, 708171601,\n 708171605, 728159001, 728159005, 728159049, 728159053,\n 728159329, 728159333, 728159569, 728159573, 728159593,\n 728159597, 806388153, 806388157, 806388225, 806388229,\n 806388401, 806388405, 806388905, 806388906, 806388909,\n 806389233, 806389237, 841877257, 841877261, 841877849,\n 841877853, 841878177, 841878181, 841878505, 841878509,\n 841878510, 943749401, 943749405, 943749465, 943749469,\n 943749470, 943749471, 943750385, 943750389, 963188601,\n 963188605, 963188665, 963188669, 963285425, 963285426,\n 963285427, 963285429, 963285433, 963285437, 963285438,\n 963285439, 963286193, 963286197, 963286345, 963286346,\n 963286349, 969957321, 969957325, 1004301497, 1004301501,\n 1004301761, 1004301765, 1004301785, 1004301789, 1004301873,\n 1004301877, 1045822217, 1045822221, 1045822409, 1045822410,\n 1045822413, 1045822737, 1045822741, 1045823001, 1045823005,\n 1045823006, 1045823007, 1045823065, 1045823069, 1045823393,\n 1045823394, 1045823397, 1045823398, 1060575737, 1060575741,\n 1060575801, 1060575805, 1060575809, 1060575813, 1060575849,\n 1060575853, 1091733225, 1091733229, 1091770617, 1091770621,\n 1091770681, 1091770685, 1091770897, 1091770898, 1091770901,\n 1146269081, 1146269085, 1146269345, 1146269349, 1146269369,\n 1146269373, 1146269374, 1146269375, 1146269376, 1146386553,\n 1146386557, 1146386561, 1146386565, 1146386881, 1146386885,\n 1146472377, 1146472381, 1146472489, 1146472493, 1146472537,\n 1146472538, 1146472539, 1146472540, 1146472541, 1146472817,\n 1146472821, 1179607441, 1179607445, 1179607657, 1179607661,\n 1179607662, 1179607663, 1179608009, 1179608013, 1276281769,\n 1276281773, 1276282049, 1276282053, 1276282313, 1276282317,\n 1276282321, 1276282325, 1276282361, 1276282365, 1368289969,\n 1368289973, 1368290273, 1368290277, 1368290689, 1368290690,\n 1368290693, 1368291297, 1368291301, 1368291302, 1368291609,\n 1368291613, 1368292281, 1368292285, 1368292377, 1368292381,\n 1368292382, 1368292657, 1368292658, 1368292661, 1368292662,\n 1582205617, 1582205621, 1582205641, 1582205645, 1582205729,\n 1582205733, 1658748793, 1658748797, 1658806913, 1658806917,\n 1658807089, 1658807093, 1658807129, 1658807133, 1658807153,\n 1658807157, 1658807177, 1658807181, 1658947233, 1658947234,\n 1658947237, 1658947238, 1766872681, 1766872685, 1766872897,\n 1766872901, 1766872921, 1766872925, 1767013721, 1767013725,\n 1767013726, 1767013785, 1767013789, 1767182857, 1767182861,\n 1767182865, 1767182869, 1767182945, 1767182949, 1767183169,\n 1767183173, 1767183297, 1767183301, 1767186249, 1767186253,\n 1767186577, 1767186578, 1767186581, 1767186582, 1767666161,\n 1767666165, 1767666233, 1767666237, 1767666441, 1767666445,\n 1767666769, 1767666773, 1768236897, 1768236901, 1768237161,\n 1768237165, 1768237166, 1768237167, 1768237489, 1768237493,\n 2396217321, 2396217325, 2396217433, 2396217437, 2396217761,\n 2396217765, 2396217769, 2396217773, 2444719729, 2444719733,\n 2463663417, 2463663421, 2463971785, 2463971789, 2463971790,\n 2463972065, 2463972069, 2463972070, 2463972071, 2464104641,\n 2464104645, 2464104857, 2464104861, 2464104881, 2464104885,\n 2464131801, 2464131805, 2464406009, 2464406013, 2464443401,\n 2464443405, 2464446025, 2464446029, 2464449633, 2464449634,\n 2464449637, 2467713777, 2467713781, 2467756745, 2467756749,\n 2472945097, 2472945101, 2472945113, 2472945117, 2472945118,\n 2473024473, 2473024477])" }, - "execution_count": 16, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "base_probs[0]" + "pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values" ], "metadata": { "collapsed": false, @@ -409,8 +460,7 @@ "execution_count": null, "outputs": [], "source": [ - "# for later: this seems to be a non-driving segment - grab workers or something later?\n", - "# for now: implement a run with seeding so I can reproduce these counts" + "pipeline._PIPELINE.rng().add_channel(\"trips\", )" ], "metadata": { "collapsed": false, @@ -420,10 +470,344 @@ } }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 59, + "outputs": [ + { + "data": { + "text/plain": "" + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pipeline._PIPELINE.rng().Sim" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 65, + "outputs": [], + "source": [ + "def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples):\n", + " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", + " resume_after = \"trip_scheduling\"\n", + " model_name = \"trip_mode_choice\"\n", + " chunk_size = 0 # test_mtc means no chunking\n", + "\n", + " pipeline.open_pipeline(resume_after)\n", + " # preload any bulky injectables (e.g. skims) not in pipeline\n", + " inject.get_injectable('preload_injectables', None)\n", + " pipeline._PIPELINE.rng().begin_step(model_name)\n", + " #step_name = model_name\n", + " args = {}\n", + " #checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + " inject.set_step_args(args)\n", + "\n", + " trips = inject.get_table('trips')\n", + " tours_merged = inject.get_table('tours_merged')\n", + " network_los = inject.get_injectable('network_los')\n", + "\n", + " trace_label = 'trip_mode_choice'\n", + " model_settings_file_name = 'trip_mode_choice.yaml'\n", + " model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + " mode_column_name = 'trip_mode'\n", + "\n", + " trips_df = trips.to_frame()\n", + " print(\"Running with %d trips\", trips_df.shape[0])\n", + "\n", + " tours_merged = tours_merged.to_frame()\n", + " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "\n", + " # - trips_merged - merge trips and tours_merged\n", + " trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + " assert trips_merged.index.equals(trips.index)\n", + "\n", + " # setup skim keys\n", + " assert ('trip_period' not in trips_merged)\n", + " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "\n", + " orig_col = 'origin'\n", + " dest_col = 'destination'\n", + "\n", + " constants = {}\n", + " constants.update(config.get_model_constants(model_settings))\n", + " constants.update({\n", + " 'ORIGIN': orig_col,\n", + " 'DESTINATION': dest_col\n", + " })\n", + "\n", + " skim_dict = network_los.get_default_skim_dict()\n", + "\n", + " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", + " dim3_key='trip_period')\n", + " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", + " dim3_key='trip_period')\n", + " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", + "\n", + " skims = {\n", + " \"odt_skims\": odt_skim_stack_wrapper,\n", + " \"dot_skims\": dot_skim_stack_wrapper,\n", + " \"od_skims\": od_skim_wrapper,\n", + " }\n", + "\n", + " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + " nest_specs = config.get_logit_model_settings(model_settings)\n", + "\n", + " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", + "\n", + " choices_list = []\n", + "\n", + "\n", + " # grab one, duplicate num_samples times\n", + " trips_segment = trips_merged.loc[trips_merged.index == trip_id_to_check].copy()\n", + " primary_purpose = trips_segment['primary_purpose'].values[0]\n", + " trips_segment = trips_segment.loc[trips_segment.index.repeat(num_samples)]\n", + "\n", + " # need to add new row_states for rng here, need to ensure there are no collisions with existing keys\n", + " existing_indexes = pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values\n", + " num_new_indexes = trips_segment.shape[0]\n", + " now find indexes - random draw and then remove existing and do again? simple loop for each, or batch and do\n", + " recursively? simple loop is fast enough here\n", + "\n", + "\n", + " trips_segment.index += np.arange(num_samples)\n", + " # name index so tracing knows how to slice\n", + " assert trips_segment.index.name == 'trip_id'\n", + "\n", + "\n", + " # #logger.warning(\"Change seeding back when done with testing\")\n", + " # pipeline._PIPELINE.rng.row_states = pd.DataFrame(columns=['row_seed', 'offset'], index=trips_segment.index)\n", + " # pipeline._PIPELINE.rng.row_states[\"row_seed\"] = trips_segment.index.values\n", + " # pipeline._PIPELINE.rng.row_states[\"offset\"] = 0\n", + " pipeline._PIPELINE.rng().add_channel(\"trips\", trips_segment)\n", + "\n", + "\n", + " #for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", + " #if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", + " # continue\n", + "\n", + " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", + " (primary_purpose, len(trips_segment.index), ))\n", + "\n", + " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", + "\n", + " locals_dict = {}\n", + " locals_dict.update(constants)\n", + " locals_dict.update(coefficients)\n", + "\n", + " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", + "\n", + " expressions.annotate_preprocessors(\n", + " trips_segment, locals_dict, skims,\n", + " model_settings, segment_trace_label)\n", + "\n", + " locals_dict.update(skims)\n", + "\n", + " ################ Replace wrapper function\n", + " # choices = mode_choice_simulate(...)\n", + " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", + " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", + " choices = simulate.simple_simulate(\n", + " choosers=trips_segment,\n", + " spec=spec,\n", + " nest_spec=nest_spec,\n", + " skims=skims,\n", + " locals_d=locals_dict,\n", + " chunk_size=chunk_size,\n", + " want_logsums=logsum_column_name is not None,\n", + " trace_label=segment_trace_label,\n", + " trace_choice_name='trip_mode_choice',\n", + " estimator=estimator,\n", + " trace_column_names=None,\n", + " choose_individual_max_utility=choose_individual_max_utility)\n", + " # for consistency, always return dataframe, whether or not logsums were requested\n", + " if isinstance(choices, pd.Series):\n", + " choices = choices.to_frame('choice')\n", + " choices.rename(columns={'logsum': logsum_column_name,\n", + " 'choice': mode_column_name},\n", + " inplace=True)\n", + " if not choose_individual_max_utility:\n", + " alts = spec.columns\n", + " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", + " ################\n", + " choices_list.append(choices)\n", + " choices_df_asim = pd.concat(choices_list)\n", + "\n", + " # update trips table with choices (and potionally logssums)\n", + " trips_df = trips.to_frame()\n", + "\n", + " #if (do_these_purposes is not None):\n", + " # trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", + "\n", + " assign_in_place(trips_df, choices_df_asim)\n", + " assert not trips_df[mode_column_name].isnull().any()\n", + "\n", + " finalise = True\n", + " if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + " print(\"Done\")\n", + " return trips_df" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 66, + "outputs": [], + "source": [ + "#pipeline.close_pipeline()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 67, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n" + ] + }, + { + "ename": "AssertionError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mAssertionError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/tmp/ipykernel_30125/1730006119.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 2\u001B[0m \u001B[0mnum_samples\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m100\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[0mtrip_id_to_check\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m642446345\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m \u001B[0mt_\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmode_choice_for_trip\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mchoose_individual_max_utility\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mchoose_individual_max_utility\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrip_id_to_check\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mtrip_id_to_check\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnum_samples\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mnum_samples\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", + "\u001B[0;32m/tmp/ipykernel_30125/3946923567.py\u001B[0m in \u001B[0;36mmode_choice_for_trip\u001B[0;34m(choose_individual_max_utility, trip_id_to_check, num_samples)\u001B[0m\n\u001B[1;32m 89\u001B[0m \u001B[0;31m# pipeline._PIPELINE.rng.row_states[\"row_seed\"] = trips_segment.index.values\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 90\u001B[0m \u001B[0;31m# pipeline._PIPELINE.rng.row_states[\"offset\"] = 0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 91\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_PIPELINE\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrng\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0madd_channel\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"trips\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrips_segment\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 92\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 93\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/random.py\u001B[0m in \u001B[0;36madd_channel\u001B[0;34m(self, channel_name, domain_df)\u001B[0m\n\u001B[1;32m 467\u001B[0m \u001B[0mchannel\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchannels\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mchannel_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 468\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 469\u001B[0;31m \u001B[0mchannel\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mextend_domain\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdomain_df\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 470\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 471\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/random.py\u001B[0m in \u001B[0;36mextend_domain\u001B[0;34m(self, domain_df)\u001B[0m\n\u001B[1;32m 142\u001B[0m \u001B[0;31m# row_states already exists, so we are extending\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 143\u001B[0m \u001B[0;31m# if extending, these should be new rows, no intersection with existing row_states\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 144\u001B[0;31m \u001B[0;32massert\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrow_states\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mintersection\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdomain_df\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 145\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrow_states\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconcat\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrow_states\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrow_states\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 146\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mAssertionError\u001B[0m: " + ] + } + ], + "source": [ + "choose_individual_max_utility = False\n", + "num_samples = 100\n", + "trip_id_to_check = 642446345\n", + "t_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, trip_id_to_check=trip_id_to_check, num_samples=num_samples)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 18, + "outputs": [], + "source": [ + "#pipeline.registered_tables()\n", + "#list(map(lambda x: x['checkpoint_name'], pipeline._PIPELINE.checkpoints))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], "source": [], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 24, + "outputs": [], + "source": [ + "num_samples = 100\n", + "trip_id_to_check = 642446345\n", + "\n", + "trips_dup = trips_merged.loc[trips_merged.index == trip_id_to_check].copy()\n", + "trips_dup = trips_dup.loc[trips_dup.index.repeat(num_samples)]\n", + "trips_dup.index += np.arange(num_samples)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } } }, { @@ -476,7 +860,18 @@ }, { "cell_type": "code", - "execution_count": 453, + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", "metadata": { "execution": { @@ -486,24 +881,13 @@ "shell.execute_reply": "2021-09-01T04:55:59.770801Z", "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" }, - "tags": [] + "tags": [], + "pycharm": { + "name": "#%% md\n" + } }, - "outputs": [], "source": [ - "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - " nested_probabilities = \\\n", - " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", - " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) \n", - " return base_probabilities\n", - "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" + "# OLD" ] }, { @@ -521,6 +905,22 @@ }, "outputs": [], "source": [ + "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", + " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", + " logit.validate_nest_spec(nest_spec, trace_label)\n", + " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", + " log_alt_losers=log_alt_losers,\n", + " trace_label=trace_label, have_trace_targets=False,\n", + " estimator=estimator, trace_column_names=trace_column_names)\n", + " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", + " nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec,\n", + " trace_label=trace_label)\n", + " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", + " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", + " return base_probabilities\n", + "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)\n", + "\n", + "\n", "## caculate probabilities with Asim methodology, should be correct\n", "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", @@ -906,139 +1306,10 @@ "outputs": [], "source": [] }, - { - "cell_type": "markdown", - "id": "f9e8ef02-1d57-4b21-8aef-25a6e1095c02", - "metadata": {}, - "source": [ - "### try zenith normalisation of simple_simulate_rum" - ] - }, { "cell_type": "code", - "execution_count": 629, - "id": "db235e90-d2e5-4e90-b41c-6d90a670e41d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:24.045881Z", - "iopub.status.busy": "2021-09-01T11:37:24.045637Z", - "iopub.status.idle": "2021-09-01T11:37:24.276488Z", - "shell.execute_reply": "2021-09-01T11:37:24.274898Z", - "shell.execute_reply.started": "2021-09-01T11:37:24.045844Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation belowi s equivalent.\n", - "def compute_nested_utilities_zenith(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) # / nest.product_of_coefficients #coefficient\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", - " \n", - " # now go over all leaves and correct for scale\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] /= nest.coefficient\n", - " \n", - " return nested_utilities\n", - "\n", - "\n", - "def simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " utils_df = compute_nested_utilities_zenith(raw_utilities, nest_spec)\n", - "\n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices\n", - "\n", - "#simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" - ] - }, - { - "cell_type": "code", - "execution_count": 630, - "id": "f69bc3dc-84e4-4dd5-9a57-c053cadba4b5", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:24.828926Z", - "iopub.status.busy": "2021-09-01T11:37:24.828707Z", - "iopub.status.idle": "2021-09-01T11:37:25.046409Z", - "shell.execute_reply": "2021-09-01T11:37:25.045399Z", - "shell.execute_reply.started": "2021-09-01T11:37:24.828902Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " choices = simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " segment_trace_label, log_alt_losers=log_alt_losers, \n", - " trace_column_names=trace_column_names, seed=seed)\n", - "\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " choices_list.append(choices)\n", - "\n", - " choices_df = pd.concat(choices_list)\n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 631, - "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", + "execution_count": 631, + "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", "metadata": { "execution": { "iopub.execute_input": "2021-09-01T11:37:32.250808Z", @@ -1381,977 +1652,6 @@ " display((100.0 * full_share_incl_zenith).T)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c0d3081-b45b-4573-9246-e4a9d39591dd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "612b2f04-45e2-4eb2-b52a-9f753427877f", - "metadata": {}, - "source": [ - "## investigate diverging seed and look at diff in formulation" - ] - }, - { - "cell_type": "code", - "execution_count": 619, - "id": "ac09dc75-a92f-43fb-a0cd-fe7de1f8b66d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:12:02.988141Z", - "iopub.status.busy": "2021-09-01T11:12:02.987900Z", - "iopub.status.idle": "2021-09-01T11:12:03.212179Z", - "shell.execute_reply": "2021-09-01T11:12:03.210993Z", - "shell.execute_reply.started": "2021-09-01T11:12:02.988102Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation based on asim probabilities is equivalent\n", - "def compute_nested_utilities_asim(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) / nest.product_of_coefficients\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", - " return nested_utilities\n", - "\n", - "def compute_nested_utilities_zenith_check(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) # scale correction is below\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", - " \n", - " # now go over all leaves and correct for scale\n", - " for nest in logit.each_nest(nest_spec):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] /= nest.coefficient\n", - " \n", - " return nested_utilities" - ] - }, - { - "cell_type": "code", - "execution_count": 688, - "id": "4b7c6368-21ca-4442-969e-f2f8ff868842", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:49:56.958862Z", - "iopub.status.busy": "2021-09-01T20:49:56.958623Z", - "iopub.status.idle": "2021-09-01T20:49:57.191884Z", - "shell.execute_reply": "2021-09-01T20:49:57.190975Z", - "shell.execute_reply.started": "2021-09-01T20:49:56.958827Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None, use_zenith=True, raw_utilities=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " \n", - " if raw_utilities is None:\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " if use_zenith:\n", - " utils_df = compute_nested_utilities_zenith_check(raw_utilities, nest_spec)\n", - " else:\n", - " utils_df = compute_nested_utilities_asim(raw_utilities, nest_spec)\n", - "\n", - " # test\n", - " #return compute_nested_utilities_zenith_check(raw_utilities, nest_spec), compute_nested_utilities_asim(raw_utilities, nest_spec)\n", - " \n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices" - ] - }, - { - "cell_type": "code", - "execution_count": 626, - "id": "bd6cadc3-79f5-4824-b55e-639c39d5ab18", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:13:12.077886Z", - "iopub.status.busy": "2021-09-01T11:13:12.077652Z", - "iopub.status.idle": "2021-09-01T11:13:12.290222Z", - "shell.execute_reply": "2021-09-01T11:13:12.288861Z", - "shell.execute_reply.started": "2021-09-01T11:13:12.077863Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# t_z, t_a = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed)\n", - "# tr_id = 86673661\n", - "# display(t_z.loc[t_z.index==tr_id])\n", - "# display(t_a.loc[t_a.index==tr_id])" - ] - }, - { - "cell_type": "code", - "execution_count": 625, - "id": "42e6da62-58e8-4b16-9169-9faa1fc8a162", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:13:06.847583Z", - "iopub.status.busy": "2021-09-01T11:13:06.847204Z", - "iopub.status.idle": "2021-09-01T11:13:08.401700Z", - "shell.execute_reply": "2021-09-01T11:13:08.400711Z", - "shell.execute_reply.started": "2021-09-01T11:13:06.847554Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed 51006 leads to 34 differences out of 168. tripids Int64Index([ 86673657, 86673661, 106741681, 106741682, 106741685,\n", - " 106741686, 106741687, 106741688, 211327433, 211327437,\n", - " 444793574, 484173905, 484173909, 535170694, 535620053,\n", - " 708171014, 943749470, 943749471, 1060575853, 1091770617,\n", - " 1146472489, 1146472493, 1276281769, 1276281773, 1658748793,\n", - " 1658748797, 1767013726, 1767186577, 1767186578, 1768237161,\n", - " 1768237165, 1768237166, 2463663417, 2463663421],\n", - " dtype='int64', name='trip_id')\n" - ] - } - ], - "source": [ - "seed = 51006\n", - "x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True)\n", - "y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False)\n", - "t_ = x_.to_frame('trip_mode_zenith').merge(y_.to_frame('trip_mode_asim'), left_index=True, right_index=True)\n", - "diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", - "print(f\"seed {seed} leads to {diffs.shape[0]} differences out of {t_.shape[0]}. tripids {diffs.index}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2f3c0506-a79d-4a31-9d56-71b9966d0a92", - "metadata": {}, - "outputs": [], - "source": [ - "seed 51006 leads to 3 differences. tripids Int64Index([86673661, 535170689, 1060575849], dtype='int64', name='trip_id')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1a5f332a-b1d7-4a80-bb21-d37a1611cf9f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24c5e4a1-6142-49d4-8da8-a1e266b3cb9b", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 753, - "id": "2098907f-28af-4f53-af74-6aaee8a6c53c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T23:04:23.861048Z", - "iopub.status.busy": "2021-09-01T23:04:23.860748Z", - "iopub.status.idle": "2021-09-01T23:04:24.069307Z", - "shell.execute_reply": "2021-09-01T23:04:24.068434Z", - "shell.execute_reply.started": "2021-09-01T23:04:23.861024Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "rng_ = default_rng(seed=100)\n", - "new_utils = pd.DataFrame(0.1 * -np.log(-np.log(rng_.uniform(0,1,raw_utilities.shape))), columns=raw_utilities.columns)\n", - "new_utils.index = raw_utilities.index" - ] - }, - { - "cell_type": "code", - "execution_count": 767, - "id": "da48a178-82b2-44a5-b45a-e9de3bb5e5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T23:17:20.369675Z", - "iopub.status.busy": "2021-09-01T23:17:20.369424Z", - "iopub.status.idle": "2021-09-01T23:17:20.666506Z", - "shell.execute_reply": "2021-09-01T23:17:20.665626Z", - "shell.execute_reply.started": "2021-09-01T23:17:20.369637Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
1372487210.0566420.0373050.0427800.0295760.0861540.0265860.0883130.1003310.0280210.0185160.0643910.0200090.0263990.0226220.0230070.0217540.0871260.0211710.0364750.0276500.135171
1372487250.1019020.0228760.0312270.0699900.0569130.0466850.0884110.0877850.0254140.0381620.0239880.0288000.0393180.0201600.0483640.0230030.0339590.0345970.0377080.0995230.041213
2113882010.0375620.0700330.0559510.0403890.0200410.0959740.0946780.0943680.0360140.0250510.0348600.0349530.0354990.0280400.0436500.0247130.0313880.0259310.0669990.0526780.051228
2113882050.0354800.0562560.0439440.0480850.0420780.0622210.1068190.0872650.0351170.0303990.0351540.0264690.0506460.0357270.0242840.0329050.0394270.0410920.0547140.0484590.063458
8063884010.0549180.0404880.0331490.0542940.0434770.0679820.1524650.0741520.0334040.0287350.0592690.0243390.0369680.0277600.0235220.0609320.0234450.0239310.0446430.0337730.058353
8063884050.0327640.0573010.0184370.0770990.0469630.0635210.0949510.0890720.0308260.0378960.0357520.0198780.0495360.0796100.0269220.0248330.0248400.0266950.0799360.0401460.043022
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n", - "trip_id \n", - "137248721 0.056642 0.037305 0.042780 0.029576 \n", - "137248725 0.101902 0.022876 0.031227 0.069990 \n", - "211388201 0.037562 0.070033 0.055951 0.040389 \n", - "211388205 0.035480 0.056256 0.043944 0.048085 \n", - "806388401 0.054918 0.040488 0.033149 0.054294 \n", - "806388405 0.032764 0.057301 0.018437 0.077099 \n", - "\n", - " SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\n", - "trip_id \n", - "137248721 0.086154 0.026586 0.088313 0.100331 0.028021 0.018516 \n", - "137248725 0.056913 0.046685 0.088411 0.087785 0.025414 0.038162 \n", - "211388201 0.020041 0.095974 0.094678 0.094368 0.036014 0.025051 \n", - "211388205 0.042078 0.062221 0.106819 0.087265 0.035117 0.030399 \n", - "806388401 0.043477 0.067982 0.152465 0.074152 0.033404 0.028735 \n", - "806388405 0.046963 0.063521 0.094951 0.089072 0.030826 0.037896 \n", - "\n", - " WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\n", - "trip_id \n", - "137248721 0.064391 0.020009 0.026399 0.022622 0.023007 0.021754 \n", - "137248725 0.023988 0.028800 0.039318 0.020160 0.048364 0.023003 \n", - "211388201 0.034860 0.034953 0.035499 0.028040 0.043650 0.024713 \n", - "211388205 0.035154 0.026469 0.050646 0.035727 0.024284 0.032905 \n", - "806388401 0.059269 0.024339 0.036968 0.027760 0.023522 0.060932 \n", - "806388405 0.035752 0.019878 0.049536 0.079610 0.026922 0.024833 \n", - "\n", - " DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n", - "trip_id \n", - "137248721 0.087126 0.021171 0.036475 0.027650 0.135171 \n", - "137248725 0.033959 0.034597 0.037708 0.099523 0.041213 \n", - "211388201 0.031388 0.025931 0.066999 0.052678 0.051228 \n", - "211388205 0.039427 0.041092 0.054714 0.048459 0.063458 \n", - "806388401 0.023445 0.023931 0.044643 0.033773 0.058353 \n", - "806388405 0.024840 0.026695 0.079936 0.040146 0.043022 " - ] - }, - "execution_count": 767, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nested_exp_utilities_cf = simulate.compute_nested_exp_utilities(new_utils, nest_spec)\n", - "nested_probabilities_cf = simulate.compute_nested_probabilities(nested_exp_utilities_cf, nest_spec, trace_label=None)\n", - "base_probabilities_cf = simulate.compute_base_probabilities(nested_probabilities_cf, nest_spec, spec)\n", - "base_probabilities_cf" - ] - }, - { - "cell_type": "code", - "execution_count": 799, - "id": "50dfd1ae-10c3-475c-94be-de783c2fa5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:15:29.056902Z", - "iopub.status.busy": "2021-09-02T00:15:29.056683Z", - "iopub.status.idle": "2021-09-02T00:27:21.965206Z", - "shell.execute_reply": "2021-09-02T00:27:21.964340Z", - "shell.execute_reply.started": "2021-09-02T00:15:29.056877Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 11min 49s, sys: 766 ms, total: 11min 50s\n", - "Wall time: 11min 52s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "num_samples = 10000 # 7.5s per 100\n", - "\n", - "data_zenith = []\n", - "data_asim = []\n", - "for i in range(num_samples):\n", - " seed = rng_.integers(0, 100000)\n", - " x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True, raw_utilities=new_utils)\n", - " y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False, raw_utilities=new_utils)\n", - " data_zenith.append(x_)\n", - " data_asim.append(y_)\n", - " \n", - "data_asim = pd.concat(data_asim, axis=1)\n", - "data_zenith = pd.concat(data_zenith, axis=1)\n", - "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", - "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_asim = counts_asim / data_asim.shape[1]\n", - "\n", - "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", - "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", - "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", - "# assert np.allclose(mode_share_zenith.sum(), 1)\n", - "# assert np.allclose(mode_share_asim.sum(), 1)\n", - "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", - "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 796, - "id": "bb16f9d7-44f5-4316-b981-d65216a5e217", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:14:11.772592Z", - "iopub.status.busy": "2021-09-02T00:14:11.772347Z", - "iopub.status.idle": "2021-09-02T00:14:12.001077Z", - "shell.execute_reply": "2021-09-02T00:14:12.000166Z", - "shell.execute_reply.started": "2021-09-02T00:14:11.772556Z" - } - }, - "outputs": [], - "source": [ - "#temp_z = data_zenith.copy()\n", - "#temp_a = data_asim.copy()\n", - "#data_asim = data_asim.join(temp_a, lsuffix=\"_o\", rsuffix=\"_n\")\n", - "#data_zenith = data_zenith.join(temp_z, lsuffix=\"_o\", rsuffix=\"_n\")\n", - "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", - "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_asim = counts_asim / data_asim.shape[1]\n", - "\n", - "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", - "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", - "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", - "# assert np.allclose(mode_share_zenith.sum(), 1)\n", - "# assert np.allclose(mode_share_asim.sum(), 1)\n", - "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", - "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 805, - "id": "031d8340-0af0-4608-8a83-ed2af4aaa3b2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:41:01.948669Z", - "iopub.status.busy": "2021-09-02T00:41:01.948422Z", - "iopub.status.idle": "2021-09-02T00:41:02.175972Z", - "shell.execute_reply": "2021-09-02T00:41:02.174759Z", - "shell.execute_reply.started": "2021-09-02T00:41:01.948633Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
zenith8.0555.1674.6313.5023.6064.2663.9633.4983.8955.1764.7995.8514.9996.1214.6689.4864.2284.1252.9773.6713.317
asim6.0966.6984.9773.2023.5224.3344.1293.1843.2245.1496.2838.2964.7936.5844.5327.4844.1524.2732.6293.4333.023
probs8.8835.3214.7382.8903.1364.0033.5653.1623.7585.3244.9276.0495.3416.5415.03710.4273.9734.2242.5743.1472.979
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP DRIVE_HVY \\\n", - "zenith 8.055 5.167 4.631 3.502 3.606 4.266 \n", - "asim 6.096 6.698 4.977 3.202 3.522 4.334 \n", - "probs 8.883 5.321 4.738 2.890 3.136 4.003 \n", - "\n", - " DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY SHARED3FREE \\\n", - "zenith 3.963 3.498 3.895 5.176 4.799 \n", - "asim 4.129 3.184 3.224 5.149 6.283 \n", - "probs 3.565 3.162 3.758 5.324 4.927 \n", - "\n", - " SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK WALK_COM WALK_EXP \\\n", - "zenith 5.851 4.999 6.121 4.668 9.486 4.228 4.125 \n", - "asim 8.296 4.793 6.584 4.532 7.484 4.152 4.273 \n", - "probs 6.049 5.341 6.541 5.037 10.427 3.973 4.224 \n", - "\n", - " WALK_HVY WALK_LOC WALK_LRF \n", - "zenith 2.977 3.671 3.317 \n", - "asim 2.629 3.433 3.023 \n", - "probs 2.574 3.147 2.979 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# sample of 20000\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * mode_share_comp).T)" - ] - }, - { - "cell_type": "code", - "execution_count": 806, - "id": "aece448f-4055-4d2a-b90d-8530caec9cb9", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:43:32.971972Z", - "iopub.status.busy": "2021-09-02T00:43:32.971684Z", - "iopub.status.idle": "2021-09-02T00:43:33.223345Z", - "shell.execute_reply": "2021-09-02T00:43:33.222488Z", - "shell.execute_reply.started": "2021-09-02T00:43:32.971932Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
trip_id137248721_z137248721_a137248721
BIKE0.092700.054350.100331
DRIVEALONEFREE0.053650.050050.056642
DRIVEALONEPAY0.038800.031550.037305
DRIVE_COM0.028700.024700.021171
DRIVE_EXP0.028800.026050.021754
DRIVE_HVY0.078150.099700.087126
DRIVE_LOC0.028750.027000.022622
DRIVE_LRF0.027800.024450.023007
SHARED2FREE0.043250.023000.042780
SHARED2PAY0.033600.015850.029576
SHARED3FREE0.073650.119200.086154
SHARED3PAY0.032600.039600.026586
TAXI0.035100.050300.036475
TNC_SHARED0.126200.181150.135171
TNC_SINGLE0.026250.038450.027650
WALK0.079400.048250.088313
WALK_COM0.031850.024600.026399
WALK_EXP0.057050.058950.064391
WALK_HVY0.025200.018400.020009
WALK_LOC0.034550.027400.028021
WALK_LRF0.023950.017000.018516
\n", - "
" - ], - "text/plain": [ - "trip_id 137248721_z 137248721_a 137248721\n", - "BIKE 0.09270 0.05435 0.100331\n", - "DRIVEALONEFREE 0.05365 0.05005 0.056642\n", - "DRIVEALONEPAY 0.03880 0.03155 0.037305\n", - "DRIVE_COM 0.02870 0.02470 0.021171\n", - "DRIVE_EXP 0.02880 0.02605 0.021754\n", - "DRIVE_HVY 0.07815 0.09970 0.087126\n", - "DRIVE_LOC 0.02875 0.02700 0.022622\n", - "DRIVE_LRF 0.02780 0.02445 0.023007\n", - "SHARED2FREE 0.04325 0.02300 0.042780\n", - "SHARED2PAY 0.03360 0.01585 0.029576\n", - "SHARED3FREE 0.07365 0.11920 0.086154\n", - "SHARED3PAY 0.03260 0.03960 0.026586\n", - "TAXI 0.03510 0.05030 0.036475\n", - "TNC_SHARED 0.12620 0.18115 0.135171\n", - "TNC_SINGLE 0.02625 0.03845 0.027650\n", - "WALK 0.07940 0.04825 0.088313\n", - "WALK_COM 0.03185 0.02460 0.026399\n", - "WALK_EXP 0.05705 0.05895 0.064391\n", - "WALK_HVY 0.02520 0.01840 0.020009\n", - "WALK_LOC 0.03455 0.02740 0.028021\n", - "WALK_LRF 0.02395 0.01700 0.018516" - ] - }, - "execution_count": 806, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# TODO: look at probs per trip, i.e. do not sum across trips\n", - "trip_id = 137248721\n", - "\n", - "counts_zenith.loc[counts_zenith.index == trip_id].T.merge(\n", - " counts_asim.loc[counts_asim.index == trip_id].T, suffixes=['_z', '_a'], left_index=True, right_index=True, how='outer').merge(\n", - " base_probabilities_cf.loc[base_probabilities_cf.index == trip_id].T, suffixes=['', '_probs'], left_index=True, right_index=True, how='outer').fillna(0)" - ] - }, { "cell_type": "code", "execution_count": null, From a31d97f94e9698a37a032b463884f77e3d5cceac Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 5 May 2022 14:08:14 +1000 Subject: [PATCH 009/135] probs trip set up --- notebooks/validate_frozen_impl.ipynb | 243 ++++----------------------- 1 file changed, 31 insertions(+), 212 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index dbc4ea7990..90240ad13f 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -434,67 +434,7 @@ }, { "cell_type": "code", - "execution_count": 70, - "outputs": [ - { - "data": { - "text/plain": "array([ 8684833, 8684837, 8685009, 8685013, 8753057,\n 8753061, 8753233, 8753237, 8805121, 8805125,\n 8805126, 9094329, 9094333, 9717265, 9717269,\n 35302033, 35302037, 36757193, 36757197, 36757257,\n 36757261, 86627409, 86627413, 86627625, 86627629,\n 86673657, 86673658, 86673659, 86673660, 86673661,\n 106170305, 106170309, 106289041, 106289045, 106289089,\n 106289090, 106289091, 106289092, 106289093, 106289369,\n 106289373, 106570377, 106570381, 106570382, 106570383,\n 106570465, 106570469, 106741681, 106741682, 106741685,\n 106741686, 106741687, 106741688, 106741961, 106741965,\n 106804593, 106804597, 106804598, 106804657, 106804661,\n 106804662, 106804663, 106804985, 106804989, 126610345,\n 126610349, 126610561, 126610565, 126610566, 126610673,\n 126610677, 126610873, 126610877, 126628233, 126628237,\n 126628585, 126628589, 137248721, 137248725, 137248873,\n 137248877, 137248961, 137248965, 137249225, 137249229,\n 195385257, 195385261, 211327433, 211327437, 211327673,\n 211327677, 211327761, 211327765, 211328025, 211328029,\n 211328030, 211388201, 211388205, 211388329, 211388333,\n 211388353, 211388357, 211388441, 211388445, 211388721,\n 211388725, 211389033, 211389037, 414508737, 414508741,\n 415214745, 415214749, 415556913, 415556917, 444793569,\n 444793573, 444793574, 444853217, 444853221, 444853265,\n 444853269, 444928617, 444928621, 467399281, 467399285,\n 468119569, 468119573, 468119857, 468119861, 468119921,\n 468119922, 468119925, 468119926, 468119927, 468119928,\n 484173817, 484173821, 484173905, 484173909, 484174185,\n 484174189, 515832417, 515832418, 515832419, 515832421,\n 515832422, 515832423, 515921265, 515921266, 515921269,\n 515921305, 515921309, 535170409, 535170413, 535170689,\n 535170693, 535170694, 535377657, 535377661, 535388201,\n 535388202, 535388205, 535388481, 535388485, 535620049,\n 535620053, 535672921, 535672925, 535694417, 535694421,\n 535694457, 535694461, 535694465, 535694466, 535694469,\n 581959169, 581959170, 581959171, 581959172, 581959173,\n 581959174, 581959561, 581959565, 615236801, 615236805,\n 615236865, 615236866, 615236869, 615350289, 615350293,\n 615350553, 615350557, 615350558, 615350617, 615350621,\n 642446345, 642446349, 642446633, 642446637, 708171009,\n 708171013, 708171014, 708171273, 708171277, 708171601,\n 708171605, 728159001, 728159005, 728159049, 728159053,\n 728159329, 728159333, 728159569, 728159573, 728159593,\n 728159597, 806388153, 806388157, 806388225, 806388229,\n 806388401, 806388405, 806388905, 806388906, 806388909,\n 806389233, 806389237, 841877257, 841877261, 841877849,\n 841877853, 841878177, 841878181, 841878505, 841878509,\n 841878510, 943749401, 943749405, 943749465, 943749469,\n 943749470, 943749471, 943750385, 943750389, 963188601,\n 963188605, 963188665, 963188669, 963285425, 963285426,\n 963285427, 963285429, 963285433, 963285437, 963285438,\n 963285439, 963286193, 963286197, 963286345, 963286346,\n 963286349, 969957321, 969957325, 1004301497, 1004301501,\n 1004301761, 1004301765, 1004301785, 1004301789, 1004301873,\n 1004301877, 1045822217, 1045822221, 1045822409, 1045822410,\n 1045822413, 1045822737, 1045822741, 1045823001, 1045823005,\n 1045823006, 1045823007, 1045823065, 1045823069, 1045823393,\n 1045823394, 1045823397, 1045823398, 1060575737, 1060575741,\n 1060575801, 1060575805, 1060575809, 1060575813, 1060575849,\n 1060575853, 1091733225, 1091733229, 1091770617, 1091770621,\n 1091770681, 1091770685, 1091770897, 1091770898, 1091770901,\n 1146269081, 1146269085, 1146269345, 1146269349, 1146269369,\n 1146269373, 1146269374, 1146269375, 1146269376, 1146386553,\n 1146386557, 1146386561, 1146386565, 1146386881, 1146386885,\n 1146472377, 1146472381, 1146472489, 1146472493, 1146472537,\n 1146472538, 1146472539, 1146472540, 1146472541, 1146472817,\n 1146472821, 1179607441, 1179607445, 1179607657, 1179607661,\n 1179607662, 1179607663, 1179608009, 1179608013, 1276281769,\n 1276281773, 1276282049, 1276282053, 1276282313, 1276282317,\n 1276282321, 1276282325, 1276282361, 1276282365, 1368289969,\n 1368289973, 1368290273, 1368290277, 1368290689, 1368290690,\n 1368290693, 1368291297, 1368291301, 1368291302, 1368291609,\n 1368291613, 1368292281, 1368292285, 1368292377, 1368292381,\n 1368292382, 1368292657, 1368292658, 1368292661, 1368292662,\n 1582205617, 1582205621, 1582205641, 1582205645, 1582205729,\n 1582205733, 1658748793, 1658748797, 1658806913, 1658806917,\n 1658807089, 1658807093, 1658807129, 1658807133, 1658807153,\n 1658807157, 1658807177, 1658807181, 1658947233, 1658947234,\n 1658947237, 1658947238, 1766872681, 1766872685, 1766872897,\n 1766872901, 1766872921, 1766872925, 1767013721, 1767013725,\n 1767013726, 1767013785, 1767013789, 1767182857, 1767182861,\n 1767182865, 1767182869, 1767182945, 1767182949, 1767183169,\n 1767183173, 1767183297, 1767183301, 1767186249, 1767186253,\n 1767186577, 1767186578, 1767186581, 1767186582, 1767666161,\n 1767666165, 1767666233, 1767666237, 1767666441, 1767666445,\n 1767666769, 1767666773, 1768236897, 1768236901, 1768237161,\n 1768237165, 1768237166, 1768237167, 1768237489, 1768237493,\n 2396217321, 2396217325, 2396217433, 2396217437, 2396217761,\n 2396217765, 2396217769, 2396217773, 2444719729, 2444719733,\n 2463663417, 2463663421, 2463971785, 2463971789, 2463971790,\n 2463972065, 2463972069, 2463972070, 2463972071, 2464104641,\n 2464104645, 2464104857, 2464104861, 2464104881, 2464104885,\n 2464131801, 2464131805, 2464406009, 2464406013, 2464443401,\n 2464443405, 2464446025, 2464446029, 2464449633, 2464449634,\n 2464449637, 2467713777, 2467713781, 2467756745, 2467756749,\n 2472945097, 2472945101, 2472945113, 2472945117, 2472945118,\n 2473024473, 2473024477])" - }, - "execution_count": 70, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "pipeline._PIPELINE.rng().add_channel(\"trips\", )" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 59, - "outputs": [ - { - "data": { - "text/plain": "" - }, - "execution_count": 59, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pipeline._PIPELINE.rng().Sim" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 65, + "execution_count": 83, "outputs": [], "source": [ "def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples):\n", @@ -571,7 +511,7 @@ "\n", " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", "\n", - " choices_list = []\n", + " #choices_list = []\n", "\n", "\n", " # grab one, duplicate num_samples times\n", @@ -582,13 +522,11 @@ " # need to add new row_states for rng here, need to ensure there are no collisions with existing keys\n", " existing_indexes = pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values\n", " num_new_indexes = trips_segment.shape[0]\n", - " now find indexes - random draw and then remove existing and do again? simple loop for each, or batch and do\n", - " recursively? simple loop is fast enough here\n", - "\n", + " new_indexes = np.arange(existing_indexes.max()+1, existing_indexes.max() + num_new_indexes + 1)\n", "\n", - " trips_segment.index += np.arange(num_samples)\n", + " trips_segment.index = new_indexes #+= np.arange(num_samples)\n", " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", + " trips_segment.index.name = 'trip_id'\n", "\n", "\n", " # #logger.warning(\"Change seeding back when done with testing\")\n", @@ -646,17 +584,17 @@ " alts = spec.columns\n", " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", " ################\n", - " choices_list.append(choices)\n", - " choices_df_asim = pd.concat(choices_list)\n", + " #choices_list.append(choices)\n", + " #choices_df_asim = pd.concat(choices_list)\n", "\n", " # update trips table with choices (and potionally logssums)\n", - " trips_df = trips.to_frame()\n", + " #trips_df = trips_merged.copy() # trips.to_frame()\n", "\n", " #if (do_these_purposes is not None):\n", " # trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", "\n", - " assign_in_place(trips_df, choices_df_asim)\n", - " assert not trips_df[mode_column_name].isnull().any()\n", + " #assign_in_place(trips_df, choices)\n", + " #assert not trips_df[mode_column_name].isnull().any()\n", "\n", " finalise = True\n", " if finalise:\n", @@ -670,7 +608,7 @@ " pipeline.close_pipeline()\n", "\n", " print(\"Done\")\n", - " return trips_df" + " return trips_merged, choices" ], "metadata": { "collapsed": false, @@ -681,10 +619,10 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 84, "outputs": [], "source": [ - "#pipeline.close_pipeline()" + "pipeline.close_pipeline()" ], "metadata": { "collapsed": false, @@ -695,7 +633,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "outputs": [ { "name": "stderr", @@ -709,152 +647,33 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running with %d trips 482\n" - ] - }, - { - "ename": "AssertionError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mAssertionError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/tmp/ipykernel_30125/1730006119.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 2\u001B[0m \u001B[0mnum_samples\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m100\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[0mtrip_id_to_check\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m642446345\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m \u001B[0mt_\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmode_choice_for_trip\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mchoose_individual_max_utility\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mchoose_individual_max_utility\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrip_id_to_check\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mtrip_id_to_check\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnum_samples\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mnum_samples\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", - "\u001B[0;32m/tmp/ipykernel_30125/3946923567.py\u001B[0m in \u001B[0;36mmode_choice_for_trip\u001B[0;34m(choose_individual_max_utility, trip_id_to_check, num_samples)\u001B[0m\n\u001B[1;32m 89\u001B[0m \u001B[0;31m# pipeline._PIPELINE.rng.row_states[\"row_seed\"] = trips_segment.index.values\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 90\u001B[0m \u001B[0;31m# pipeline._PIPELINE.rng.row_states[\"offset\"] = 0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 91\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_PIPELINE\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrng\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0madd_channel\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"trips\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrips_segment\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 92\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 93\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/random.py\u001B[0m in \u001B[0;36madd_channel\u001B[0;34m(self, channel_name, domain_df)\u001B[0m\n\u001B[1;32m 467\u001B[0m \u001B[0mchannel\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchannels\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mchannel_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 468\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 469\u001B[0;31m \u001B[0mchannel\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mextend_domain\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdomain_df\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 470\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 471\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/random.py\u001B[0m in \u001B[0;36mextend_domain\u001B[0;34m(self, domain_df)\u001B[0m\n\u001B[1;32m 142\u001B[0m \u001B[0;31m# row_states already exists, so we are extending\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 143\u001B[0m \u001B[0;31m# if extending, these should be new rows, no intersection with existing row_states\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 144\u001B[0;31m \u001B[0;32massert\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrow_states\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mintersection\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdomain_df\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 145\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrow_states\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconcat\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrow_states\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrow_states\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 146\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mAssertionError\u001B[0m: " + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'social' (1000000 trips)\n" ] } ], "source": [ "choose_individual_max_utility = False\n", - "num_samples = 100\n", - "trip_id_to_check = 642446345\n", - "t_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, trip_id_to_check=trip_id_to_check, num_samples=num_samples)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 18, - "outputs": [], - "source": [ - "#pipeline.registered_tables()\n", - "#list(map(lambda x: x['checkpoint_name'], pipeline._PIPELINE.checkpoints))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 24, - "outputs": [], - "source": [ - "num_samples = 100\n", + "num_samples = 1000000\n", "trip_id_to_check = 642446345\n", - "\n", - "trips_dup = trips_merged.loc[trips_merged.index == trip_id_to_check].copy()\n", - "trips_dup = trips_dup.loc[trips_dup.index.repeat(num_samples)]\n", - "trips_dup.index += np.arange(num_samples)" + "t_, c_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility,\n", + " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", + "\n", + "sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", + "#sim_mode_shares.columns = [\"mode_share_sim\"]\n", + "obs_probs = base_probs[0].loc[base_probs[0].index == trip_id_to_check].T\n", + "obs_probs.columns = [\"mode_share_obs\"]\n", + "ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how=\"outer\").fillna(0)\n", + "ms_comp[\"diff\"] = ms_comp[\"trip_mode\"] - ms_comp[\"mode_share_obs\"]\n", + "ms_comp[\"rel_diff\"] = ms_comp[\"diff\"] / ms_comp[\"mode_share_obs\"]\n", + "#ms_comp.style.format('{:.2}')\n", + "ms_comp.loc[(ms_comp.mode_share_obs != 0) & (ms_comp.trip_mode != 0), [\"mode_share_obs\", \"trip_mode\"]].T" ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" + "name": "#%%\n", + "is_executing": true } } }, From ab8a212977ba886a0a7041f425a3c35580b911b7 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 5 May 2022 14:11:42 +1000 Subject: [PATCH 010/135] probs validation clean up --- notebooks/validate_frozen_impl.ipynb | 840 ++------------------------- 1 file changed, 44 insertions(+), 796 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 90240ad13f..95127ec5c1 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -633,7 +633,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 138, "outputs": [ { "name": "stderr", @@ -650,6 +650,41 @@ "Running with %d trips 482\n", "trip_mode_choice tour_type 'social' (1000000 trips)\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n" + ] + }, + { + "data": { + "text/plain": " SHARED2FREE TAXI TNC_SHARED TNC_SINGLE WALK \\\nmode_share_obs 0.004883 0.000096 0.000287 0.000131 0.044905 \ntrip_mode 0.004885 0.000095 0.000291 0.000125 0.044628 \n\n WALK_LOC WALK_LRF \nmode_share_obs 0.059962 0.889736 \ntrip_mode 0.060309 0.889667 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048830.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0048850.0000950.0002910.0001250.0446280.0603090.889667
\n
" + }, + "execution_count": 138, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -669,19 +704,6 @@ "#ms_comp.style.format('{:.2}')\n", "ms_comp.loc[(ms_comp.mode_share_obs != 0) & (ms_comp.trip_mode != 0), [\"mode_share_obs\", \"trip_mode\"]].T" ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": true - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], "metadata": { "collapsed": false, "pycharm": { @@ -689,803 +711,29 @@ } } }, - { - "cell_type": "markdown", - "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:55:59.542669Z", - "iopub.status.busy": "2021-09-01T04:55:59.542360Z", - "iopub.status.idle": "2021-09-01T04:55:59.771722Z", - "shell.execute_reply": "2021-09-01T04:55:59.770801Z", - "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" - }, - "tags": [], - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# OLD" - ] - }, { "cell_type": "code", - "execution_count": 454, - "id": "74eeedd3-4ade-4729-8170-79fa9dcf11f8", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:58:52.825042Z", - "iopub.status.busy": "2021-09-01T04:58:52.824768Z", - "iopub.status.idle": "2021-09-01T04:58:53.042884Z", - "shell.execute_reply": "2021-09-01T04:58:53.042061Z", - "shell.execute_reply.started": "2021-09-01T04:58:52.825004Z" - } - }, - "outputs": [], - "source": [ - "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - " nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec,\n", - " trace_label=trace_label)\n", - " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", - " return base_probabilities\n", - "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)\n", - "\n", - "\n", - "## caculate probabilities with Asim methodology, should be correct\n", - "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", - " full_probs = [] # analytical probs\n", - "\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " #choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " # segment_trace_label, log_alt_losers=log_alt_losers, \n", - " # trace_column_names=trace_column_names, seed=seed)\n", - " probs = simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, \n", - " estimator, segment_trace_label, log_alt_losers, \n", - " trace_column_names)\n", - " full_probs.append(probs)\n", - " probs_df = pd.concat(full_probs)\n", - " return probs_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": 455, - "id": "295a4fd1-8e85-47e2-af82-9f1beac31d63", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:59:46.126555Z", - "iopub.status.busy": "2021-09-01T04:59:46.126262Z", - "iopub.status.idle": "2021-09-01T04:59:52.979908Z", - "shell.execute_reply": "2021-09-01T04:59:52.979043Z", - "shell.execute_reply.started": "2021-09-01T04:59:46.126522Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2021-09-01 14:59:46.353007 Start\n", - "2021-09-01 14:59:52.976736 End\n" - ] - } - ], - "source": [ - "print(f\"{datetime.now()} Start\")\n", - "pipeline.open_pipeline(resume_after)\n", - "inject.get_injectable('preload_injectables', None)\n", - "model_name = \"trip_mode_choice\"\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "\n", - "step_name = model_name\n", - "args = {}\n", - "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "chunk_size = 0\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "trips_df = trips.to_frame()\n", - "#print(\"Running with %d trips\", trips_df.shape[0])\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - "orig_col = 'origin'\n", - "dest_col = 'destination'\n", - "\n", - "constants = {}\n", - "constants.update(config.get_model_constants(model_settings))\n", - "constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - "})\n", - "\n", - "skim_dict = network_los.get_default_skim_dict()\n", - "\n", - "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - "skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - "}\n", - "\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "\n", - "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - "\n", - "all_choices = []\n", - "for i in range(100):\n", - " if i % 10 == 0:\n", - " print(f\"{datetime.now()} iteration {i}\")\n", - " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=None)\n", - " all_choices.append(choices_df)\n", - "all_choices = pd.concat(all_choices, axis=1)\n", - "\n", - "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", - " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", - " log_alt_losers=None, trace_column_names=None)\n", - "\n", - "# update trips table with choices (and potionally logssums)\n", - "#trips_df = trips.to_frame()\n", - "#\n", - "#assign_in_place(trips_df, choices_df)\n", - "#assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - "print(f\"{datetime.now()} End\")" - ] - }, - { - "cell_type": "code", - "execution_count": 463, - "id": "dc87e904-16d1-44d4-81b7-cafbe4e223c3", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:01:40.901982Z", - "iopub.status.busy": "2021-09-01T05:01:40.901759Z", - "iopub.status.idle": "2021-09-01T05:01:41.337758Z", - "shell.execute_reply": "2021-09-01T05:01:41.336757Z", - "shell.execute_reply.started": "2021-09-01T05:01:40.901946Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#all_choices.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", - "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts = val_counts / all_choices.shape[1]\n", - "#val_counts = val_counts.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", - "#val_counts['prob_of_asim_choice'] = val_counts.apply(lambda x: x[x.asim], axis=1) # this is what our simulation says w.r.t. to asim choice\n", - "# for 100% and many samples should mostly agree" - ] - }, - { - "cell_type": "code", - "execution_count": 464, - "id": "09bf151c-a39a-4823-acfd-9c8ae747c338", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:01:46.119117Z", - "iopub.status.busy": "2021-09-01T05:01:46.118849Z", - "iopub.status.idle": "2021-09-01T05:01:46.354114Z", - "shell.execute_reply": "2021-09-01T05:01:46.353248Z", - "shell.execute_reply.started": "2021-09-01T05:01:46.119092Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#val_counts['prob_of_asim_choice'].hist(bins=100);" - ] - }, - { - "cell_type": "code", - "execution_count": 490, - "id": "f3b6ade9-d875-4104-b438-b53ef42e342c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:10:08.653227Z", - "iopub.status.busy": "2021-09-01T05:10:08.652978Z", - "iopub.status.idle": "2021-09-01T05:10:08.915620Z", - "shell.execute_reply": "2021-09-01T05:10:08.914513Z", - "shell.execute_reply.started": "2021-09-01T05:10:08.653189Z" - }, - "tags": [] - }, + "execution_count": 140, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", - "analytical 3.152 0.852 0.0 0.0 0.0 \n", - "experiment 3.243 0.880 0.0 0.0 0.0 \n", - "\n", - " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", - "analytical 0.0 0.0 0.0 0.685 0.0 \n", - "experiment 0.0 0.0 0.0 0.772 0.0 \n", - "\n", - " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK \\\n", - "analytical 0.129 0.0 0.182 0.260 1.334 63.708 \n", - "experiment 0.156 0.0 0.174 0.259 1.434 63.243 \n", - "\n", - " WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", - "analytical 0.0 0.0 0.455 18.355 10.887 \n", - "experiment 0.0 0.0 0.394 18.639 10.807 " - ] + "text/plain": " mode_share_obs trip_mode diff rel_diff\nBIKE 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVEALONEFREE 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVEALONEPAY 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_COM 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_EXP 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_HVY 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_LOC 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_LRF 0.000e+00 0.000e+00 0.000e+00 NaN\nSHARED2FREE 4.883e-03 4.885e-03 2.162e-06 4.428e-04\nSHARED2PAY 0.000e+00 0.000e+00 0.000e+00 NaN\nSHARED3FREE 4.701e-14 0.000e+00 -4.701e-14 -1.000e+00\nSHARED3PAY 0.000e+00 0.000e+00 0.000e+00 NaN\nTAXI 9.611e-05 9.500e-05 -1.108e-06 -1.152e-02\nTNC_SHARED 2.868e-04 2.910e-04 4.245e-06 1.480e-02\nTNC_SINGLE 1.308e-04 1.250e-04 -5.755e-06 -4.401e-02\nWALK 4.490e-02 4.463e-02 -2.769e-04 -6.166e-03\nWALK_COM 0.000e+00 0.000e+00 0.000e+00 NaN\nWALK_EXP 0.000e+00 0.000e+00 0.000e+00 NaN\nWALK_HVY 0.000e+00 0.000e+00 0.000e+00 NaN\nWALK_LOC 5.996e-02 6.031e-02 3.467e-04 5.782e-03\nWALK_LRF 8.897e-01 8.897e-01 -6.933e-05 -7.792e-05", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
mode_share_obstrip_modediffrel_diff
BIKE0.000e+000.000e+000.000e+00NaN
DRIVEALONEFREE0.000e+000.000e+000.000e+00NaN
DRIVEALONEPAY0.000e+000.000e+000.000e+00NaN
DRIVE_COM0.000e+000.000e+000.000e+00NaN
DRIVE_EXP0.000e+000.000e+000.000e+00NaN
DRIVE_HVY0.000e+000.000e+000.000e+00NaN
DRIVE_LOC0.000e+000.000e+000.000e+00NaN
DRIVE_LRF0.000e+000.000e+000.000e+00NaN
SHARED2FREE4.883e-034.885e-032.162e-064.428e-04
SHARED2PAY0.000e+000.000e+000.000e+00NaN
SHARED3FREE4.701e-140.000e+00-4.701e-14-1.000e+00
SHARED3PAY0.000e+000.000e+000.000e+00NaN
TAXI9.611e-059.500e-05-1.108e-06-1.152e-02
TNC_SHARED2.868e-042.910e-044.245e-061.480e-02
TNC_SINGLE1.308e-041.250e-04-5.755e-06-4.401e-02
WALK4.490e-024.463e-02-2.769e-04-6.166e-03
WALK_COM0.000e+000.000e+000.000e+00NaN
WALK_EXP0.000e+000.000e+000.000e+00NaN
WALK_HVY0.000e+000.000e+000.000e+00NaN
WALK_LOC5.996e-026.031e-023.467e-045.782e-03
WALK_LRF8.897e-018.897e-01-6.933e-05-7.792e-05
\n
" }, "metadata": {}, "output_type": "display_data" } ], "source": [ - "mode_share_analytical = (probs_nl.sum(axis=0) / probs_nl.shape[0]).to_frame('analytical')\n", - "assert np.allclose(mode_share_analytical.sum(), 1)\n", - "mode_share_rum = (val_counts.sum(axis=0) / val_counts.shape[0]).to_frame('experiment')\n", - "assert np.allclose(mode_share_rum.sum(), 1)\n", - "full_share = mode_share_analytical.join(mode_share_rum, how='outer').fillna(0)\n", "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * full_share).T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e48a56e-d613-4dfd-bef6-62bee524f12f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50914e1d-750d-440c-9617-f96bd2a46c56", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2d70e0e-520e-49a6-8024-ae31345f6ead", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 631, - "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:32.250808Z", - "iopub.status.busy": "2021-09-01T11:37:32.250548Z", - "iopub.status.idle": "2021-09-01T20:20:35.257463Z", - "shell.execute_reply": "2021-09-01T20:20:35.255964Z", - "shell.execute_reply.started": "2021-09-01T11:37:32.250782Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "num_samples = 10\n", - "\n", - "rng_test = default_rng(23423)\n", - "\n", - "print(f\"{datetime.now()} Start\")\n", - "pipeline.open_pipeline(resume_after)\n", - "inject.get_injectable('preload_injectables', None)\n", - "model_name = \"trip_mode_choice\"\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "\n", - "step_name = model_name\n", - "args = {}\n", - "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "chunk_size = 0\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "trips_df = trips.to_frame()\n", - "#print(\"Running with %d trips\", trips_df.shape[0])\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - "orig_col = 'origin'\n", - "dest_col = 'destination'\n", - "\n", - "constants = {}\n", - "constants.update(config.get_model_constants(model_settings))\n", - "constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - "})\n", - "\n", - "skim_dict = network_los.get_default_skim_dict()\n", - "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - "}\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - "all_choices_zenith = []\n", - "all_choices = []\n", - "for i in range(num_samples):\n", - " \n", - " seed = rng_test.integers(0, 100000) #int(9.3 * (i+1)**3) # why not\n", - " \n", - " if i % 50 == 0:\n", - " print(f\"{datetime.now()} iteration {i}\")\n", - " choices_df_zenith = stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", - " all_choices_zenith.append(choices_df_zenith)\n", - "\n", - " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", - " all_choices.append(choices_df)\n", - " \n", - " t_ = choices_df_zenith.merge(choices_df, left_index=True, right_index=True, suffixes=['_zenith', '_asim'])\n", - " diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", - " \n", - " #print(f\"seed {seed} leads to {diffs.shape[0]} differences. tripids {diffs.index}\")\n", - "\n", - "all_choices_zenith = pd.concat(all_choices_zenith, axis=1)\n", - "all_choices = pd.concat(all_choices, axis=1)\n", - "\n", - "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", - " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", - " log_alt_losers=None, trace_column_names=None)\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - "print(f\"{datetime.now()} End\")" - ] - }, - { - "cell_type": "code", - "execution_count": 643, - "id": "3abd7949-dc9c-469b-845e-26c17741f70e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:33:21.533110Z", - "iopub.status.busy": "2021-09-01T20:33:21.531915Z", - "iopub.status.idle": "2021-09-01T20:33:21.740946Z", - "shell.execute_reply": "2021-09-01T20:33:21.739713Z", - "shell.execute_reply.started": "2021-09-01T20:33:21.533074Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Have 2038 samples\n" - ] - } + " display(ms_comp)" ], - "source": [ - "print(f\"Have {all_choices_zenith.shape[1]} samples\")" - ] - }, - { - "cell_type": "code", - "execution_count": 633, - "id": "8762bd6d-ac04-458c-869f-be86d8297351", "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:21:27.778803Z", - "iopub.status.busy": "2021-09-01T20:21:27.778531Z", - "iopub.status.idle": "2021-09-01T20:21:28.675645Z", - "shell.execute_reply": "2021-09-01T20:21:28.674734Z", - "shell.execute_reply.started": "2021-09-01T20:21:27.778754Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "val_counts_zenith = all_choices_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts_zenith = val_counts_zenith / all_choices_zenith.shape[1]\n", - "\n", - "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts = val_counts / all_choices.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 634, - "id": "0f566105-9424-457b-86f5-d60c1d63aac5", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:21:30.954894Z", - "iopub.status.busy": "2021-09-01T20:21:30.954654Z", - "iopub.status.idle": "2021-09-01T20:21:31.179689Z", - "shell.execute_reply": "2021-09-01T20:21:31.178304Z", - "shell.execute_reply.started": "2021-09-01T20:21:30.954867Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
experiment_zenith3.1500.8740.00.00.00.00.00.00.6800.00.1360.00.1540.2361.28563.7670.00.00.51518.82410.379
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", - "analytical 3.152 0.852 0.0 0.0 0.0 \n", - "experiment 3.243 0.880 0.0 0.0 0.0 \n", - "experiment_zenith 3.150 0.874 0.0 0.0 0.0 \n", - "\n", - " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", - "analytical 0.0 0.0 0.0 0.685 0.0 \n", - "experiment 0.0 0.0 0.0 0.772 0.0 \n", - "experiment_zenith 0.0 0.0 0.0 0.680 0.0 \n", - "\n", - " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE \\\n", - "analytical 0.129 0.0 0.182 0.260 1.334 \n", - "experiment 0.156 0.0 0.174 0.259 1.434 \n", - "experiment_zenith 0.136 0.0 0.154 0.236 1.285 \n", - "\n", - " WALK WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", - "analytical 63.708 0.0 0.0 0.455 18.355 10.887 \n", - "experiment 63.243 0.0 0.0 0.394 18.639 10.807 \n", - "experiment_zenith 63.767 0.0 0.0 0.515 18.824 10.379 " - ] - }, - "metadata": {}, - "output_type": "display_data" + "collapsed": false, + "pycharm": { + "name": "#%%\n" } - ], - "source": [ - "mode_share_rum_zenith = (val_counts_zenith.sum(axis=0) / val_counts_zenith.shape[0]).to_frame('experiment_zenith')\n", - "assert np.allclose(mode_share_rum_zenith.sum(), 1)\n", - "full_share_incl_zenith = full_share.merge(mode_share_rum_zenith, left_index=True, right_index=True, how='outer').fillna(0)\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * full_share_incl_zenith).T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ef84f10-d616-40e4-87e5-425499027ff3", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "431cee49-40a9-4dc7-93a5-75308fb302dd", - "metadata": {}, - "outputs": [], - "source": [] + } }, { "cell_type": "code", From fd5ca14771973a4929d4116be0c752f98e7b2214 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 5 May 2022 21:18:50 +1000 Subject: [PATCH 011/135] fixes scale of nest utilities --- activitysim/core/logit.py | 4 +- activitysim/core/simulate.py | 18 +- notebooks/validate_frozen_impl.ipynb | 625 +++++++++++++++++++++++---- 3 files changed, 552 insertions(+), 95 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index a2b732c52a..8f432e5c1e 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -475,8 +475,8 @@ def _each_nest(spec, parent_nest, post_order): nest.coefficient = parent_nest.coefficient # this is used for frozen individual ru - see discussion in simulate.py l.740 - nest.parent_nest_scale = parent_nest.product_of_coefficients - #nest.parent_nest_scale = parent_nest.coefficient + #nest.parent_nest_scale = parent_nest.product_of_coefficients + nest.parent_nest_scale = parent_nest.coefficient yield spec, nest diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 27ed38e5fd..289ba90d92 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -736,18 +736,20 @@ def compute_nested_utilities(raw_utilities, nest_spec): for nest in logit.each_nest(nest_spec, post_order=True): name = nest.name if nest.is_leaf: - nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients - #nested_utilities[name] = raw_utilities[name].astype(float) / nest.coefficient - # WHY does asim use product of coeffs here? the above gives us identical root logsums - # but then we need to dscale ev1 accordingly for alternatives, right? - # can be achieved by + #nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients + nested_utilities[name] = raw_utilities[name].astype(float) / nest.coefficient + # WHY does asim use product of coeffs here? using that gives us identical root logsums + # but then we need to scale ev1 accordingly for alternatives, right? else: # the alternative nested_utilities will already have been computed due to post_order - # this will RuntimeWarning: divide by zero encountered in log - # if all nest alternative utilities are zero and produce -inf + if nest.level == 1: # FIXME: work out the parent scale mess + scale = 1.0 + else: + scale = nest.parent_nest_scale + with np.errstate(divide='ignore'): nested_utilities[name] = \ - nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) + nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / scale).sum(axis=1)) return nested_utilities diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 95127ec5c1..f68019d396 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -170,7 +170,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 188, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -219,8 +219,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Done\n", - "Running with %d trips 482\n" + "Done\n" ] }, { @@ -234,16 +233,306 @@ "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'atwork' (27 trips)\n", - "trip_mode_choice tour_type 'eatout' (33 trips)\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", - "trip_mode_choice tour_type 'othmaint' (46 trips)\n", - "trip_mode_choice tour_type 'school' (37 trips)\n", - "trip_mode_choice tour_type 'shopping' (77 trips)\n", - "trip_mode_choice tour_type 'social' (19 trips)\n", - "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'eatout' (33 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'escort' (6 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'othmaint' (46 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'school' (37 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'shopping' (77 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'social' (19 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'univ' (26 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'work' (168 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "Done\n" ] } @@ -256,14 +545,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 189, "outputs": [ { "data": { - "text/plain": " trip_mode_fru trip_mode_asim\nWALK 299 304\nWALK_LOC 92 85\nWALK_LRF 61 53\nBIKE 17 17\nDRIVEALONEFREE 6 4\nSHARED2FREE 3 4\nTNC_SINGLE 2 4\nWALK_HVY 2 4", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK299304
WALK_LOC9285
WALK_LRF6153
BIKE1717
DRIVEALONEFREE64
SHARED2FREE34
TNC_SINGLE24
WALK_HVY24
\n
" + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 299 304\nWALK_LOC 94 85\nWALK_LRF 57 53\nBIKE 17 17\nDRIVEALONEFREE 6 4\nWALK_HVY 4 4\nSHARED2FREE 3 4\nTNC_SINGLE 2 4", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK299304
WALK_LOC9485
WALK_LRF5753
BIKE1717
DRIVEALONEFREE64
WALK_HVY44
SHARED2FREE34
TNC_SINGLE24
\n
" }, - "execution_count": 9, + "execution_count": 189, "metadata": {}, "output_type": "execute_result" } @@ -283,7 +572,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 190, "outputs": [ { "name": "stdout", @@ -308,21 +597,21 @@ }, { "cell_type": "code", - "execution_count": 10, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" + "execution_count": 187, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n8805121 -5709.149390 -8563.435105 -inf -2856.962423 \n8805125 -5710.236852 -8564.522566 -inf -2857.805670 \n8805126 -5709.506304 -8563.792018 -inf -2857.185402 \n468119921 -2856.413143 -5710.698857 -inf -2855.880864 \n468119922 -2856.976497 -5711.262211 -inf -2856.444428 \n468119925 -2870.186517 -5724.472231 -inf -2863.993439 \n468119926 -2857.935513 -5712.221227 -inf -2856.918910 \n468119927 -2859.140866 -5713.426580 -inf -2857.537407 \n468119928 -2869.283122 -5723.568836 -inf -2863.287255 \n642446345 -5709.613741 -8563.899455 -inf -10.654154 \n642446349 -5711.108042 -8565.393757 -inf -11.507365 \n1179608009 -8563.419303 -11417.705017 -inf -17.616599 \n1179608013 -8571.196804 -11425.482519 -inf -22.061100 \n1658807153 -5709.137752 -8563.423466 -inf -19.719413 \n1658807157 -5710.005638 -8564.291352 -inf -20.220421 \n1767182945 -5709.423410 -8563.709124 -inf -10.469202 \n1767182949 -5709.668094 -8563.953808 -inf -10.618407 \n2464104881 -5709.489682 -8563.775396 -inf -12.625725 \n2464104885 -5709.489682 -8563.775396 -inf -12.625725 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n8805121 -5711.248137 -inf -2856.962379 -5711.248093 -inf \n8805125 -5712.091384 -inf -2857.707940 -5711.993654 -inf \n8805126 -5711.471116 -inf -2857.131784 -5711.417498 -inf \n468119921 -5710.166578 -inf -2855.667952 -5709.953666 -inf \n468119922 -5710.730142 -inf -2856.231600 -5710.517314 -inf \n468119925 -5718.279154 -inf -2861.516208 -5715.801922 -inf \n468119926 -5711.204625 -inf -2856.512269 -5710.797984 -inf \n468119927 -5711.823121 -inf -2856.896023 -5711.181738 -inf \n468119928 -5717.572969 -inf -2860.888909 -5715.174623 -inf \n642446345 -2864.939868 -5.179103 -62.836260 -2917.121974 -30.545404 \n642446349 -2865.793079 -5.593858 -63.433034 -2917.718749 -30.835503 \n1179608009 -2871.902313 -8.563624 -19.761289 -2874.047003 -9.606182 \n1179608013 -2876.346814 -10.724146 -22.872589 -2877.158304 -11.118620 \n1658807153 -2874.005127 -9.585826 -21.864019 -2876.149734 -10.628343 \n1658807157 -2874.506135 -9.829371 -22.218276 -2876.503991 -10.800551 \n1767182945 -2864.754916 -5.089195 -62.653459 -2916.939174 -30.456543 \n1767182949 -2864.904122 -5.161726 -62.764474 -2917.050188 -30.510508 \n2464104881 -2866.911439 -6.137505 -64.806540 -2919.092254 -31.503179 \n2464104885 -2866.911439 -6.137505 -64.806540 -2919.092254 -31.503179 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n8805121 -inf 12.460532 -1375.039468 8.971583 -1980.409272 \n8805125 -inf 12.557754 -1374.990857 9.041583 -3978.191234 \n8805126 -inf 12.207754 -1375.165857 8.789583 -1980.441864 \n468119921 -inf 10.647096 -1376.595265 7.665909 -1984.028282 \n468119922 -inf -0.158141 -1387.385919 -0.113862 -2003.585155 \n468119925 -inf -0.158141 -1387.385919 -0.113862 -2003.585155 \n468119926 -inf 10.734596 -1376.551515 7.728909 -1987.731834 \n468119927 -inf 10.765788 -1376.855740 7.751367 -1987.818125 \n468119928 -inf 11.707060 -1375.408912 8.429083 -1985.936364 \n642446345 -3.728954 -2.097408 -1387.343797 -1.510134 1.093980 \n642446349 -4.027578 -2.146019 -1387.353519 -1.545134 1.192207 \n1179608009 -5.948375 4.880186 -1378.113147 3.513734 -1984.050332 \n1179608013 -7.350415 4.880186 -1378.113147 3.513734 -1984.050332 \n1658807153 -6.684352 6.456402 -1376.357071 4.648609 15.268960 \n1658807157 -6.845959 6.407790 -1376.381376 4.613609 14.530278 \n1767182945 -3.664221 -3.606195 -1388.969251 -2.596460 -0.555751 \n1767182949 -3.716443 -3.776334 -1389.003278 -2.718960 -0.579012 \n2464104881 -4.419004 8.134630 -1378.288147 5.856934 10.502444 \n2464104885 -4.419004 8.134630 -1378.288147 5.856934 10.666689 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n8805121 -3978.409272 -3978.409272 -3978.409272 -3978.409272 -inf \n8805125 -3978.191234 -3978.191234 -3978.191234 -3978.191234 -inf \n8805126 -3978.441864 -3978.791864 -3978.441864 -3978.441864 -inf \n468119921 -3982.028282 -3982.028282 -3982.028282 -3982.028282 -inf \n468119922 -4001.585155 -4001.935155 -4001.585155 -4001.585155 -inf \n468119925 -4001.585155 -4001.585155 -4001.585155 -4001.585155 -inf \n468119926 -3985.731834 -3986.081834 -3985.731834 -3985.731834 -inf \n468119927 -3985.818125 -3986.168125 -3985.818125 -3985.818125 -inf \n468119928 -3983.936364 -3984.286364 -3983.936364 -3983.936364 -inf \n642446345 3.035970 -1995.576420 -1995.597276 -1995.880220 2.140923 \n642446349 3.035270 -1994.736869 -1995.597276 -1995.040669 2.145065 \n1179608009 -1984.050332 -1984.050332 -1984.050332 -1984.050332 -inf \n1179608013 -1984.050332 -1984.050332 -1984.050332 -1984.050332 -inf \n1658807153 -1982.731040 -1982.731040 -1982.731040 -1982.731040 10.603444 \n1658807157 -1983.469722 -1983.469722 -1983.469722 -1983.469722 10.090471 \n1767182945 0.665917 -1997.106222 -1997.397020 -1997.410022 0.546586 \n1767182949 0.665917 -1997.106222 -1997.408920 -1997.410022 0.544118 \n2464104881 -1986.419156 -1986.167956 -1986.416956 -1986.471756 7.293364 \n2464104885 -1986.254911 -1986.003711 -1986.252711 -1986.307511 7.407423 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n8805121 -5976.701052 -5979.008301 -5979.008301 -5979.622112 -5980.000155 \n8805125 -5976.250034 -5976.250034 -5976.250034 -5976.250034 -5976.250034 \n8805126 -5976.561059 -5980.066360 -5980.066360 -5977.489912 -5981.139291 \n468119921 -1987.917275 -3986.634667 -3986.634667 -3988.159653 -3989.540253 \n468119922 -1999.781073 -4002.350822 -4002.350822 -4003.314328 -4005.292790 \n468119925 -1999.856900 -4003.785211 -4003.785211 -4001.364128 -4006.015621 \n468119926 -1987.922393 -3991.085771 -3991.085771 -3985.679774 -3991.883814 \n468119927 -1984.330401 -3988.035744 -3988.035744 -3985.888572 -3992.028453 \n468119928 -1986.084720 -3989.542402 -3989.542402 -3984.015305 -3990.339045 \n642446345 -5993.059741 -5996.236057 -5996.236057 -5996.847988 -5998.040211 \n642446349 -5993.114899 -5996.745065 -5996.745065 -5994.750788 -5998.886171 \n1179608009 -7978.109132 -7978.109132 -7978.109132 -7978.109132 -7978.109132 \n1179608013 -7978.109132 -7978.109132 -7978.109132 -7978.109132 -7978.109132 \n1658807153 -5978.797090 -5981.645926 -5981.645926 -5982.436880 -5983.088374 \n1658807157 -5979.569178 -5984.301635 -5984.301635 -5980.452579 -5984.874558 \n1767182945 -5994.656309 -5997.269370 -5997.269370 -5997.769890 -5998.629581 \n1767182949 -5994.664265 -5997.497589 -5997.497589 -5995.576790 -5998.540328 \n2464104881 -5983.483693 -5989.023319 -5989.023319 -5991.399546 -5994.064540 \n2464104885 -5983.469343 -5991.354620 -5991.354620 -5989.123846 -5997.222704 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED \\\ntrip_id \n8805121 -inf -inf -20.081973 -19.473153 -20.804324 \n8805125 -inf -inf -20.047538 -19.456673 -20.779823 \n8805126 -inf -inf -20.169318 -19.513014 -20.864116 \n468119921 -inf -inf -26.859498 -30.543501 -25.552660 \n468119922 -inf -inf -26.820071 -30.516140 -25.512174 \n468119925 -inf -inf -26.824219 -30.517112 -25.513632 \n468119926 -inf -inf -26.316039 -30.492180 -25.480679 \n468119927 -inf -inf -24.418379 -30.418607 -25.374051 \n468119928 -inf -inf -27.800562 -30.563790 -25.697316 \n642446345 -inf 1.541464 -18.289778 -17.981922 -17.196617 \n642446349 -inf 1.544447 -18.324954 -17.976575 -17.192002 \n1179608009 -inf -inf -8.526813 0.047647 -2.273228 \n1179608013 -inf -inf -8.527679 0.047161 -2.273957 \n1658807153 -inf 7.634480 -8.591632 0.206340 -2.227894 \n1658807157 -inf 7.265139 -8.647127 0.198076 -2.240290 \n1767182945 -inf 0.393542 -14.395026 -13.588217 -14.958046 \n1767182949 -inf 0.391765 -14.435998 -13.608147 -14.995707 \n2464104881 -inf 5.251222 -24.155944 -33.809528 -23.204042 \n2464104885 -inf 5.333344 -24.155944 -33.809528 -23.204042 \n\n RIDEHAIL root \ntrip_id \n8805121 -6.797103 8.971583 \n8805125 -6.788793 9.041583 \n8805126 -6.817566 8.789583 \n468119921 -9.110799 7.665909 \n468119922 -9.096330 -0.113736 \n468119925 -9.097058 -0.113736 \n468119926 -9.041678 7.728909 \n468119927 -8.672837 7.751368 \n468119928 -9.207112 8.429083 \n642446345 -5.980956 1.593075 \n642446349 -5.981865 1.593106 \n1179608009 0.050933 3.544670 \n1179608013 0.050750 3.544608 \n1658807153 0.104584 7.684253 \n1658807157 0.101487 7.334029 \n1767182945 -4.700641 0.464659 \n1767182949 -4.710741 0.456698 \n2464104881 -8.235935 6.292424 \n2464104885 -8.235935 6.322192 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
8805121-5709.149390-8563.435105-inf-2856.962423-5711.248137-inf-2856.962379-5711.248093-inf-inf12.460532-1375.0394688.971583-1980.409272-3978.409272-3978.409272-3978.409272-3978.409272-inf-5976.701052-5979.008301-5979.008301-5979.622112-5980.000155-inf-inf-20.081973-19.473153-20.804324-6.7971038.971583
8805125-5710.236852-8564.522566-inf-2857.805670-5712.091384-inf-2857.707940-5711.993654-inf-inf12.557754-1374.9908579.041583-3978.191234-3978.191234-3978.191234-3978.191234-3978.191234-inf-5976.250034-5976.250034-5976.250034-5976.250034-5976.250034-inf-inf-20.047538-19.456673-20.779823-6.7887939.041583
8805126-5709.506304-8563.792018-inf-2857.185402-5711.471116-inf-2857.131784-5711.417498-inf-inf12.207754-1375.1658578.789583-1980.441864-3978.441864-3978.791864-3978.441864-3978.441864-inf-5976.561059-5980.066360-5980.066360-5977.489912-5981.139291-inf-inf-20.169318-19.513014-20.864116-6.8175668.789583
468119921-2856.413143-5710.698857-inf-2855.880864-5710.166578-inf-2855.667952-5709.953666-inf-inf10.647096-1376.5952657.665909-1984.028282-3982.028282-3982.028282-3982.028282-3982.028282-inf-1987.917275-3986.634667-3986.634667-3988.159653-3989.540253-inf-inf-26.859498-30.543501-25.552660-9.1107997.665909
468119922-2856.976497-5711.262211-inf-2856.444428-5710.730142-inf-2856.231600-5710.517314-inf-inf-0.158141-1387.385919-0.113862-2003.585155-4001.585155-4001.935155-4001.585155-4001.585155-inf-1999.781073-4002.350822-4002.350822-4003.314328-4005.292790-inf-inf-26.820071-30.516140-25.512174-9.096330-0.113736
468119925-2870.186517-5724.472231-inf-2863.993439-5718.279154-inf-2861.516208-5715.801922-inf-inf-0.158141-1387.385919-0.113862-2003.585155-4001.585155-4001.585155-4001.585155-4001.585155-inf-1999.856900-4003.785211-4003.785211-4001.364128-4006.015621-inf-inf-26.824219-30.517112-25.513632-9.097058-0.113736
468119926-2857.935513-5712.221227-inf-2856.918910-5711.204625-inf-2856.512269-5710.797984-inf-inf10.734596-1376.5515157.728909-1987.731834-3985.731834-3986.081834-3985.731834-3985.731834-inf-1987.922393-3991.085771-3991.085771-3985.679774-3991.883814-inf-inf-26.316039-30.492180-25.480679-9.0416787.728909
468119927-2859.140866-5713.426580-inf-2857.537407-5711.823121-inf-2856.896023-5711.181738-inf-inf10.765788-1376.8557407.751367-1987.818125-3985.818125-3986.168125-3985.818125-3985.818125-inf-1984.330401-3988.035744-3988.035744-3985.888572-3992.028453-inf-inf-24.418379-30.418607-25.374051-8.6728377.751368
468119928-2869.283122-5723.568836-inf-2863.287255-5717.572969-inf-2860.888909-5715.174623-inf-inf11.707060-1375.4089128.429083-1985.936364-3983.936364-3984.286364-3983.936364-3983.936364-inf-1986.084720-3989.542402-3989.542402-3984.015305-3990.339045-inf-inf-27.800562-30.563790-25.697316-9.2071128.429083
642446345-5709.613741-8563.899455-inf-10.654154-2864.939868-5.179103-62.836260-2917.121974-30.545404-3.728954-2.097408-1387.343797-1.5101341.0939803.035970-1995.576420-1995.597276-1995.8802202.140923-5993.059741-5996.236057-5996.236057-5996.847988-5998.040211-inf1.541464-18.289778-17.981922-17.196617-5.9809561.593075
642446349-5711.108042-8565.393757-inf-11.507365-2865.793079-5.593858-63.433034-2917.718749-30.835503-4.027578-2.146019-1387.353519-1.5451341.1922073.035270-1994.736869-1995.597276-1995.0406692.145065-5993.114899-5996.745065-5996.745065-5994.750788-5998.886171-inf1.544447-18.324954-17.976575-17.192002-5.9818651.593106
1179608009-8563.419303-11417.705017-inf-17.616599-2871.902313-8.563624-19.761289-2874.047003-9.606182-5.9483754.880186-1378.1131473.513734-1984.050332-1984.050332-1984.050332-1984.050332-1984.050332-inf-7978.109132-7978.109132-7978.109132-7978.109132-7978.109132-inf-inf-8.5268130.047647-2.2732280.0509333.544670
1179608013-8571.196804-11425.482519-inf-22.061100-2876.346814-10.724146-22.872589-2877.158304-11.118620-7.3504154.880186-1378.1131473.513734-1984.050332-1984.050332-1984.050332-1984.050332-1984.050332-inf-7978.109132-7978.109132-7978.109132-7978.109132-7978.109132-inf-inf-8.5276790.047161-2.2739570.0507503.544608
1658807153-5709.137752-8563.423466-inf-19.719413-2874.005127-9.585826-21.864019-2876.149734-10.628343-6.6843526.456402-1376.3570714.64860915.268960-1982.731040-1982.731040-1982.731040-1982.73104010.603444-5978.797090-5981.645926-5981.645926-5982.436880-5983.088374-inf7.634480-8.5916320.206340-2.2278940.1045847.684253
1658807157-5710.005638-8564.291352-inf-20.220421-2874.506135-9.829371-22.218276-2876.503991-10.800551-6.8459596.407790-1376.3813764.61360914.530278-1983.469722-1983.469722-1983.469722-1983.46972210.090471-5979.569178-5984.301635-5984.301635-5980.452579-5984.874558-inf7.265139-8.6471270.198076-2.2402900.1014877.334029
1767182945-5709.423410-8563.709124-inf-10.469202-2864.754916-5.089195-62.653459-2916.939174-30.456543-3.664221-3.606195-1388.969251-2.596460-0.5557510.665917-1997.106222-1997.397020-1997.4100220.546586-5994.656309-5997.269370-5997.269370-5997.769890-5998.629581-inf0.393542-14.395026-13.588217-14.958046-4.7006410.464659
1767182949-5709.668094-8563.953808-inf-10.618407-2864.904122-5.161726-62.764474-2917.050188-30.510508-3.716443-3.776334-1389.003278-2.718960-0.5790120.665917-1997.106222-1997.408920-1997.4100220.544118-5994.664265-5997.497589-5997.497589-5995.576790-5998.540328-inf0.391765-14.435998-13.608147-14.995707-4.7107410.456698
2464104881-5709.489682-8563.775396-inf-12.625725-2866.911439-6.137505-64.806540-2919.092254-31.503179-4.4190048.134630-1378.2881475.85693410.502444-1986.419156-1986.167956-1986.416956-1986.4717567.293364-5983.483693-5989.023319-5989.023319-5991.399546-5994.064540-inf5.251222-24.155944-33.809528-23.204042-8.2359356.292424
2464104885-5709.489682-8563.775396-inf-12.625725-2866.911439-6.137505-64.806540-2919.092254-31.503179-4.4190048.134630-1378.2881475.85693410.666689-1986.254911-1986.003711-1986.252711-1986.3075117.407423-5983.469343-5991.354620-5991.354620-5989.123846-5997.222704-inf5.333344-24.155944-33.809528-23.204042-8.2359356.322192
\n
" + }, + "execution_count": 187, + "metadata": {}, + "output_type": "execute_result" } - } - }, - { - "cell_type": "code", - "execution_count": 10, - "outputs": [], - "source": [], + ], + "source": [ + "simulate.compute_nested_utilities(ru[0], ns[0])" + ], "metadata": { "collapsed": false, "pycharm": { @@ -349,7 +638,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 192, "outputs": [ { "name": "stderr", @@ -364,7 +653,36 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'social' (19 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", + "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "Done\n" ] } @@ -382,14 +700,140 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 194, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] parent nest scale: 0\n", + "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] parent nest scale: 1.0\n", + "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] parent nest scale: 0.72\n", + "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2'] parent nest scale: 0.72\n", + "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3'] parent nest scale: 0.72\n", + "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] parent nest scale: 1.0\n", + "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] parent nest scale: 1.0\n", + "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS'] parent nest scale: 0.72\n", + "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS'] parent nest scale: 0.72\n", + "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] parent nest scale: 1.0\n" + ] + } + ], + "source": [ + "nest_spec = ns[0]\n", + "for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", + " nest.print()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 195, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "leaf DRIVEALONEFREE with 0.252, 0.35\n", + "leaf DRIVEALONEPAY with 0.252, 0.35\n", + "node DRIVEALONE with 0.35, 0.72\n", + "leaf SHARED2FREE with 0.252, 0.35\n", + "leaf SHARED2PAY with 0.252, 0.35\n", + "node SHAREDRIDE2 with 0.35, 0.72\n", + "leaf SHARED3FREE with 0.252, 0.35\n", + "leaf SHARED3PAY with 0.252, 0.35\n", + "node SHAREDRIDE3 with 0.35, 0.72\n", + "node AUTO with 0.72, 1.0\n", + "leaf WALK with 0.72, 0.72\n", + "leaf BIKE with 0.72, 0.72\n", + "node NONMOTORIZED with 0.72, 1.0\n", + "leaf WALK_LOC with 0.36, 0.5\n", + "leaf WALK_LRF with 0.36, 0.5\n", + "leaf WALK_EXP with 0.36, 0.5\n", + "leaf WALK_HVY with 0.36, 0.5\n", + "leaf WALK_COM with 0.36, 0.5\n", + "node WALKACCESS with 0.5, 0.72\n", + "leaf DRIVE_LOC with 0.36, 0.5\n", + "leaf DRIVE_LRF with 0.36, 0.5\n", + "leaf DRIVE_EXP with 0.36, 0.5\n", + "leaf DRIVE_HVY with 0.36, 0.5\n", + "leaf DRIVE_COM with 0.36, 0.5\n", + "node DRIVEACCESS with 0.5, 0.72\n", + "node TRANSIT with 0.72, 1.0\n", + "leaf TAXI with 0.36, 0.36\n", + "leaf TNC_SINGLE with 0.36, 0.36\n", + "leaf TNC_SHARED with 0.36, 0.36\n", + "node RIDEHAIL with 0.36, 1.0\n", + "node root with 1.0, 0\n" + ] + } + ], + "source": [ + "for nest in logit.each_nest(nest_spec, post_order=True):\n", + " name = nest.name\n", + " if nest.is_leaf:\n", + " print(f\"leaf {name} with {nest.product_of_coefficients}, {nest.parent_nest_scale}\")\n", + " else:\n", + " print(f\"node {name} with {nest.coefficient}, {nest.parent_nest_scale}\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 196, "outputs": [ { "data": { "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" }, - "execution_count": 12, + "execution_count": 196, "metadata": {}, "output_type": "execute_result" } @@ -434,7 +878,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 197, "outputs": [], "source": [ "def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples):\n", @@ -608,21 +1052,25 @@ " pipeline.close_pipeline()\n", "\n", " print(\"Done\")\n", - " return trips_merged, choices" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 84, - "outputs": [], - "source": [ - "pipeline.close_pipeline()" + " return trips_merged, choices\n", + "\n", + "\n", + "def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check):\n", + " t_, c_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility,\n", + " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", + "\n", + " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", + " #sim_mode_shares.columns = [\"mode_share_sim\"]\n", + " obs_probs = base_probs[0].loc[base_probs[0].index == trip_id_to_check].T\n", + " obs_probs.columns = [\"mode_share_obs\"]\n", + " ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how=\"outer\").fillna(0)\n", + " ms_comp[\"diff\"] = ms_comp[\"trip_mode\"] - ms_comp[\"mode_share_obs\"]\n", + " ms_comp[\"rel_diff\"] = ms_comp[\"diff\"] / ms_comp[\"mode_share_obs\"]\n", + " #ms_comp.style.format('{:.2}')\n", + " #with pd.option_context(\"precision\", 3):\n", + " # display(ms_comp)\n", + " display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", + " return ms_comp" ], "metadata": { "collapsed": false, @@ -633,7 +1081,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 198, "outputs": [ { "name": "stderr", @@ -648,7 +1096,7 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (1000000 trips)\n" + "trip_mode_choice tour_type 'social' (100000 trips)\n" ] }, { @@ -679,30 +1127,18 @@ }, { "data": { - "text/plain": " SHARED2FREE TAXI TNC_SHARED TNC_SINGLE WALK \\\nmode_share_obs 0.004883 0.000096 0.000287 0.000131 0.044905 \ntrip_mode 0.004885 0.000095 0.000291 0.000125 0.044628 \n\n WALK_LOC WALK_LRF \nmode_share_obs 0.059962 0.889736 \ntrip_mode 0.060309 0.889667 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048830.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0048850.0000950.0002910.0001250.0446280.0603090.889667
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.005210 0.000000e+00 0.000130 0.000280 0.000100 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.045410 0.061630 0.887240 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0052100.000000e+000.0001300.0002800.0001000.0454100.0616300.887240
\n
" }, - "execution_count": 138, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ "choose_individual_max_utility = False\n", - "num_samples = 1000000\n", + "num_samples = 100000\n", "trip_id_to_check = 642446345\n", - "t_, c_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility,\n", - " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", - "\n", - "sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", - "#sim_mode_shares.columns = [\"mode_share_sim\"]\n", - "obs_probs = base_probs[0].loc[base_probs[0].index == trip_id_to_check].T\n", - "obs_probs.columns = [\"mode_share_obs\"]\n", - "ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how=\"outer\").fillna(0)\n", - "ms_comp[\"diff\"] = ms_comp[\"trip_mode\"] - ms_comp[\"mode_share_obs\"]\n", - "ms_comp[\"rel_diff\"] = ms_comp[\"diff\"] / ms_comp[\"mode_share_obs\"]\n", - "#ms_comp.style.format('{:.2}')\n", - "ms_comp.loc[(ms_comp.mode_share_obs != 0) & (ms_comp.trip_mode != 0), [\"mode_share_obs\", \"trip_mode\"]].T" + "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" ], "metadata": { "collapsed": false, @@ -713,35 +1149,54 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 199, + "id": "1fcdc0ff-c510-4878-8672-8365e046442c", + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with %d trips 482\n", + "trip_mode_choice tour_type 'social' (100000 trips)\n", + "Done\n" + ] + }, { "data": { - "text/plain": " mode_share_obs trip_mode diff rel_diff\nBIKE 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVEALONEFREE 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVEALONEPAY 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_COM 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_EXP 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_HVY 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_LOC 0.000e+00 0.000e+00 0.000e+00 NaN\nDRIVE_LRF 0.000e+00 0.000e+00 0.000e+00 NaN\nSHARED2FREE 4.883e-03 4.885e-03 2.162e-06 4.428e-04\nSHARED2PAY 0.000e+00 0.000e+00 0.000e+00 NaN\nSHARED3FREE 4.701e-14 0.000e+00 -4.701e-14 -1.000e+00\nSHARED3PAY 0.000e+00 0.000e+00 0.000e+00 NaN\nTAXI 9.611e-05 9.500e-05 -1.108e-06 -1.152e-02\nTNC_SHARED 2.868e-04 2.910e-04 4.245e-06 1.480e-02\nTNC_SINGLE 1.308e-04 1.250e-04 -5.755e-06 -4.401e-02\nWALK 4.490e-02 4.463e-02 -2.769e-04 -6.166e-03\nWALK_COM 0.000e+00 0.000e+00 0.000e+00 NaN\nWALK_EXP 0.000e+00 0.000e+00 0.000e+00 NaN\nWALK_HVY 0.000e+00 0.000e+00 0.000e+00 NaN\nWALK_LOC 5.996e-02 6.031e-02 3.467e-04 5.782e-03\nWALK_LRF 8.897e-01 8.897e-01 -6.933e-05 -7.792e-05", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
mode_share_obstrip_modediffrel_diff
BIKE0.000e+000.000e+000.000e+00NaN
DRIVEALONEFREE0.000e+000.000e+000.000e+00NaN
DRIVEALONEPAY0.000e+000.000e+000.000e+00NaN
DRIVE_COM0.000e+000.000e+000.000e+00NaN
DRIVE_EXP0.000e+000.000e+000.000e+00NaN
DRIVE_HVY0.000e+000.000e+000.000e+00NaN
DRIVE_LOC0.000e+000.000e+000.000e+00NaN
DRIVE_LRF0.000e+000.000e+000.000e+00NaN
SHARED2FREE4.883e-034.885e-032.162e-064.428e-04
SHARED2PAY0.000e+000.000e+000.000e+00NaN
SHARED3FREE4.701e-140.000e+00-4.701e-14-1.000e+00
SHARED3PAY0.000e+000.000e+000.000e+00NaN
TAXI9.611e-059.500e-05-1.108e-06-1.152e-02
TNC_SHARED2.868e-042.910e-044.245e-061.480e-02
TNC_SINGLE1.308e-041.250e-04-5.755e-06-4.401e-02
WALK4.490e-024.463e-02-2.769e-04-6.166e-03
WALK_COM0.000e+000.000e+000.000e+00NaN
WALK_EXP0.000e+000.000e+000.000e+00NaN
WALK_HVY0.000e+000.000e+000.000e+00NaN
WALK_LOC5.996e-026.031e-023.467e-045.782e-03
WALK_LRF8.897e-018.897e-01-6.933e-05-7.792e-05
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.000000 0.000000e+00 0.000000 0.000000 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.002600 0.020660 0.976740 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0000000.000000e+000.0000000.0000000.0000000.0026000.0206600.976740
\n
" }, "metadata": {}, "output_type": "display_data" } ], "source": [ - "with pd.option_context(\"precision\", 3):\n", - " display(ms_comp)" - ], + "choose_individual_max_utility = True\n", + "num_samples = 100000 # 1e5 takes about 25s\n", + "trip_id_to_check = 642446345\n", + "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } } - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1fcdc0ff-c510-4878-8672-8365e046442c", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From f2cea72263cb23473544ebeedeb51c7be1060f2a Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 6 May 2022 11:59:01 +1000 Subject: [PATCH 012/135] two level still off --- activitysim/core/logit.py | 11 +- activitysim/core/simulate.py | 11 +- ...ested_logit_frozen_rand_individ_util.ipynb | 314 +++++++++--- notebooks/validate_frozen_impl.ipynb | 473 +++--------------- 4 files changed, 320 insertions(+), 489 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 8f432e5c1e..dd3c401eb2 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -198,9 +198,13 @@ def add_ev1_random(df, nest_spec): for n in each_nest(nest_spec): if n.level == 1: continue # skip the root level - # TODO: check parent nest level scale is what we want this is right + # TODO: check parent nest level scale is what we want uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice) - rands = inverse_ev1_cdf(uniform_rands, scale=n.parent_nest_scale) + #if n.is_leaf: + # scale = n.coefficient # product_of_coefficients parent_nest_scale coefficient + #else: + scale = 1.0 # n.coefficient + rands = inverse_ev1_cdf(uniform_rands, scale=scale) nest_utils_for_choice.loc[:, n.name] += rands[:, 0] # inverse_ev1_cdf of single-row df adds dimension return nest_utils_for_choice @@ -224,9 +228,6 @@ def is_alternative(name): assert not nest_utils_for_choice["choice"].isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(nest_utils_for_choice["choice"], index=nested_utilities.index) - - assert not choices.isnull().any(), "No choice for XXX - implement reporting" - return choices diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 289ba90d92..f3843bd0c7 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -737,15 +737,16 @@ def compute_nested_utilities(raw_utilities, nest_spec): name = nest.name if nest.is_leaf: #nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients - nested_utilities[name] = raw_utilities[name].astype(float) / nest.coefficient + nested_utilities[name] = raw_utilities[name].astype(float) / nest.coefficient # WHY does asim use product of coeffs here? using that gives us identical root logsums # but then we need to scale ev1 accordingly for alternatives, right? else: # the alternative nested_utilities will already have been computed due to post_order - if nest.level == 1: # FIXME: work out the parent scale mess - scale = 1.0 - else: - scale = nest.parent_nest_scale + #if nest.level == 1: # FIXME: work out the parent scale mess + # scale = 1.0 + #else: + # scale = nest.parent_nest_scale + scale = 1.0 # IT'S RECURSIVE for lowest level but check for higher level with np.errstate(divide='ignore'): nested_utilities[name] = \ diff --git a/notebooks/nested_logit_frozen_rand_individ_util.ipynb b/notebooks/nested_logit_frozen_rand_individ_util.ipynb index dede399ca8..4f06fca9ef 100644 --- a/notebooks/nested_logit_frozen_rand_individ_util.ipynb +++ b/notebooks/nested_logit_frozen_rand_individ_util.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 27, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:14.957470Z", @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 50, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:15.467608Z", @@ -116,14 +116,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "0.4452265282367507 0.5330453677531714 0.02172810401007798\n", + "0.4402110749048506 0.5234199037442896 0.036369021350859815\n", "1.0\n" ] } ], "source": [ "cost = 3.0\n", - "nest_scale = 0.5\n", + "nest_scale = 0.6\n", "\n", "util_3 = utility(cost, utility_spec, 3)\n", "util_2 = utility(cost, utility_spec, 2)\n", @@ -144,17 +144,9 @@ "print(sum([prob_1, prob_2, prob_3]))" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## The ActivitySim way\n", - "Kind of pointless here but this is how we choose a single value - draw from U and pick whichever interval it falls into" - ] - }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 32, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:20.523480Z", @@ -164,37 +156,25 @@ "shell.execute_reply.started": "2022-05-01T09:55:20.523453Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "closed form: [0.4452265282367507, 0.5330453677531714, 0.02172810401007798],\n", - "simulated: [0.4450967 0.5331544 0.0217489]\n", - "CPU times: user 750 ms, sys: 78.1 ms, total: 828 ms\n", - "Wall time: 811 ms\n" - ] - } - ], + "outputs": [], "source": [ - "%%time\n", - "\n", - "probs = [prob_1, prob_2, prob_3]\n", - "cum_probs = [0] + list(np.cumsum(probs))\n", - "\n", - "num_draws = 10000000\n", - "\n", - "# now draw from U and put into arrays, then value count?\n", - "rng = default_rng(999)\n", - "rands = rng.uniform(size=num_draws)\n", - "\n", - "hits, bins = np.histogram(rands, bins=cum_probs)\n", - "print(f\"closed form: {probs},\\nsimulated: {hits / num_draws}\")" + "# Asim does this:\n", + "# probs = [prob_1, prob_2, prob_3]\n", + "# cum_probs = [0] + list(np.cumsum(probs))\n", + "#\n", + "# num_draws = 10000000\n", + "#\n", + "# # now draw from U and put into arrays, then value count?\n", + "# rng = default_rng(999)\n", + "# rands = rng.uniform(size=num_draws)\n", + "#\n", + "# hits, bins = np.histogram(rands, bins=cum_probs)\n", + "# print(f\"closed form: {probs},\\nsimulated: {hits / num_draws}\")" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 30, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T09:55:24.027745Z", @@ -237,57 +217,177 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 52, "metadata": { "execution": { - "iopub.execute_input": "2022-05-01T10:02:14.035272Z", - "iopub.status.busy": "2022-05-01T10:02:14.035014Z", - "iopub.status.idle": "2022-05-01T10:02:14.040376Z", - "shell.execute_reply": "2022-05-01T10:02:14.039207Z", - "shell.execute_reply.started": "2022-05-01T10:02:14.035235Z" + "iopub.execute_input": "2022-05-01T10:03:23.999857Z", + "iopub.status.busy": "2022-05-01T10:03:23.999571Z", + "iopub.status.idle": "2022-05-01T10:03:30.033848Z", + "shell.execute_reply": "2022-05-01T10:03:30.032689Z", + "shell.execute_reply.started": "2022-05-01T10:03:23.999816Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "closed form probs: 0.440211, 0.523420, 0.036369\n", + " simulated probs: 0.4403354, 0.5233336, 0.036331\n", + "CPU times: user 3.88 s, sys: 2.52 s, total: 6.39 s\n", + "Wall time: 6.73 s\n" + ] + } + ], "source": [ + "%%time\n", + "\n", + "# conditional error term are given by logit with scale given by nest scale\n", + "num_draws_dec = 10000000\n", + "#mu = 1.0 / nest_scale\n", + "\n", "util_3 = utility(cost, utility_spec, 3)\n", "util_2 = utility(cost, utility_spec, 2)\n", "logsum_bus = logsum(np.array([util_2, util_3]), nest_scale=nest_scale)\n", - "util_1 = utility(cost, utility_spec, 1)" + "util_1 = utility(cost, utility_spec, 1)\n", + "\n", + "\n", + "rng_dec = default_rng(9)\n", + "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", + "ev1_lower = inverse_ev1_cdf(rands_dec)\n", + "\n", + "lower_utils_2 = util_2 / nest_scale + ev1_lower[num_draws_dec:]\n", + "lower_utils_3 = util_3 / nest_scale + ev1_lower[:num_draws_dec]\n", + "\n", + "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec))\n", + "nest_util = nest_scale * logsum_bus + ev1_upper\n", + "\n", + "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec))\n", + "full_util_1 = util_1 + ev1_upper\n", + "\n", + "choices = np.array([full_util_1, nest_util]).argmax(axis=0)\n", + "nest_indexes = np.nonzero(choices == 1)[0]\n", + "nest_choices = np.array([lower_utils_2[nest_indexes], lower_utils_3[nest_indexes]]).argmax(axis=0)\n", + "nest_choices += 1\n", + "choices = np.append(choices[choices == 0], nest_choices)\n", + "\n", + "vals, counts = np.unique(choices, return_counts=True)\n", + "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", + "\n", + "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", + "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, + "outputs": [], + "source": [], "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:03:23.999857Z", - "iopub.status.busy": "2022-05-01T10:03:23.999571Z", - "iopub.status.idle": "2022-05-01T10:03:30.033848Z", - "shell.execute_reply": "2022-05-01T10:03:30.032689Z", - "shell.execute_reply.started": "2022-05-01T10:03:23.999816Z" + "collapsed": false, + "pycharm": { + "name": "#%%\n" } - }, + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# This is the Zenith way as it was designed originally\n", + "\n", + "w/o scaling utilities in logsums, everything else is identical" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 47, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "closed form probs: 0.445227, 0.533045, 0.021728\n", - " simulated probs: 0.4452013, 0.5330709, 0.0217278\n" + "0.19304564457406473 0.6714008596782736 0.1355534957476616\n" + ] + } + ], + "source": [ + "cost = 3.0\n", + "nest_scale = 0.6\n", + "\n", + "util_3 = utility(cost, utility_spec, 3)\n", + "util_2 = utility(cost, utility_spec, 2)\n", + "logsum_bus = logsum(np.array([util_2, util_3])) #, nest_scale=nest_scale) # DIFF\n", + "nest_util = nest_scale * logsum_bus\n", + "\n", + "util_1 = utility(cost, utility_spec, 1)\n", + "prob_1 = np.exp(util_1) / (np.exp(util_1) + np.exp(nest_util))\n", + "\n", + "nest_cond_prob = np.exp(nest_util) / (np.exp(util_1) + np.exp(nest_util))\n", + "nest_marg_prob_2 = np.exp(util_2) / (np.exp(util_2) + np.exp(util_3)) # DIFF\n", + "nest_marg_prob_3 = np.exp(util_3) / (np.exp(util_2) + np.exp(util_3)) # DIFF\n", + "\n", + "prob_2 = nest_cond_prob * nest_marg_prob_2\n", + "prob_3 = nest_cond_prob * nest_marg_prob_3\n", + "\n", + "print(prob_1, prob_2, prob_3)\n", + "assert np.isclose(1.0, sum([prob_1, prob_2, prob_3]))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 48, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "closed form probs: 0.193046, 0.671401, 0.135553\n", + " simulated probs: 0.1930163, 0.6715585, 0.1354252\n" ] } ], "source": [ + "# test w/o scaling in utility? is this what previous zenith version does?\n", + "\n", "# conditional error term are given by logit with scale given by nest scale\n", "num_draws_dec = 10000000\n", - "#mu = 1.0 / nest_scale\n", + "\n", + "util_3 = utility(cost, utility_spec, 3)\n", + "util_2 = utility(cost, utility_spec, 2)\n", + "logsum_bus = logsum(np.array([util_2, util_3])) # , nest_scale=nest_scale) # THAT's the only diff right #\n", + "util_1 = utility(cost, utility_spec, 1)\n", + "\n", "\n", "rng_dec = default_rng(9)\n", "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", - "ev1_lower = inverse_ev1_cdf(rands_dec, scale=nest_scale)\n", + "ev1_lower = inverse_ev1_cdf(rands_dec) #, scale=nest_scale)\n", "\n", - "lower_utils_2 = util_2 + ev1_lower[num_draws_dec:] \n", - "lower_utils_3 = util_3 + ev1_lower[:num_draws_dec] \n", + "lower_utils_2 = util_2 + ev1_lower[num_draws_dec:]\n", + "lower_utils_3 = util_3 + ev1_lower[:num_draws_dec]\n", "\n", "#logsum_bus = logsum(np.array([lower_utils_2, lower_utils_3]), nest_scale=nest_scale)\n", "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=2*num_draws_dec))\n", @@ -306,7 +406,97 @@ "\n", "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" - ] + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } }, { "cell_type": "code", diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index f68019d396..bfb9b39faf 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -170,7 +170,7 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 8, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -234,305 +234,16 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'atwork' (27 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'eatout' (33 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'escort' (6 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'othmaint' (46 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'school' (37 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'shopping' (77 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'social' (19 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'univ' (26 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'work' (168 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "trip_mode_choice tour_type 'atwork' (27 trips)\n", + "trip_mode_choice tour_type 'eatout' (33 trips)\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n", + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", + "trip_mode_choice tour_type 'othmaint' (46 trips)\n", + "trip_mode_choice tour_type 'school' (37 trips)\n", + "trip_mode_choice tour_type 'shopping' (77 trips)\n", + "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'univ' (26 trips)\n", + "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n" ] } @@ -545,14 +256,14 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 9, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 299 304\nWALK_LOC 94 85\nWALK_LRF 57 53\nBIKE 17 17\nDRIVEALONEFREE 6 4\nWALK_HVY 4 4\nSHARED2FREE 3 4\nTNC_SINGLE 2 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK299304
WALK_LOC9485
WALK_LRF5753
BIKE1717
DRIVEALONEFREE64
WALK_HVY44
SHARED2FREE34
TNC_SINGLE24
\n
" }, - "execution_count": 189, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -572,7 +283,7 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 10, "outputs": [ { "name": "stdout", @@ -597,20 +308,10 @@ }, { "cell_type": "code", - "execution_count": 187, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n8805121 -5709.149390 -8563.435105 -inf -2856.962423 \n8805125 -5710.236852 -8564.522566 -inf -2857.805670 \n8805126 -5709.506304 -8563.792018 -inf -2857.185402 \n468119921 -2856.413143 -5710.698857 -inf -2855.880864 \n468119922 -2856.976497 -5711.262211 -inf -2856.444428 \n468119925 -2870.186517 -5724.472231 -inf -2863.993439 \n468119926 -2857.935513 -5712.221227 -inf -2856.918910 \n468119927 -2859.140866 -5713.426580 -inf -2857.537407 \n468119928 -2869.283122 -5723.568836 -inf -2863.287255 \n642446345 -5709.613741 -8563.899455 -inf -10.654154 \n642446349 -5711.108042 -8565.393757 -inf -11.507365 \n1179608009 -8563.419303 -11417.705017 -inf -17.616599 \n1179608013 -8571.196804 -11425.482519 -inf -22.061100 \n1658807153 -5709.137752 -8563.423466 -inf -19.719413 \n1658807157 -5710.005638 -8564.291352 -inf -20.220421 \n1767182945 -5709.423410 -8563.709124 -inf -10.469202 \n1767182949 -5709.668094 -8563.953808 -inf -10.618407 \n2464104881 -5709.489682 -8563.775396 -inf -12.625725 \n2464104885 -5709.489682 -8563.775396 -inf -12.625725 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n8805121 -5711.248137 -inf -2856.962379 -5711.248093 -inf \n8805125 -5712.091384 -inf -2857.707940 -5711.993654 -inf \n8805126 -5711.471116 -inf -2857.131784 -5711.417498 -inf \n468119921 -5710.166578 -inf -2855.667952 -5709.953666 -inf \n468119922 -5710.730142 -inf -2856.231600 -5710.517314 -inf \n468119925 -5718.279154 -inf -2861.516208 -5715.801922 -inf \n468119926 -5711.204625 -inf -2856.512269 -5710.797984 -inf \n468119927 -5711.823121 -inf -2856.896023 -5711.181738 -inf \n468119928 -5717.572969 -inf -2860.888909 -5715.174623 -inf \n642446345 -2864.939868 -5.179103 -62.836260 -2917.121974 -30.545404 \n642446349 -2865.793079 -5.593858 -63.433034 -2917.718749 -30.835503 \n1179608009 -2871.902313 -8.563624 -19.761289 -2874.047003 -9.606182 \n1179608013 -2876.346814 -10.724146 -22.872589 -2877.158304 -11.118620 \n1658807153 -2874.005127 -9.585826 -21.864019 -2876.149734 -10.628343 \n1658807157 -2874.506135 -9.829371 -22.218276 -2876.503991 -10.800551 \n1767182945 -2864.754916 -5.089195 -62.653459 -2916.939174 -30.456543 \n1767182949 -2864.904122 -5.161726 -62.764474 -2917.050188 -30.510508 \n2464104881 -2866.911439 -6.137505 -64.806540 -2919.092254 -31.503179 \n2464104885 -2866.911439 -6.137505 -64.806540 -2919.092254 -31.503179 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n8805121 -inf 12.460532 -1375.039468 8.971583 -1980.409272 \n8805125 -inf 12.557754 -1374.990857 9.041583 -3978.191234 \n8805126 -inf 12.207754 -1375.165857 8.789583 -1980.441864 \n468119921 -inf 10.647096 -1376.595265 7.665909 -1984.028282 \n468119922 -inf -0.158141 -1387.385919 -0.113862 -2003.585155 \n468119925 -inf -0.158141 -1387.385919 -0.113862 -2003.585155 \n468119926 -inf 10.734596 -1376.551515 7.728909 -1987.731834 \n468119927 -inf 10.765788 -1376.855740 7.751367 -1987.818125 \n468119928 -inf 11.707060 -1375.408912 8.429083 -1985.936364 \n642446345 -3.728954 -2.097408 -1387.343797 -1.510134 1.093980 \n642446349 -4.027578 -2.146019 -1387.353519 -1.545134 1.192207 \n1179608009 -5.948375 4.880186 -1378.113147 3.513734 -1984.050332 \n1179608013 -7.350415 4.880186 -1378.113147 3.513734 -1984.050332 \n1658807153 -6.684352 6.456402 -1376.357071 4.648609 15.268960 \n1658807157 -6.845959 6.407790 -1376.381376 4.613609 14.530278 \n1767182945 -3.664221 -3.606195 -1388.969251 -2.596460 -0.555751 \n1767182949 -3.716443 -3.776334 -1389.003278 -2.718960 -0.579012 \n2464104881 -4.419004 8.134630 -1378.288147 5.856934 10.502444 \n2464104885 -4.419004 8.134630 -1378.288147 5.856934 10.666689 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n8805121 -3978.409272 -3978.409272 -3978.409272 -3978.409272 -inf \n8805125 -3978.191234 -3978.191234 -3978.191234 -3978.191234 -inf \n8805126 -3978.441864 -3978.791864 -3978.441864 -3978.441864 -inf \n468119921 -3982.028282 -3982.028282 -3982.028282 -3982.028282 -inf \n468119922 -4001.585155 -4001.935155 -4001.585155 -4001.585155 -inf \n468119925 -4001.585155 -4001.585155 -4001.585155 -4001.585155 -inf \n468119926 -3985.731834 -3986.081834 -3985.731834 -3985.731834 -inf \n468119927 -3985.818125 -3986.168125 -3985.818125 -3985.818125 -inf \n468119928 -3983.936364 -3984.286364 -3983.936364 -3983.936364 -inf \n642446345 3.035970 -1995.576420 -1995.597276 -1995.880220 2.140923 \n642446349 3.035270 -1994.736869 -1995.597276 -1995.040669 2.145065 \n1179608009 -1984.050332 -1984.050332 -1984.050332 -1984.050332 -inf \n1179608013 -1984.050332 -1984.050332 -1984.050332 -1984.050332 -inf \n1658807153 -1982.731040 -1982.731040 -1982.731040 -1982.731040 10.603444 \n1658807157 -1983.469722 -1983.469722 -1983.469722 -1983.469722 10.090471 \n1767182945 0.665917 -1997.106222 -1997.397020 -1997.410022 0.546586 \n1767182949 0.665917 -1997.106222 -1997.408920 -1997.410022 0.544118 \n2464104881 -1986.419156 -1986.167956 -1986.416956 -1986.471756 7.293364 \n2464104885 -1986.254911 -1986.003711 -1986.252711 -1986.307511 7.407423 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n8805121 -5976.701052 -5979.008301 -5979.008301 -5979.622112 -5980.000155 \n8805125 -5976.250034 -5976.250034 -5976.250034 -5976.250034 -5976.250034 \n8805126 -5976.561059 -5980.066360 -5980.066360 -5977.489912 -5981.139291 \n468119921 -1987.917275 -3986.634667 -3986.634667 -3988.159653 -3989.540253 \n468119922 -1999.781073 -4002.350822 -4002.350822 -4003.314328 -4005.292790 \n468119925 -1999.856900 -4003.785211 -4003.785211 -4001.364128 -4006.015621 \n468119926 -1987.922393 -3991.085771 -3991.085771 -3985.679774 -3991.883814 \n468119927 -1984.330401 -3988.035744 -3988.035744 -3985.888572 -3992.028453 \n468119928 -1986.084720 -3989.542402 -3989.542402 -3984.015305 -3990.339045 \n642446345 -5993.059741 -5996.236057 -5996.236057 -5996.847988 -5998.040211 \n642446349 -5993.114899 -5996.745065 -5996.745065 -5994.750788 -5998.886171 \n1179608009 -7978.109132 -7978.109132 -7978.109132 -7978.109132 -7978.109132 \n1179608013 -7978.109132 -7978.109132 -7978.109132 -7978.109132 -7978.109132 \n1658807153 -5978.797090 -5981.645926 -5981.645926 -5982.436880 -5983.088374 \n1658807157 -5979.569178 -5984.301635 -5984.301635 -5980.452579 -5984.874558 \n1767182945 -5994.656309 -5997.269370 -5997.269370 -5997.769890 -5998.629581 \n1767182949 -5994.664265 -5997.497589 -5997.497589 -5995.576790 -5998.540328 \n2464104881 -5983.483693 -5989.023319 -5989.023319 -5991.399546 -5994.064540 \n2464104885 -5983.469343 -5991.354620 -5991.354620 -5989.123846 -5997.222704 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED \\\ntrip_id \n8805121 -inf -inf -20.081973 -19.473153 -20.804324 \n8805125 -inf -inf -20.047538 -19.456673 -20.779823 \n8805126 -inf -inf -20.169318 -19.513014 -20.864116 \n468119921 -inf -inf -26.859498 -30.543501 -25.552660 \n468119922 -inf -inf -26.820071 -30.516140 -25.512174 \n468119925 -inf -inf -26.824219 -30.517112 -25.513632 \n468119926 -inf -inf -26.316039 -30.492180 -25.480679 \n468119927 -inf -inf -24.418379 -30.418607 -25.374051 \n468119928 -inf -inf -27.800562 -30.563790 -25.697316 \n642446345 -inf 1.541464 -18.289778 -17.981922 -17.196617 \n642446349 -inf 1.544447 -18.324954 -17.976575 -17.192002 \n1179608009 -inf -inf -8.526813 0.047647 -2.273228 \n1179608013 -inf -inf -8.527679 0.047161 -2.273957 \n1658807153 -inf 7.634480 -8.591632 0.206340 -2.227894 \n1658807157 -inf 7.265139 -8.647127 0.198076 -2.240290 \n1767182945 -inf 0.393542 -14.395026 -13.588217 -14.958046 \n1767182949 -inf 0.391765 -14.435998 -13.608147 -14.995707 \n2464104881 -inf 5.251222 -24.155944 -33.809528 -23.204042 \n2464104885 -inf 5.333344 -24.155944 -33.809528 -23.204042 \n\n RIDEHAIL root \ntrip_id \n8805121 -6.797103 8.971583 \n8805125 -6.788793 9.041583 \n8805126 -6.817566 8.789583 \n468119921 -9.110799 7.665909 \n468119922 -9.096330 -0.113736 \n468119925 -9.097058 -0.113736 \n468119926 -9.041678 7.728909 \n468119927 -8.672837 7.751368 \n468119928 -9.207112 8.429083 \n642446345 -5.980956 1.593075 \n642446349 -5.981865 1.593106 \n1179608009 0.050933 3.544670 \n1179608013 0.050750 3.544608 \n1658807153 0.104584 7.684253 \n1658807157 0.101487 7.334029 \n1767182945 -4.700641 0.464659 \n1767182949 -4.710741 0.456698 \n2464104881 -8.235935 6.292424 \n2464104885 -8.235935 6.322192 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
8805121-5709.149390-8563.435105-inf-2856.962423-5711.248137-inf-2856.962379-5711.248093-inf-inf12.460532-1375.0394688.971583-1980.409272-3978.409272-3978.409272-3978.409272-3978.409272-inf-5976.701052-5979.008301-5979.008301-5979.622112-5980.000155-inf-inf-20.081973-19.473153-20.804324-6.7971038.971583
8805125-5710.236852-8564.522566-inf-2857.805670-5712.091384-inf-2857.707940-5711.993654-inf-inf12.557754-1374.9908579.041583-3978.191234-3978.191234-3978.191234-3978.191234-3978.191234-inf-5976.250034-5976.250034-5976.250034-5976.250034-5976.250034-inf-inf-20.047538-19.456673-20.779823-6.7887939.041583
8805126-5709.506304-8563.792018-inf-2857.185402-5711.471116-inf-2857.131784-5711.417498-inf-inf12.207754-1375.1658578.789583-1980.441864-3978.441864-3978.791864-3978.441864-3978.441864-inf-5976.561059-5980.066360-5980.066360-5977.489912-5981.139291-inf-inf-20.169318-19.513014-20.864116-6.8175668.789583
468119921-2856.413143-5710.698857-inf-2855.880864-5710.166578-inf-2855.667952-5709.953666-inf-inf10.647096-1376.5952657.665909-1984.028282-3982.028282-3982.028282-3982.028282-3982.028282-inf-1987.917275-3986.634667-3986.634667-3988.159653-3989.540253-inf-inf-26.859498-30.543501-25.552660-9.1107997.665909
468119922-2856.976497-5711.262211-inf-2856.444428-5710.730142-inf-2856.231600-5710.517314-inf-inf-0.158141-1387.385919-0.113862-2003.585155-4001.585155-4001.935155-4001.585155-4001.585155-inf-1999.781073-4002.350822-4002.350822-4003.314328-4005.292790-inf-inf-26.820071-30.516140-25.512174-9.096330-0.113736
468119925-2870.186517-5724.472231-inf-2863.993439-5718.279154-inf-2861.516208-5715.801922-inf-inf-0.158141-1387.385919-0.113862-2003.585155-4001.585155-4001.585155-4001.585155-4001.585155-inf-1999.856900-4003.785211-4003.785211-4001.364128-4006.015621-inf-inf-26.824219-30.517112-25.513632-9.097058-0.113736
468119926-2857.935513-5712.221227-inf-2856.918910-5711.204625-inf-2856.512269-5710.797984-inf-inf10.734596-1376.5515157.728909-1987.731834-3985.731834-3986.081834-3985.731834-3985.731834-inf-1987.922393-3991.085771-3991.085771-3985.679774-3991.883814-inf-inf-26.316039-30.492180-25.480679-9.0416787.728909
468119927-2859.140866-5713.426580-inf-2857.537407-5711.823121-inf-2856.896023-5711.181738-inf-inf10.765788-1376.8557407.751367-1987.818125-3985.818125-3986.168125-3985.818125-3985.818125-inf-1984.330401-3988.035744-3988.035744-3985.888572-3992.028453-inf-inf-24.418379-30.418607-25.374051-8.6728377.751368
468119928-2869.283122-5723.568836-inf-2863.287255-5717.572969-inf-2860.888909-5715.174623-inf-inf11.707060-1375.4089128.429083-1985.936364-3983.936364-3984.286364-3983.936364-3983.936364-inf-1986.084720-3989.542402-3989.542402-3984.015305-3990.339045-inf-inf-27.800562-30.563790-25.697316-9.2071128.429083
642446345-5709.613741-8563.899455-inf-10.654154-2864.939868-5.179103-62.836260-2917.121974-30.545404-3.728954-2.097408-1387.343797-1.5101341.0939803.035970-1995.576420-1995.597276-1995.8802202.140923-5993.059741-5996.236057-5996.236057-5996.847988-5998.040211-inf1.541464-18.289778-17.981922-17.196617-5.9809561.593075
642446349-5711.108042-8565.393757-inf-11.507365-2865.793079-5.593858-63.433034-2917.718749-30.835503-4.027578-2.146019-1387.353519-1.5451341.1922073.035270-1994.736869-1995.597276-1995.0406692.145065-5993.114899-5996.745065-5996.745065-5994.750788-5998.886171-inf1.544447-18.324954-17.976575-17.192002-5.9818651.593106
1179608009-8563.419303-11417.705017-inf-17.616599-2871.902313-8.563624-19.761289-2874.047003-9.606182-5.9483754.880186-1378.1131473.513734-1984.050332-1984.050332-1984.050332-1984.050332-1984.050332-inf-7978.109132-7978.109132-7978.109132-7978.109132-7978.109132-inf-inf-8.5268130.047647-2.2732280.0509333.544670
1179608013-8571.196804-11425.482519-inf-22.061100-2876.346814-10.724146-22.872589-2877.158304-11.118620-7.3504154.880186-1378.1131473.513734-1984.050332-1984.050332-1984.050332-1984.050332-1984.050332-inf-7978.109132-7978.109132-7978.109132-7978.109132-7978.109132-inf-inf-8.5276790.047161-2.2739570.0507503.544608
1658807153-5709.137752-8563.423466-inf-19.719413-2874.005127-9.585826-21.864019-2876.149734-10.628343-6.6843526.456402-1376.3570714.64860915.268960-1982.731040-1982.731040-1982.731040-1982.73104010.603444-5978.797090-5981.645926-5981.645926-5982.436880-5983.088374-inf7.634480-8.5916320.206340-2.2278940.1045847.684253
1658807157-5710.005638-8564.291352-inf-20.220421-2874.506135-9.829371-22.218276-2876.503991-10.800551-6.8459596.407790-1376.3813764.61360914.530278-1983.469722-1983.469722-1983.469722-1983.46972210.090471-5979.569178-5984.301635-5984.301635-5980.452579-5984.874558-inf7.265139-8.6471270.198076-2.2402900.1014877.334029
1767182945-5709.423410-8563.709124-inf-10.469202-2864.754916-5.089195-62.653459-2916.939174-30.456543-3.664221-3.606195-1388.969251-2.596460-0.5557510.665917-1997.106222-1997.397020-1997.4100220.546586-5994.656309-5997.269370-5997.269370-5997.769890-5998.629581-inf0.393542-14.395026-13.588217-14.958046-4.7006410.464659
1767182949-5709.668094-8563.953808-inf-10.618407-2864.904122-5.161726-62.764474-2917.050188-30.510508-3.716443-3.776334-1389.003278-2.718960-0.5790120.665917-1997.106222-1997.408920-1997.4100220.544118-5994.664265-5997.497589-5997.497589-5995.576790-5998.540328-inf0.391765-14.435998-13.608147-14.995707-4.7107410.456698
2464104881-5709.489682-8563.775396-inf-12.625725-2866.911439-6.137505-64.806540-2919.092254-31.503179-4.4190048.134630-1378.2881475.85693410.502444-1986.419156-1986.167956-1986.416956-1986.4717567.293364-5983.483693-5989.023319-5989.023319-5991.399546-5994.064540-inf5.251222-24.155944-33.809528-23.204042-8.2359356.292424
2464104885-5709.489682-8563.775396-inf-12.625725-2866.911439-6.137505-64.806540-2919.092254-31.503179-4.4190048.134630-1378.2881475.85693410.666689-1986.254911-1986.003711-1986.252711-1986.3075117.407423-5983.469343-5991.354620-5991.354620-5989.123846-5997.222704-inf5.333344-24.155944-33.809528-23.204042-8.2359356.322192
\n
" - }, - "execution_count": 187, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 12, + "outputs": [], "source": [ - "simulate.compute_nested_utilities(ru[0], ns[0])" + "#simulate.compute_nested_utilities(ru[0], ns[0])" ], "metadata": { "collapsed": false, @@ -638,7 +339,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 26, "outputs": [ { "name": "stderr", @@ -653,36 +354,7 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (19 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "trip_mode_choice tour_type 'social' (19 trips)\n", "Done\n" ] } @@ -700,7 +372,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 27, "outputs": [ { "name": "stdout", @@ -708,13 +380,8 @@ "text": [ "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] parent nest scale: 0\n", "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] parent nest scale: 1.0\n", - "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] parent nest scale: 0.72\n", - "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2'] parent nest scale: 0.72\n", - "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3'] parent nest scale: 0.72\n", "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] parent nest scale: 1.0\n", "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] parent nest scale: 1.0\n", - "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS'] parent nest scale: 0.72\n", - "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS'] parent nest scale: 0.72\n", "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] parent nest scale: 1.0\n" ] } @@ -733,37 +400,32 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 28, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "leaf DRIVEALONEFREE with 0.252, 0.35\n", - "leaf DRIVEALONEPAY with 0.252, 0.35\n", - "node DRIVEALONE with 0.35, 0.72\n", - "leaf SHARED2FREE with 0.252, 0.35\n", - "leaf SHARED2PAY with 0.252, 0.35\n", - "node SHAREDRIDE2 with 0.35, 0.72\n", - "leaf SHARED3FREE with 0.252, 0.35\n", - "leaf SHARED3PAY with 0.252, 0.35\n", - "node SHAREDRIDE3 with 0.35, 0.72\n", + "leaf DRIVEALONEFREE with 0.72, 0.72\n", + "leaf DRIVEALONEPAY with 0.72, 0.72\n", + "leaf SHARED2FREE with 0.72, 0.72\n", + "leaf SHARED2PAY with 0.72, 0.72\n", + "leaf SHARED3FREE with 0.72, 0.72\n", + "leaf SHARED3PAY with 0.72, 0.72\n", "node AUTO with 0.72, 1.0\n", "leaf WALK with 0.72, 0.72\n", "leaf BIKE with 0.72, 0.72\n", "node NONMOTORIZED with 0.72, 1.0\n", - "leaf WALK_LOC with 0.36, 0.5\n", - "leaf WALK_LRF with 0.36, 0.5\n", - "leaf WALK_EXP with 0.36, 0.5\n", - "leaf WALK_HVY with 0.36, 0.5\n", - "leaf WALK_COM with 0.36, 0.5\n", - "node WALKACCESS with 0.5, 0.72\n", - "leaf DRIVE_LOC with 0.36, 0.5\n", - "leaf DRIVE_LRF with 0.36, 0.5\n", - "leaf DRIVE_EXP with 0.36, 0.5\n", - "leaf DRIVE_HVY with 0.36, 0.5\n", - "leaf DRIVE_COM with 0.36, 0.5\n", - "node DRIVEACCESS with 0.5, 0.72\n", + "leaf WALK_LOC with 0.72, 0.72\n", + "leaf WALK_LRF with 0.72, 0.72\n", + "leaf WALK_EXP with 0.72, 0.72\n", + "leaf WALK_HVY with 0.72, 0.72\n", + "leaf WALK_COM with 0.72, 0.72\n", + "leaf DRIVE_LOC with 0.72, 0.72\n", + "leaf DRIVE_LRF with 0.72, 0.72\n", + "leaf DRIVE_EXP with 0.72, 0.72\n", + "leaf DRIVE_HVY with 0.72, 0.72\n", + "leaf DRIVE_COM with 0.72, 0.72\n", "node TRANSIT with 0.72, 1.0\n", "leaf TAXI with 0.36, 0.36\n", "leaf TNC_SINGLE with 0.36, 0.36\n", @@ -826,14 +488,14 @@ }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 29, "outputs": [ { "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.262023e-03 0.0 \n642446349 0.0 0.0 3.138876e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.339899e-02 0.0 \n1767182949 0.0 0.0 1.282607e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.103657e-14 0.0 0.039196 0.0 0.197050 0.759044 \n642446349 3.423474e-14 0.0 0.037574 0.0 0.208612 0.750231 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.288763e-13 0.0 0.038976 0.0 0.282647 0.660225 \n1767182949 1.256775e-13 0.0 0.034777 0.0 0.280877 0.666774 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 8.388831e-05 1.141304e-04 2.502967e-04 \n642446349 0.0 0.0 8.053321e-05 1.140970e-04 2.500402e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.247409e-03 2.795129e-03 7.103828e-04 \n1767182949 0.0 0.0 1.229459e-03 2.813489e-03 7.024828e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.262023e-030.04.103657e-140.00.0391960.00.1970500.7590440.00.00.00.00.00.00.00.08.388831e-051.141304e-042.502967e-04
6424463490.00.03.138876e-030.03.423474e-140.00.0375740.00.2086120.7502310.00.00.00.00.00.00.00.08.053321e-051.140970e-042.500402e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.339899e-020.01.288763e-130.00.0389760.00.2826470.6602250.00.00.00.00.00.00.00.01.247409e-032.795129e-037.103828e-04
17671829490.00.01.282607e-020.01.256775e-130.00.0347770.00.2808770.6667740.00.00.00.00.00.00.00.01.229459e-032.813489e-037.024828e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" }, - "execution_count": 196, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -878,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 30, "outputs": [], "source": [ "def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples):\n", @@ -1081,7 +743,7 @@ }, { "cell_type": "code", - "execution_count": 198, + "execution_count": 31, "outputs": [ { "name": "stderr", @@ -1096,39 +758,14 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (100000 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n", - "numpy: Warning: underflow encountered in exp expression: origTaxiWaitTime = rng.lognormal_for_df(df, mu=origin_zone_taxi_wait_time_mean, sigma=origin_zone_taxi_wait_time_sd, broadcast=True, scale=True).clip(min_waitTime, max_waitTime)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "trip_mode_choice tour_type 'social' (10000 trips)\n", "Done\n" ] }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.005210 0.000000e+00 0.000130 0.000280 0.000100 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.045410 0.061630 0.887240 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0052100.000000e+000.0001300.0002800.0001000.0454100.0616300.887240
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004262 4.103657e-14 0.000084 0.00025 0.000114 \ntrip_mode 0.004300 0.000000e+00 0.000100 0.00050 0.000100 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.039196 0.19705 0.759044 \ntrip_mode 0.039100 0.20000 0.755900 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0042624.103657e-140.0000840.000250.0001140.0391960.197050.759044
trip_mode0.0043000.000000e+000.0001000.000500.0001000.0391000.200000.755900
\n
" }, "metadata": {}, "output_type": "display_data" @@ -1136,7 +773,7 @@ ], "source": [ "choose_individual_max_utility = False\n", - "num_samples = 100000\n", + "num_samples = 10000\n", "trip_id_to_check = 642446345\n", "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" ], @@ -1149,7 +786,7 @@ }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 38, "id": "1fcdc0ff-c510-4878-8672-8365e046442c", "metadata": {}, "outputs": [ @@ -1166,14 +803,14 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (100000 trips)\n", + "trip_mode_choice tour_type 'social' (10000 trips)\n", "Done\n" ] }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.000000 0.000000e+00 0.000000 0.000000 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.002600 0.020660 0.976740 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0000000.000000e+000.0000000.0000000.0000000.0026000.0206600.976740
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004262 4.103657e-14 0.000084 0.00025 0.000114 \ntrip_mode 0.000400 0.000000e+00 0.000000 0.00000 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.039196 0.19705 0.759044 \ntrip_mode 0.011600 0.20270 0.785300 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0042624.103657e-140.0000840.000250.0001140.0391960.197050.759044
trip_mode0.0004000.000000e+000.0000000.000000.0000000.0116000.202700.785300
\n
" }, "metadata": {}, "output_type": "display_data" @@ -1181,8 +818,8 @@ ], "source": [ "choose_individual_max_utility = True\n", - "num_samples = 100000 # 1e5 takes about 25s\n", - "trip_id_to_check = 642446345\n", + "num_samples = 10000 # 1e5 takes about 25s\n", + "trip_id_to_check = 642446345 # 2464104885 # 642446345 # 1767182945\n", "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" ] }, @@ -1190,7 +827,9 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [], + "source": [ + "# something is still slightly off. WHY?" + ], "metadata": { "collapsed": false, "pycharm": { From 4883c3342bcc0766fb4ea9f43b712b86e0ac5044 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 6 May 2022 12:13:41 +1000 Subject: [PATCH 013/135] recursive utility calculation fix --- activitysim/core/logit.py | 20 +- activitysim/core/simulate.py | 19 +- ...ested_logit_frozen_rand_individ_util.ipynb | 196 ------------------ notebooks/validate_frozen_impl.ipynb | 68 ++++-- 4 files changed, 59 insertions(+), 244 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index dd3c401eb2..048b37da42 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -197,14 +197,9 @@ def add_ev1_random(df, nest_spec): nest_utils_for_choice = df.copy() for n in each_nest(nest_spec): if n.level == 1: - continue # skip the root level - # TODO: check parent nest level scale is what we want + continue # skip the root level, not needed uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice) - #if n.is_leaf: - # scale = n.coefficient # product_of_coefficients parent_nest_scale coefficient - #else: - scale = 1.0 # n.coefficient - rands = inverse_ev1_cdf(uniform_rands, scale=scale) + rands = inverse_ev1_cdf(uniform_rands) nest_utils_for_choice.loc[:, n.name] += rands[:, 0] # inverse_ev1_cdf of single-row df adds dimension return nest_utils_for_choice @@ -377,12 +372,10 @@ def __init__(self, name=None, level=0): self.ancestors = [] self.alternatives = None self.coefficient = 0 - self.parent_nest_scale = 1.0 def print(self): - print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s parent nest scale: " - "%s" % - (self.name, self.level, self.coefficient, self.product_of_coefficients, self.ancestors, self.parent_nest_scale)) + print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s " % + (self.name, self.level, self.coefficient, self.product_of_coefficients, self.ancestors)) @property def is_leaf(self): @@ -453,7 +446,6 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients * coefficient nest.alternatives = alternatives nest.ancestors = parent_nest.ancestors + [name] - nest.parent_nest_scale = parent_nest.coefficient if pre_order: yield spec, nest @@ -475,10 +467,6 @@ def _each_nest(spec, parent_nest, post_order): nest.ancestors = parent_nest.ancestors + [name] nest.coefficient = parent_nest.coefficient - # this is used for frozen individual ru - see discussion in simulate.py l.740 - #nest.parent_nest_scale = parent_nest.product_of_coefficients - nest.parent_nest_scale = parent_nest.coefficient - yield spec, nest diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index f3843bd0c7..f5667ef307 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -736,21 +736,18 @@ def compute_nested_utilities(raw_utilities, nest_spec): for nest in logit.each_nest(nest_spec, post_order=True): name = nest.name if nest.is_leaf: - #nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients - nested_utilities[name] = raw_utilities[name].astype(float) / nest.coefficient - # WHY does asim use product of coeffs here? using that gives us identical root logsums - # but then we need to scale ev1 accordingly for alternatives, right? + # do not scale here, do afterwards so recursive structure works + nested_utilities[name] = raw_utilities[name].astype(float) #/ nest.coefficient else: # the alternative nested_utilities will already have been computed due to post_order - #if nest.level == 1: # FIXME: work out the parent scale mess - # scale = 1.0 - #else: - # scale = nest.parent_nest_scale - scale = 1.0 # IT'S RECURSIVE for lowest level but check for higher level - with np.errstate(divide='ignore'): nested_utilities[name] = \ - nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / scale).sum(axis=1)) + nest.coefficient * np.log( + np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1) + ) + + for nest in logit.each_nest(nest_spec, type="leaf"): + nested_utilities[nest.name] /= nest.coefficient return nested_utilities diff --git a/notebooks/nested_logit_frozen_rand_individ_util.ipynb b/notebooks/nested_logit_frozen_rand_individ_util.ipynb index 4f06fca9ef..fbf4fce6be 100644 --- a/notebooks/nested_logit_frozen_rand_individ_util.ipynb +++ b/notebooks/nested_logit_frozen_rand_individ_util.ipynb @@ -302,202 +302,6 @@ } } }, - { - "cell_type": "markdown", - "source": [ - "# This is the Zenith way as it was designed originally\n", - "\n", - "w/o scaling utilities in logsums, everything else is identical" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 47, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.19304564457406473 0.6714008596782736 0.1355534957476616\n" - ] - } - ], - "source": [ - "cost = 3.0\n", - "nest_scale = 0.6\n", - "\n", - "util_3 = utility(cost, utility_spec, 3)\n", - "util_2 = utility(cost, utility_spec, 2)\n", - "logsum_bus = logsum(np.array([util_2, util_3])) #, nest_scale=nest_scale) # DIFF\n", - "nest_util = nest_scale * logsum_bus\n", - "\n", - "util_1 = utility(cost, utility_spec, 1)\n", - "prob_1 = np.exp(util_1) / (np.exp(util_1) + np.exp(nest_util))\n", - "\n", - "nest_cond_prob = np.exp(nest_util) / (np.exp(util_1) + np.exp(nest_util))\n", - "nest_marg_prob_2 = np.exp(util_2) / (np.exp(util_2) + np.exp(util_3)) # DIFF\n", - "nest_marg_prob_3 = np.exp(util_3) / (np.exp(util_2) + np.exp(util_3)) # DIFF\n", - "\n", - "prob_2 = nest_cond_prob * nest_marg_prob_2\n", - "prob_3 = nest_cond_prob * nest_marg_prob_3\n", - "\n", - "print(prob_1, prob_2, prob_3)\n", - "assert np.isclose(1.0, sum([prob_1, prob_2, prob_3]))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 48, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "closed form probs: 0.193046, 0.671401, 0.135553\n", - " simulated probs: 0.1930163, 0.6715585, 0.1354252\n" - ] - } - ], - "source": [ - "# test w/o scaling in utility? is this what previous zenith version does?\n", - "\n", - "# conditional error term are given by logit with scale given by nest scale\n", - "num_draws_dec = 10000000\n", - "\n", - "util_3 = utility(cost, utility_spec, 3)\n", - "util_2 = utility(cost, utility_spec, 2)\n", - "logsum_bus = logsum(np.array([util_2, util_3])) # , nest_scale=nest_scale) # THAT's the only diff right #\n", - "util_1 = utility(cost, utility_spec, 1)\n", - "\n", - "\n", - "rng_dec = default_rng(9)\n", - "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", - "ev1_lower = inverse_ev1_cdf(rands_dec) #, scale=nest_scale)\n", - "\n", - "lower_utils_2 = util_2 + ev1_lower[num_draws_dec:]\n", - "lower_utils_3 = util_3 + ev1_lower[:num_draws_dec]\n", - "\n", - "#logsum_bus = logsum(np.array([lower_utils_2, lower_utils_3]), nest_scale=nest_scale)\n", - "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=2*num_draws_dec))\n", - "nest_util = nest_scale * logsum_bus + ev1_upper[num_draws_dec:]\n", - "\n", - "full_util_1 = util_1 + ev1_upper[:num_draws_dec]\n", - "\n", - "choices = np.array([full_util_1, nest_util]).argmax(axis=0)\n", - "nest_indexes = np.nonzero(choices == 1)[0]\n", - "nest_choices = np.array([lower_utils_2[nest_indexes], lower_utils_3[nest_indexes]]).argmax(axis=0)\n", - "nest_choices += 1\n", - "choices = np.append(choices[choices == 0], nest_choices)\n", - "\n", - "vals, counts = np.unique(choices, return_counts=True)\n", - "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", - "\n", - "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", - "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index bfb9b39faf..8af08252eb 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -452,9 +452,41 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 39, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: DRIVEALONEFREE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'DRIVEALONEFREE'] parent nest scale: 0.72\n", + "Nest name: DRIVEALONEPAY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'DRIVEALONEPAY'] parent nest scale: 0.72\n", + "Nest name: SHARED2FREE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED2FREE'] parent nest scale: 0.72\n", + "Nest name: SHARED2PAY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED2PAY'] parent nest scale: 0.72\n", + "Nest name: SHARED3FREE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED3FREE'] parent nest scale: 0.72\n", + "Nest name: SHARED3PAY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED3PAY'] parent nest scale: 0.72\n", + "Nest name: WALK level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'WALK'] parent nest scale: 0.72\n", + "Nest name: BIKE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'BIKE'] parent nest scale: 0.72\n", + "Nest name: WALK_LOC level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_LOC'] parent nest scale: 0.72\n", + "Nest name: WALK_LRF level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_LRF'] parent nest scale: 0.72\n", + "Nest name: WALK_EXP level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_EXP'] parent nest scale: 0.72\n", + "Nest name: WALK_HVY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_HVY'] parent nest scale: 0.72\n", + "Nest name: WALK_COM level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_COM'] parent nest scale: 0.72\n", + "Nest name: DRIVE_LOC level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_LOC'] parent nest scale: 0.72\n", + "Nest name: DRIVE_LRF level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_LRF'] parent nest scale: 0.72\n", + "Nest name: DRIVE_EXP level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_EXP'] parent nest scale: 0.72\n", + "Nest name: DRIVE_HVY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_HVY'] parent nest scale: 0.72\n", + "Nest name: DRIVE_COM level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_COM'] parent nest scale: 0.72\n", + "Nest name: TAXI level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TAXI'] parent nest scale: 0.36\n", + "Nest name: TNC_SINGLE level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SINGLE'] parent nest scale: 0.36\n", + "Nest name: TNC_SHARED level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SHARED'] parent nest scale: 0.36\n" + ] + } + ], + "source": [ + "nest_spec = ns[0]\n", + "for nest in logit.each_nest(nest_spec, type='leaf'):\n", + " nest.print()" + ], "metadata": { "collapsed": false, "pycharm": { @@ -540,7 +572,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 40, "outputs": [], "source": [ "def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples):\n", @@ -743,7 +775,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 41, "outputs": [ { "name": "stderr", @@ -786,9 +818,14 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "1fcdc0ff-c510-4878-8672-8365e046442c", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + }, "outputs": [ { "name": "stderr", @@ -803,22 +840,13 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (10000 trips)\n", - "Done\n" + "trip_mode_choice tour_type 'social' (100000 trips)\n" ] - }, - { - "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004262 4.103657e-14 0.000084 0.00025 0.000114 \ntrip_mode 0.000400 0.000000e+00 0.000000 0.00000 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.039196 0.19705 0.759044 \ntrip_mode 0.011600 0.20270 0.785300 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0042624.103657e-140.0000840.000250.0001140.0391960.197050.759044
trip_mode0.0004000.000000e+000.0000000.000000.0000000.0116000.202700.785300
\n
" - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ "choose_individual_max_utility = True\n", - "num_samples = 10000 # 1e5 takes about 25s\n", + "num_samples = 100000 # 1e5 takes about 25s\n", "trip_id_to_check = 642446345 # 2464104885 # 642446345 # 1767182945\n", "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" ] @@ -827,9 +855,7 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [ - "# something is still slightly off. WHY?" - ], + "source": [], "metadata": { "collapsed": false, "pycharm": { From b60702b438bc8612eda6a124b61af7d9569f1cca Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 6 May 2022 12:21:50 +1000 Subject: [PATCH 014/135] clean up --- activitysim/core/simulate.py | 2 +- notebooks/fru_utils.py | 193 +++++++++++ notebooks/validate_frozen_impl.ipynb | 467 +++------------------------ 3 files changed, 236 insertions(+), 426 deletions(-) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index f5667ef307..add013e668 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -737,7 +737,7 @@ def compute_nested_utilities(raw_utilities, nest_spec): name = nest.name if nest.is_leaf: # do not scale here, do afterwards so recursive structure works - nested_utilities[name] = raw_utilities[name].astype(float) #/ nest.coefficient + nested_utilities[name] = raw_utilities[name].astype(float) else: # the alternative nested_utilities will already have been computed due to post_order with np.errstate(divide='ignore'): diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index 4ead449bb9..935e8b0289 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -18,6 +18,199 @@ from activitysim.core.util import assign_in_place + +def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples): + """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" + resume_after = "trip_scheduling" + model_name = "trip_mode_choice" + chunk_size = 0 # test_mtc means no chunking + + pipeline.open_pipeline(resume_after) + # preload any bulky injectables (e.g. skims) not in pipeline + inject.get_injectable('preload_injectables', None) + pipeline._PIPELINE.rng().begin_step(model_name) + #step_name = model_name + args = {} + #checkpoint = pipeline.intermediate_checkpoint(model_name) + inject.set_step_args(args) + + trips = inject.get_table('trips') + tours_merged = inject.get_table('tours_merged') + network_los = inject.get_injectable('network_los') + + trace_label = 'trip_mode_choice' + model_settings_file_name = 'trip_mode_choice.yaml' + model_settings = config.read_model_settings(model_settings_file_name) + + logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') + mode_column_name = 'trip_mode' + + trips_df = trips.to_frame() + print("Running with %d trips", trips_df.shape[0]) + + tours_merged = tours_merged.to_frame() + tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] + + # - trips_merged - merge trips and tours_merged + trips_merged = pd.merge( + trips_df, + tours_merged, + left_on='tour_id', + right_index=True, + how="left") + assert trips_merged.index.equals(trips.index) + + # setup skim keys + assert ('trip_period' not in trips_merged) + trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) + + orig_col = 'origin' + dest_col = 'destination' + + constants = {} + constants.update(config.get_model_constants(model_settings)) + constants.update({ + 'ORIGIN': orig_col, + 'DESTINATION': dest_col + }) + + skim_dict = network_los.get_default_skim_dict() + + odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col, + dim3_key='trip_period') + dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col, + dim3_key='trip_period') + od_skim_wrapper = skim_dict.wrap('origin', 'destination') + + skims = { + "odt_skims": odt_skim_stack_wrapper, + "dot_skims": dot_skim_stack_wrapper, + "od_skims": od_skim_wrapper, + } + + model_spec = simulate.read_model_spec(file_name=model_settings['SPEC']) + nest_specs = config.get_logit_model_settings(model_settings) + + estimator = estimation.manager.begin_estimation('trip_mode_choice') + + #choices_list = [] + + + # grab one, duplicate num_samples times + trips_segment = trips_merged.loc[trips_merged.index == trip_id_to_check].copy() + primary_purpose = trips_segment['primary_purpose'].values[0] + trips_segment = trips_segment.loc[trips_segment.index.repeat(num_samples)] + + # need to add new row_states for rng here, need to ensure there are no collisions with existing keys + existing_indexes = pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values + num_new_indexes = trips_segment.shape[0] + new_indexes = np.arange(existing_indexes.max()+1, existing_indexes.max() + num_new_indexes + 1) + + trips_segment.index = new_indexes #+= np.arange(num_samples) + # name index so tracing knows how to slice + trips_segment.index.name = 'trip_id' + + + # #logger.warning("Change seeding back when done with testing") + # pipeline._PIPELINE.rng.row_states = pd.DataFrame(columns=['row_seed', 'offset'], index=trips_segment.index) + # pipeline._PIPELINE.rng.row_states["row_seed"] = trips_segment.index.values + # pipeline._PIPELINE.rng.row_states["offset"] = 0 + pipeline._PIPELINE.rng().add_channel("trips", trips_segment) + + + #for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'): + #if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes): + # continue + + print("trip_mode_choice tour_type '%s' (%s trips)" % + (primary_purpose, len(trips_segment.index), )) + + coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose) + + locals_dict = {} + locals_dict.update(constants) + locals_dict.update(coefficients) + + segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose) + + expressions.annotate_preprocessors( + trips_segment, locals_dict, skims, + model_settings, segment_trace_label) + + locals_dict.update(skims) + + ################ Replace wrapper function + # choices = mode_choice_simulate(...) + spec=simulate.eval_coefficients(model_spec, coefficients, estimator) + nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label) + choices = simulate.simple_simulate( + choosers=trips_segment, + spec=spec, + nest_spec=nest_spec, + skims=skims, + locals_d=locals_dict, + chunk_size=chunk_size, + want_logsums=logsum_column_name is not None, + trace_label=segment_trace_label, + trace_choice_name='trip_mode_choice', + estimator=estimator, + trace_column_names=None, + choose_individual_max_utility=choose_individual_max_utility) + # for consistency, always return dataframe, whether or not logsums were requested + if isinstance(choices, pd.Series): + choices = choices.to_frame('choice') + choices.rename(columns={'logsum': logsum_column_name, + 'choice': mode_column_name}, + inplace=True) + if not choose_individual_max_utility: + alts = spec.columns + choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) + ################ + #choices_list.append(choices) + #choices_df_asim = pd.concat(choices_list) + + # update trips table with choices (and potionally logssums) + #trips_df = trips_merged.copy() # trips.to_frame() + + #if (do_these_purposes is not None): + # trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)] + + #assign_in_place(trips_df, choices) + #assert not trips_df[mode_column_name].isnull().any() + + finalise = True + if finalise: + inject.set_step_args(None) + # + pipeline._PIPELINE.rng().end_step(model_name) + pipeline.add_checkpoint(model_name) + if not pipeline.intermediate_checkpoint(): + pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME) + + pipeline.close_pipeline() + + print("Done") + return trips_merged, choices + + +def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check): + t_, c_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, + trip_id_to_check=trip_id_to_check, num_samples=num_samples) + + sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0] + #sim_mode_shares.columns = ["mode_share_sim"] + obs_probs = base_probs[0].loc[base_probs[0].index == trip_id_to_check].T + obs_probs.columns = ["mode_share_obs"] + ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how="outer").fillna(0) + ms_comp["diff"] = ms_comp["trip_mode"] - ms_comp["mode_share_obs"] + ms_comp["rel_diff"] = ms_comp["diff"] / ms_comp["mode_share_obs"] + #ms_comp.style.format('{:.2}') + #with pd.option_context("precision", 3): + # display(ms_comp) + return ms_comp + + + def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True): """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 8af08252eb..58b7fae0c4 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 56, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -72,15 +72,8 @@ "\n", "from activitysim.cli import run\n", "from activitysim.core import inject\n", - "from activitysim.core import tracing\n", - "from activitysim.core import config\n", - "from activitysim.core import pipeline\n", - "from activitysim.core import chunk\n", - "from activitysim.core import simulate\n", - "from activitysim.core import logit\n", - "from activitysim.abm.models.util import estimation\n", - "from activitysim.core import expressions\n", - "from activitysim.core.util import assign_in_place" + "\n", + "from fru_utils import run_trip_mode_choice, mode_choice_for_trip" ] }, { @@ -170,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 57, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -219,7 +212,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Done\n" + "Done\n", + "Running with %d trips 482\n" ] }, { @@ -233,7 +227,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", @@ -249,21 +242,20 @@ } ], "source": [ - "from fru_utils import run_trip_mode_choice\n", "trips_df_ru = run_trip_mode_choice()\n", "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 58, "outputs": [ { "data": { - "text/plain": " trip_mode_fru trip_mode_asim\nWALK 299 304\nWALK_LOC 94 85\nWALK_LRF 57 53\nBIKE 17 17\nDRIVEALONEFREE 6 4\nWALK_HVY 4 4\nSHARED2FREE 3 4\nTNC_SINGLE 2 4", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK299304
WALK_LOC9485
WALK_LRF5753
BIKE1717
DRIVEALONEFREE64
WALK_HVY44
SHARED2FREE34
TNC_SINGLE24
\n
" + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 332 304\nWALK_LOC 72 85\nWALK_LRF 45 53\nBIKE 16 17\nDRIVEALONEFREE 7 4\nWALK_HVY 4 4\nTNC_SINGLE 3 4\nSHARED2FREE 2 4\nTNC_SHARED 1 4", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK332304
WALK_LOC7285
WALK_LRF4553
BIKE1617
DRIVEALONEFREE74
WALK_HVY44
TNC_SINGLE34
SHARED2FREE24
TNC_SHARED14
\n
" }, - "execution_count": 9, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -283,14 +275,17 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 59, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logsums agree\n" - ] + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN+ElEQVR4nO3df2xd9XnH8fdTXCgiG2kb5rEkmpGIOiHc0WIBU/+YA/sRoFrYBBUdKkmXKZoEUqdmak2RVk3btFSIoXarKkWlWjqhGcRWgQjVygCr6h9hS1pK+NEOF4U1EQXR0nQGusnrsz984NqpzXXiY9/cx++XZOV8v9/jr5/7CH18cnLuJTITSVItb+t1AZKk9hnuklSQ4S5JBRnuklSQ4S5JBQ30ugCAdevW5dDQUCt7vfrqq5x11lmt7NXv7EWHveiwFx393ouDBw++nJnnzLd2SoT70NAQBw4caGWviYkJRkdHW9mr39mLDnvRYS86+r0XEfH8QmvelpGkggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekgk6Jd6hK0koYGts3Z7xreJrtY/s4vPvqHlW0fLxyl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKmjR4R4Rp0XEtyLigWZ8XkQ8FhGTEXF3RJzezJ/RjCeb9aFlql2StIATuXL/GPDMrPFngDsy83zgFWBHM78DeKWZv6M5T5K0ghYV7hGxAbga+GIzDuBy4N7mlL3ANc3x1mZMs35Fc74kaYVEZnY/KeJe4G+AXwD+DNgO7G+uzomIjcBXM/PCiHgS2JKZR5q17wGXZubLx+25E9gJMDg4ePH4+HgrL2hqaoo1a9a0sle/sxcd9qJjNffi0NFjc8aDZ8KLr8Pw+rN7VNHSbN68+WBmjsy3NtDtmyPig8BLmXkwIkbbKioz9wB7AEZGRnJ0tJ2tJyYmaGuvfmcvOuxFx2ruxfaxfXPGu4anuf3QAIdvGO1NQcuoa7gDHwB+LyKuAt4B/CLwWWBtRAxk5jSwATjanH8U2AgciYgB4Gzgh61XLklaUNd77pl5S2ZuyMwh4Hrgkcy8AXgUuLY5bRtwX3N8fzOmWX8kF3PvR5LUmqU85/5J4OMRMQm8G7izmb8TeHcz/3FgbGklSpJO1GJuy7wpMyeAieb4OeCSec75KXBdC7VJkk6S71CVpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIK6hntEvCMi/j0ivh0RT0XEXzTz50XEYxExGRF3R8TpzfwZzXiyWR9a5tcgSTrOYq7c/we4PDN/HbgI2BIRlwGfAe7IzPOBV4Adzfk7gFea+Tua8yRJK6hruOeMqWb49uYrgcuBe5v5vcA1zfHWZkyzfkVERFsFS5K6i8zsflLEacBB4Hzg88BtwP7m6pyI2Ah8NTMvjIgngS2ZeaRZ+x5waWa+fNyeO4GdAIODgxePj4+38oKmpqZYs2ZNK3v1O3vRYS86VnMvDh09Nmc8eCa8+DoMrz+7RxUtzebNmw9m5sh8awOL2SAz/w+4KCLWAl8Bfm2pRWXmHmAPwMjISI6Oji51SwAmJiZoa69+Zy867EXHau7F9rF9c8a7hqe5/dAAh28Y7U1By+iEnpbJzB8DjwK/AayNiDd+OWwAjjbHR4GNAM362cAP2yhWkrQ4i3la5pzmip2IOBP4beAZZkL+2ua0bcB9zfH9zZhm/ZFczL0fSVJrFnNb5lxgb3Pf/W3APZn5QEQ8DYxHxF8B3wLubM6/E/jHiJgEfgRcvwx1S5LeQtdwz8wngPfNM/8ccMk88z8FrmulOknSSfEdqpJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUUNdwj4iNEfFoRDwdEU9FxMea+XdFxEMR8Wzz5zub+YiIz0XEZEQ8ERHvX+4XIUmaazFX7tPArsy8ALgMuCkiLgDGgIczcxPwcDMGuBLY1HztBL7QetWSpLfUNdwz84XM/GZz/N/AM8B6YCuwtzltL3BNc7wV+HLO2A+sjYhz2y5ckrSwE7rnHhFDwPuAx4DBzHyhWfoBMNgcrwe+P+vbjjRzkqQVMrDYEyNiDfDPwJ9m5k8i4s21zMyIyBP5wRGxk5nbNgwODjIxMXEi376gqamp1vbqd/aiw150rOZe7BqenjMePHNmrmI/FhXuEfF2ZoL9rsz8l2b6xYg4NzNfaG67vNTMHwU2zvr2Dc3cHJm5B9gDMDIykqOjoyf3Co4zMTFBW3v1O3vRYS86VnMvto/tmzPeNTzN7YcGOHzDaG8KWkaLeVomgDuBZzLzb2ct3Q9sa463AffNmr+xeWrmMuDYrNs3kqQVsJgr9w8AHwEORcTjzdyngN3APRGxA3ge+FCz9iBwFTAJvAZ8tM2CJUnddQ33zPwGEAssXzHP+QnctMS6JElL4DtUJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCuoa7hHxpYh4KSKenDX3roh4KCKebf58ZzMfEfG5iJiMiCci4v3LWbwkaX6LuXL/B2DLcXNjwMOZuQl4uBkDXAlsar52Al9op0xJ0onoGu6Z+XXgR8dNbwX2Nsd7gWtmzX85Z+wH1kbEuS3VKklapMjM7idFDAEPZOaFzfjHmbm2OQ7glcxcGxEPALsz8xvN2sPAJzPzwDx77mTm6p7BwcGLx8fHW3lBU1NTrFmzppW9+p296LAXHau5F4eOHpszHjwTXnwdhtef3aOKlmbz5s0HM3NkvrWBpW6emRkR3X9D/Pz37QH2AIyMjOTo6OhSSwFgYmKCtvbqd/aiw150rOZebB/bN2e8a3ia2w8NcPiG0d4UtIxO9mmZF9+43dL8+VIzfxTYOOu8Dc2cJGkFnWy43w9sa463AffNmr+xeWrmMuBYZr6wxBolSSeo622ZiPgnYBRYFxFHgE8Du4F7ImIH8Dzwoeb0B4GrgEngNeCjy1CzJKmLruGemR9eYOmKec5N4KalFiVJWhrfoSpJBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBQ30ugBJ6rWhsX3zzh/effUKV9Ier9wlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaBlec49IrYAnwVOA76YmbuX4+dI0nLq5+ffW79yj4jTgM8DVwIXAB+OiAva/jmSpIUtx5X7JcBkZj4HEBHjwFbg6WX4WT/3m3XX8DTbx/b1xW9Wqbp+vvJ9Kyfzula6F5GZ7W4YcS2wJTP/uBl/BLg0M28+7rydwM5m+B7guy2VsA54uaW9+p296LAXHfaio9978auZec58Cz37bJnM3APsaXvfiDiQmSNt79uP7EWHveiwFx2Ve7EcT8scBTbOGm9o5iRJK2Q5wv0/gE0RcV5EnA5cD9y/DD9HkrSA1m/LZOZ0RNwM/Cszj0J+KTOfavvnvIXWb/X0MXvRYS867EVH2V60/g+qkqTe8x2qklSQ4S5JBZUN94jYFREZEet6XUsvRcRfRsQTEfF4RHwtIn6l1zX1SkTcFhHfafrxlYhY2+uaeiUirouIpyLiZxFR8lHAbiJiS0R8NyImI2Ks1/W0rWS4R8RG4HeA/+p1LaeA2zLzvZl5EfAA8Oc9rqeXHgIuzMz3Av8J3NLjenrpSeAPgK/3upBeWA0fk1Iy3IE7gE8Aq/5fizPzJ7OGZ7GKe5KZX8vM6Wa4n5n3YKxKmflMZrb1rvB+9ObHpGTm/wJvfExKGT17h+pyiYitwNHM/HZE9LqcU0JE/DVwI3AM2Nzjck4VfwTc3esi1DPrge/PGh8BLu1RLcuiL8M9Iv4N+OV5lm4FPsXMLZlV4636kZn3ZeatwK0RcQtwM/DpFS1wBXXrRXPOrcA0cNdK1rbSFtML1dWX4Z6ZvzXffEQMA+cBb1y1bwC+GRGXZOYPVrDEFbVQP+ZxF/AghcO9Wy8iYjvwQeCKLP4mjxP472I1Kv8xKX0Z7gvJzEPAL70xjojDwEhm9vOnvi1JRGzKzGeb4VbgO72sp5ea/4nMJ4DfzMzXel2PeurNj0lhJtSvB/6wtyW1q1S4a167I+I9wM+A54E/6XE9vfT3wBnAQ83f7PZn5qrsR0T8PvB3wDnAvoh4PDN/t8dlrZhT4GNSlp0fPyBJBVV9FFKSVjXDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaD/B/tepd0H3PK6AAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" } ], "source": [ @@ -306,20 +301,6 @@ } } }, - { - "cell_type": "code", - "execution_count": 12, - "outputs": [], - "source": [ - "#simulate.compute_nested_utilities(ru[0], ns[0])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "markdown", "source": [ @@ -339,7 +320,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 60, "outputs": [ { "name": "stderr", @@ -372,169 +353,12 @@ }, { "cell_type": "code", - "execution_count": 27, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] parent nest scale: 0\n", - "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] parent nest scale: 1.0\n", - "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] parent nest scale: 1.0\n", - "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] parent nest scale: 1.0\n", - "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] parent nest scale: 1.0\n" - ] - } - ], - "source": [ - "nest_spec = ns[0]\n", - "for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", - " nest.print()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 28, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "leaf DRIVEALONEFREE with 0.72, 0.72\n", - "leaf DRIVEALONEPAY with 0.72, 0.72\n", - "leaf SHARED2FREE with 0.72, 0.72\n", - "leaf SHARED2PAY with 0.72, 0.72\n", - "leaf SHARED3FREE with 0.72, 0.72\n", - "leaf SHARED3PAY with 0.72, 0.72\n", - "node AUTO with 0.72, 1.0\n", - "leaf WALK with 0.72, 0.72\n", - "leaf BIKE with 0.72, 0.72\n", - "node NONMOTORIZED with 0.72, 1.0\n", - "leaf WALK_LOC with 0.72, 0.72\n", - "leaf WALK_LRF with 0.72, 0.72\n", - "leaf WALK_EXP with 0.72, 0.72\n", - "leaf WALK_HVY with 0.72, 0.72\n", - "leaf WALK_COM with 0.72, 0.72\n", - "leaf DRIVE_LOC with 0.72, 0.72\n", - "leaf DRIVE_LRF with 0.72, 0.72\n", - "leaf DRIVE_EXP with 0.72, 0.72\n", - "leaf DRIVE_HVY with 0.72, 0.72\n", - "leaf DRIVE_COM with 0.72, 0.72\n", - "node TRANSIT with 0.72, 1.0\n", - "leaf TAXI with 0.36, 0.36\n", - "leaf TNC_SINGLE with 0.36, 0.36\n", - "leaf TNC_SHARED with 0.36, 0.36\n", - "node RIDEHAIL with 0.36, 1.0\n", - "node root with 1.0, 0\n" - ] - } - ], - "source": [ - "for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " print(f\"leaf {name} with {nest.product_of_coefficients}, {nest.parent_nest_scale}\")\n", - " else:\n", - " print(f\"node {name} with {nest.coefficient}, {nest.parent_nest_scale}\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 39, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nest name: DRIVEALONEFREE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'DRIVEALONEFREE'] parent nest scale: 0.72\n", - "Nest name: DRIVEALONEPAY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'DRIVEALONEPAY'] parent nest scale: 0.72\n", - "Nest name: SHARED2FREE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED2FREE'] parent nest scale: 0.72\n", - "Nest name: SHARED2PAY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED2PAY'] parent nest scale: 0.72\n", - "Nest name: SHARED3FREE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED3FREE'] parent nest scale: 0.72\n", - "Nest name: SHARED3PAY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO', 'SHARED3PAY'] parent nest scale: 0.72\n", - "Nest name: WALK level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'WALK'] parent nest scale: 0.72\n", - "Nest name: BIKE level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'BIKE'] parent nest scale: 0.72\n", - "Nest name: WALK_LOC level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_LOC'] parent nest scale: 0.72\n", - "Nest name: WALK_LRF level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_LRF'] parent nest scale: 0.72\n", - "Nest name: WALK_EXP level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_EXP'] parent nest scale: 0.72\n", - "Nest name: WALK_HVY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_HVY'] parent nest scale: 0.72\n", - "Nest name: WALK_COM level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'WALK_COM'] parent nest scale: 0.72\n", - "Nest name: DRIVE_LOC level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_LOC'] parent nest scale: 0.72\n", - "Nest name: DRIVE_LRF level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_LRF'] parent nest scale: 0.72\n", - "Nest name: DRIVE_EXP level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_EXP'] parent nest scale: 0.72\n", - "Nest name: DRIVE_HVY level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_HVY'] parent nest scale: 0.72\n", - "Nest name: DRIVE_COM level: 3 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT', 'DRIVE_COM'] parent nest scale: 0.72\n", - "Nest name: TAXI level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TAXI'] parent nest scale: 0.36\n", - "Nest name: TNC_SINGLE level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SINGLE'] parent nest scale: 0.36\n", - "Nest name: TNC_SHARED level: 3 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SHARED'] parent nest scale: 0.36\n" - ] - } - ], - "source": [ - "nest_spec = ns[0]\n", - "for nest in logit.each_nest(nest_spec, type='leaf'):\n", - " nest.print()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 49, "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 29, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.262023e-03 0.0 \n642446349 0.0 0.0 3.138876e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.339899e-02 0.0 \n1767182949 0.0 0.0 1.282607e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.103657e-14 0.0 0.039196 0.0 0.197050 0.759044 \n642446349 3.423474e-14 0.0 0.037574 0.0 0.208612 0.750231 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.288763e-13 0.0 0.038976 0.0 0.282647 0.660225 \n1767182949 1.256775e-13 0.0 0.034777 0.0 0.280877 0.666774 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 8.388831e-05 1.141304e-04 2.502967e-04 \n642446349 0.0 0.0 8.053321e-05 1.140970e-04 2.500402e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.247409e-03 2.795129e-03 7.103828e-04 \n1767182949 0.0 0.0 1.229459e-03 2.813489e-03 7.024828e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.262023e-030.04.103657e-140.00.0391960.00.1970500.7590440.00.00.00.00.00.00.00.08.388831e-051.141304e-042.502967e-04
6424463490.00.03.138876e-030.03.423474e-140.00.0375740.00.2086120.7502310.00.00.00.00.00.00.00.08.053321e-051.140970e-042.500402e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.339899e-020.01.288763e-130.00.0389760.00.2826470.6602250.00.00.00.00.00.00.00.01.247409e-032.795129e-037.103828e-04
17671829490.00.01.282607e-020.01.256775e-130.00.0347770.00.2808770.6667740.00.00.00.00.00.00.00.01.229459e-032.813489e-037.024828e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "# nested_probs[0] - these are relative in each nest, so the full probs are potentially just the product of these\n", - "base_probs[0]" + "# nest_spec = ns[0]\n", + "# for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", + "# nest.print()" ], "metadata": { "collapsed": false, @@ -545,226 +369,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 62, "outputs": [], "source": [ - "# for later: this seems to be a non-driving segment - grab workers or something later?\n", - "# for now: implement a run with seeding so I can reproduce these counts\n", - "\n", - "# actually, can I manipulate pipeline OR inject table dynamically? If I reproduce a single observation many times we\n", - "# can check mode shares\n", - "# 1) check tests\n", - "# 2) check code in mode choice, maybe duplicate here, see old code below" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": 40, - "outputs": [], - "source": [ - "def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples):\n", - " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", - " resume_after = \"trip_scheduling\"\n", - " model_name = \"trip_mode_choice\"\n", - " chunk_size = 0 # test_mtc means no chunking\n", - "\n", - " pipeline.open_pipeline(resume_after)\n", - " # preload any bulky injectables (e.g. skims) not in pipeline\n", - " inject.get_injectable('preload_injectables', None)\n", - " pipeline._PIPELINE.rng().begin_step(model_name)\n", - " #step_name = model_name\n", - " args = {}\n", - " #checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - " inject.set_step_args(args)\n", - "\n", - " trips = inject.get_table('trips')\n", - " tours_merged = inject.get_table('tours_merged')\n", - " network_los = inject.get_injectable('network_los')\n", - "\n", - " trace_label = 'trip_mode_choice'\n", - " model_settings_file_name = 'trip_mode_choice.yaml'\n", - " model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - " mode_column_name = 'trip_mode'\n", - "\n", - " trips_df = trips.to_frame()\n", - " print(\"Running with %d trips\", trips_df.shape[0])\n", - "\n", - " tours_merged = tours_merged.to_frame()\n", - " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "\n", - " # - trips_merged - merge trips and tours_merged\n", - " trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - " assert trips_merged.index.equals(trips.index)\n", - "\n", - " # setup skim keys\n", - " assert ('trip_period' not in trips_merged)\n", - " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - " orig_col = 'origin'\n", - " dest_col = 'destination'\n", - "\n", - " constants = {}\n", - " constants.update(config.get_model_constants(model_settings))\n", - " constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - " })\n", - "\n", - " skim_dict = network_los.get_default_skim_dict()\n", - "\n", - " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - " skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - " }\n", - "\n", - " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - " nest_specs = config.get_logit_model_settings(model_settings)\n", - "\n", - " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - " #choices_list = []\n", - "\n", - "\n", - " # grab one, duplicate num_samples times\n", - " trips_segment = trips_merged.loc[trips_merged.index == trip_id_to_check].copy()\n", - " primary_purpose = trips_segment['primary_purpose'].values[0]\n", - " trips_segment = trips_segment.loc[trips_segment.index.repeat(num_samples)]\n", - "\n", - " # need to add new row_states for rng here, need to ensure there are no collisions with existing keys\n", - " existing_indexes = pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values\n", - " num_new_indexes = trips_segment.shape[0]\n", - " new_indexes = np.arange(existing_indexes.max()+1, existing_indexes.max() + num_new_indexes + 1)\n", - "\n", - " trips_segment.index = new_indexes #+= np.arange(num_samples)\n", - " # name index so tracing knows how to slice\n", - " trips_segment.index.name = 'trip_id'\n", - "\n", - "\n", - " # #logger.warning(\"Change seeding back when done with testing\")\n", - " # pipeline._PIPELINE.rng.row_states = pd.DataFrame(columns=['row_seed', 'offset'], index=trips_segment.index)\n", - " # pipeline._PIPELINE.rng.row_states[\"row_seed\"] = trips_segment.index.values\n", - " # pipeline._PIPELINE.rng.row_states[\"offset\"] = 0\n", - " pipeline._PIPELINE.rng().add_channel(\"trips\", trips_segment)\n", - "\n", - "\n", - " #for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", - " # continue\n", - "\n", - " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " (primary_purpose, len(trips_segment.index), ))\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " ################ Replace wrapper function\n", - " # choices = mode_choice_simulate(...)\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", - " choices = simulate.simple_simulate(\n", - " choosers=trips_segment,\n", - " spec=spec,\n", - " nest_spec=nest_spec,\n", - " skims=skims,\n", - " locals_d=locals_dict,\n", - " chunk_size=chunk_size,\n", - " want_logsums=logsum_column_name is not None,\n", - " trace_label=segment_trace_label,\n", - " trace_choice_name='trip_mode_choice',\n", - " estimator=estimator,\n", - " trace_column_names=None,\n", - " choose_individual_max_utility=choose_individual_max_utility)\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " if not choose_individual_max_utility:\n", - " alts = spec.columns\n", - " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", - " ################\n", - " #choices_list.append(choices)\n", - " #choices_df_asim = pd.concat(choices_list)\n", - "\n", - " # update trips table with choices (and potionally logssums)\n", - " #trips_df = trips_merged.copy() # trips.to_frame()\n", - "\n", - " #if (do_these_purposes is not None):\n", - " # trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", - "\n", - " #assign_in_place(trips_df, choices)\n", - " #assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - " finalise = True\n", - " if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - " print(\"Done\")\n", - " return trips_merged, choices\n", - "\n", - "\n", - "def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check):\n", - " t_, c_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility,\n", - " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", - "\n", - " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", - " #sim_mode_shares.columns = [\"mode_share_sim\"]\n", - " obs_probs = base_probs[0].loc[base_probs[0].index == trip_id_to_check].T\n", - " obs_probs.columns = [\"mode_share_obs\"]\n", - " ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how=\"outer\").fillna(0)\n", - " ms_comp[\"diff\"] = ms_comp[\"trip_mode\"] - ms_comp[\"mode_share_obs\"]\n", - " ms_comp[\"rel_diff\"] = ms_comp[\"diff\"] / ms_comp[\"mode_share_obs\"]\n", - " #ms_comp.style.format('{:.2}')\n", - " #with pd.option_context(\"precision\", 3):\n", - " # display(ms_comp)\n", - " display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", - " return ms_comp" + "#base_probs[0]" ], "metadata": { "collapsed": false, @@ -775,7 +383,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 63, "outputs": [ { "name": "stderr", @@ -796,8 +404,16 @@ }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004262 4.103657e-14 0.000084 0.00025 0.000114 \ntrip_mode 0.004300 0.000000e+00 0.000100 0.00050 0.000100 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.039196 0.19705 0.759044 \ntrip_mode 0.039100 0.20000 0.755900 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0042624.103657e-140.0000840.000250.0001140.0391960.197050.759044
trip_mode0.0043000.000000e+000.0001000.000500.0001000.0391000.200000.755900
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.004800 0.000000e+00 0.000000 0.000500 0.000200 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.044300 0.060600 0.889600 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0048000.000000e+000.0000000.0005000.0002000.0443000.0606000.889600
\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.004800 0.000000e+00 0.000000 0.000500 0.000200 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.044300 0.060600 0.889600 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0048000.000000e+000.0000000.0005000.0002000.0443000.0606000.889600
\n
" }, "metadata": {}, "output_type": "display_data" @@ -807,7 +423,8 @@ "choose_individual_max_utility = False\n", "num_samples = 10000\n", "trip_id_to_check = 642446345\n", - "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" + "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)\n", + "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ], "metadata": { "collapsed": false, @@ -840,14 +457,14 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (100000 trips)\n" + "trip_mode_choice tour_type 'social' (10000 trips)\n" ] } ], "source": [ "choose_individual_max_utility = True\n", - "num_samples = 100000 # 1e5 takes about 25s\n", - "trip_id_to_check = 642446345 # 2464104885 # 642446345 # 1767182945\n", + "num_samples = 10000 # 1e5 takes about 25s\n", + "trip_id_to_check = 1767182945 # 2464104885 # 642446345 # 1767182945\n", "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" ] }, From 7c21407cfd2d667a3bf45f7c3f5ea5a830ca3750 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 6 May 2022 13:27:26 +1000 Subject: [PATCH 015/135] adds nested utility scale --- activitysim/core/logit.py | 3 + activitysim/core/simulate.py | 8 +- notebooks/validate_frozen_impl.ipynb | 137 +++++++++++++++++++-------- 3 files changed, 105 insertions(+), 43 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 048b37da42..0e7feb33a1 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -372,6 +372,7 @@ def __init__(self, name=None, level=0): self.ancestors = [] self.alternatives = None self.coefficient = 0 + self.parent_scale = 1 def print(self): print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s " % @@ -446,6 +447,7 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients * coefficient nest.alternatives = alternatives nest.ancestors = parent_nest.ancestors + [name] + nest.parent_scale = parent_nest.coefficient if pre_order: yield spec, nest @@ -466,6 +468,7 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients nest.ancestors = parent_nest.ancestors + [name] nest.coefficient = parent_nest.coefficient + nest.parent_scale = parent_nest.coefficient yield spec, nest diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index add013e668..b3e64abb65 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -746,8 +746,12 @@ def compute_nested_utilities(raw_utilities, nest_spec): np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1) ) - for nest in logit.each_nest(nest_spec, type="leaf"): - nested_utilities[nest.name] /= nest.coefficient + + # calculate scaled utilties, as would be used for probability calculations + for nest in logit.each_nest(nest_spec): + if nest.level == 1: + continue # root does not have parent + nested_utilities[nest.name] /= nest.parent_scale return nested_utilities diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 58b7fae0c4..4fd1c37a86 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 88, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -73,12 +73,12 @@ "from activitysim.cli import run\n", "from activitysim.core import inject\n", "\n", - "from fru_utils import run_trip_mode_choice, mode_choice_for_trip" + "from fru_utils import run_trip_mode_choice, comp_mode_shares" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 89, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -97,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 90, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -117,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 91, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -136,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 92, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -163,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 93, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -212,8 +212,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Done\n", - "Running with %d trips 482\n" + "Done\n" ] }, { @@ -227,6 +226,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", @@ -237,25 +237,28 @@ "trip_mode_choice tour_type 'social' (19 trips)\n", "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n" + "Done\n", + "CPU times: user 21.5 s, sys: 2.27 s, total: 23.8 s\n", + "Wall time: 21.1 s\n" ] } ], "source": [ + "%%time\n", "trips_df_ru = run_trip_mode_choice()\n", "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 94, "outputs": [ { "data": { - "text/plain": " trip_mode_fru trip_mode_asim\nWALK 332 304\nWALK_LOC 72 85\nWALK_LRF 45 53\nBIKE 16 17\nDRIVEALONEFREE 7 4\nWALK_HVY 4 4\nTNC_SINGLE 3 4\nSHARED2FREE 2 4\nTNC_SHARED 1 4", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK332304
WALK_LOC7285
WALK_LRF4553
BIKE1617
DRIVEALONEFREE74
WALK_HVY44
TNC_SINGLE34
SHARED2FREE24
TNC_SHARED14
\n
" + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 297 304\nWALK_LOC 100 85\nWALK_LRF 52 53\nBIKE 16 17\nDRIVEALONEFREE 7 4\nWALK_HVY 4 4\nTNC_SINGLE 3 4\nSHARED2FREE 2 4\nTNC_SHARED 1 4", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK297304
WALK_LOC10085
WALK_LRF5253
BIKE1617
DRIVEALONEFREE74
WALK_HVY44
TNC_SINGLE34
SHARED2FREE24
TNC_SHARED14
\n
" }, - "execution_count": 58, + "execution_count": 94, "metadata": {}, "output_type": "execute_result" } @@ -275,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 95, "outputs": [ { "data": { @@ -301,6 +304,30 @@ } } }, + { + "cell_type": "code", + "execution_count": 96, + "outputs": [ + { + "data": { + "text/plain": "trip_id\n535694417 0.143382\n943749469 0.109040\n1004301761 0.122286\n1767182865 0.119168\n1767183297 0.138448\n1767186581 0.111968\ndtype: float64" + }, + "execution_count": 96, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x_ = np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim)\n", + "x_.loc[x_ > 0.1]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "markdown", "source": [ @@ -320,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 104, "outputs": [ { "name": "stderr", @@ -353,12 +380,29 @@ }, { "cell_type": "code", - "execution_count": 49, - "outputs": [], + "execution_count": 105, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] \n", + "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] \n", + "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] \n", + "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2'] \n", + "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3'] \n", + "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] \n", + "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] \n", + "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS'] \n", + "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS'] \n", + "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] \n" + ] + } + ], "source": [ - "# nest_spec = ns[0]\n", - "# for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", - "# nest.print()" + "nest_spec = ns[0]\n", + "for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", + " nest.print()" ], "metadata": { "collapsed": false, @@ -369,10 +413,20 @@ }, { "cell_type": "code", - "execution_count": 62, - "outputs": [], + "execution_count": 106, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#base_probs[0]" + "base_probs[0]" ], "metadata": { "collapsed": false, @@ -383,7 +437,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 100, "outputs": [ { "name": "stderr", @@ -404,16 +458,8 @@ }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.004800 0.000000e+00 0.000000 0.000500 0.000200 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.044300 0.060600 0.889600 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0048000.000000e+000.0000000.0005000.0002000.0443000.0606000.889600
\n
" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.004800 0.000000e+00 0.000000 0.000500 0.000200 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.044300 0.060600 0.889600 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0048000.000000e+000.0000000.0005000.0002000.0443000.0606000.889600
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED \\\nmode_share_obs 0.000022 2.084422e-16 1.325277e-07 3.433307e-07 \ntrip_mode 0.000000 0.000000e+00 0.000000e+00 0.000000e+00 \n\n TNC_SINGLE WALK WALK_LOC \nmode_share_obs 8.507634e-12 0.627973 0.372005 \ntrip_mode 0.000000e+00 0.620900 0.379100 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOC
mode_share_obs0.0000222.084422e-161.325277e-073.433307e-078.507634e-120.6279730.372005
trip_mode0.0000000.000000e+000.000000e+000.000000e+000.000000e+000.6209000.379100
\n
" }, "metadata": {}, "output_type": "display_data" @@ -422,7 +468,7 @@ "source": [ "choose_individual_max_utility = False\n", "num_samples = 10000\n", - "trip_id_to_check = 642446345\n", + "trip_id_to_check = 2464104885\n", "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)\n", "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ], @@ -435,12 +481,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 107, "id": "1fcdc0ff-c510-4878-8672-8365e046442c", "metadata": { "pycharm": { - "name": "#%%\n", - "is_executing": true + "name": "#%%\n" } }, "outputs": [ @@ -457,15 +502,25 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (10000 trips)\n" + "trip_mode_choice tour_type 'social' (10000 trips)\n", + "Done\n" ] + }, + { + "data": { + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.016101 1.548643e-13 0.001499 0.000854 0.003359 \ntrip_mode 0.000000 0.000000e+00 0.000000 0.000000 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.046835 0.144256 0.787096 \ntrip_mode 0.012600 0.223600 0.763800 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0161011.548643e-130.0014990.0008540.0033590.0468350.1442560.787096
trip_mode0.0000000.000000e+000.0000000.0000000.0000000.0126000.2236000.763800
\n
" + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ "choose_individual_max_utility = True\n", "num_samples = 10000 # 1e5 takes about 25s\n", "trip_id_to_check = 1767182945 # 2464104885 # 642446345 # 1767182945\n", - "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)" + "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)\n", + "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ] }, { From 932d5adc13776a1f2d7a44381ac46cbe184fa529 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 6 May 2022 16:41:13 +1000 Subject: [PATCH 016/135] scaling in recursive nest utilities --- activitysim/core/simulate.py | 41 ++- notebooks/validate_frozen_impl.ipynb | 411 +++++++++++++++++++++++---- 2 files changed, 386 insertions(+), 66 deletions(-) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index b3e64abb65..7bdb5ed7e2 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -743,15 +743,10 @@ def compute_nested_utilities(raw_utilities, nest_spec): with np.errstate(divide='ignore'): nested_utilities[name] = \ nest.coefficient * np.log( - np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1) - ) + np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) - - # calculate scaled utilties, as would be used for probability calculations - for nest in logit.each_nest(nest_spec): - if nest.level == 1: - continue # root does not have parent - nested_utilities[nest.name] /= nest.parent_scale + if nest.level > 1: # root does not have parent. also, parent_scale seems to be 0. + nested_utilities[name] = nested_utilities[name] / nest.parent_scale return nested_utilities @@ -1042,6 +1037,36 @@ def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estima tracing.trace_df(nested_utilities, '%s.nested_utilities' % trace_label, column_labels=['alternative', 'utility']) + + ########### comparison only + # probabilities of alternatives relative to siblings sharing the same nest + nested_probabilities = \ + compute_nested_probabilities(np.exp(nested_utilities), nest_spec, trace_label=trace_label) + chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) + + if want_logsums: + # logsum of nest root + logsums = pd.Series(nested_utilities.root, index=choosers.index) + chunk.log_df(trace_label, "logsums", logsums) + + if have_trace_targets: + tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, + column_labels=['alternative', 'probability']) + + base_probabilities = compute_base_probabilities(nested_probabilities, nest_spec, spec) + chunk.log_df(trace_label, "base_probabilities", base_probabilities) + + del nested_probabilities + chunk.log_df(trace_label, "nested_probabilities", None) + + if have_trace_targets: + tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label, + column_labels=['alternative', 'probability']) + + del(base_probabilities) + chunk.log_df(trace_label, "base_probabilities", None) + ########### end comparison only + # TODO - check this is correct if want_logsums: # logsum of nest root diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 4fd1c37a86..ec136d150d 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 191, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "execution": { @@ -22,14 +22,23 @@ }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 192, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "execution": { @@ -48,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 193, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -78,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 194, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -97,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 195, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -117,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 196, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -136,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 197, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -163,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 198, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -205,20 +214,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -226,6 +222,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Done\n", "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", @@ -238,8 +235,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 21.5 s, sys: 2.27 s, total: 23.8 s\n", - "Wall time: 21.1 s\n" + "CPU times: user 23.4 s, sys: 2.09 s, total: 25.5 s\n", + "Wall time: 24 s\n" ] } ], @@ -251,14 +248,14 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 184, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 297 304\nWALK_LOC 100 85\nWALK_LRF 52 53\nBIKE 16 17\nDRIVEALONEFREE 7 4\nWALK_HVY 4 4\nTNC_SINGLE 3 4\nSHARED2FREE 2 4\nTNC_SHARED 1 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK297304
WALK_LOC10085
WALK_LRF5253
BIKE1617
DRIVEALONEFREE74
WALK_HVY44
TNC_SINGLE34
SHARED2FREE24
TNC_SHARED14
\n
" }, - "execution_count": 94, + "execution_count": 184, "metadata": {}, "output_type": "execute_result" } @@ -278,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 185, "outputs": [ { "data": { @@ -306,13 +303,97 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 200, + "outputs": [ + { + "data": { + "text/plain": "trip_id\n106170305 4.050638\n106170309 0.553496\n1767013721 0.482067\n535694461 0.251889\n943749469 0.203583\n ... \n535672921 0.000000\n535620049 0.000000\n535388485 0.000000\n535388481 0.000000\n943749471 0.000000\nLength: 482, dtype: float64" + }, + "execution_count": 200, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ls_ = np.abs(((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru))\n", + "ls_.sort_values(ascending=False)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 201, + "outputs": [ + { + "data": { + "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n106170305 WALK_LRF -0.002341 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n106170305 -0.011825 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
106170305WALK_LRF-0.002341WALK_LRF-0.011825
\n
" + }, + "execution_count": 201, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips.loc[trips.index == 106170305]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 210, + "outputs": [ + { + "data": { + "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n86627409 WALK_LRF 2.435199 WALK_LRF \n86627413 WALK_LRF 2.665463 WALK_LRF \n86673661 WALK_LRF 1.842576 WALK_LRF \n106170305 WALK_LRF -0.002341 WALK_LRF \n106170309 WALK_LRF -0.017607 WALK_LRF \n... ... ... ... \n1767186253 WALK_LRF 0.976319 WALK_LRF \n1767186581 WALK_HVY 1.712704 WALK_LRF \n1768237161 WALK 6.262944 WALK \n1768237167 WALK_LRF 11.191330 WALK \n2473024477 WALK_LRF 0.637076 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n86627409 2.407248 \n86627413 2.642707 \n86673661 1.834538 \n106170305 -0.011825 \n106170309 -0.027353 \n... ... \n1767186253 0.880819 \n1767186581 1.600736 \n1768237161 6.226927 \n1768237167 11.144496 \n2473024477 0.624304 \n\n[74 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
86627409WALK_LRF2.435199WALK_LRF2.407248
86627413WALK_LRF2.665463WALK_LRF2.642707
86673661WALK_LRF1.842576WALK_LRF1.834538
106170305WALK_LRF-0.002341WALK_LRF-0.011825
106170309WALK_LRF-0.017607WALK_LRF-0.027353
...............
1767186253WALK_LRF0.976319WALK_LRF0.880819
1767186581WALK_HVY1.712704WALK_LRF1.600736
1768237161WALK6.262944WALK6.226927
1768237167WALK_LRF11.191330WALK11.144496
2473024477WALK_LRF0.637076WALK_LRF0.624304
\n

74 rows × 4 columns

\n
" + }, + "execution_count": 210, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 141, "outputs": [ { "data": { "text/plain": "trip_id\n535694417 0.143382\n943749469 0.109040\n1004301761 0.122286\n1767182865 0.119168\n1767183297 0.138448\n1767186581 0.111968\ndtype: float64" }, - "execution_count": 96, + "execution_count": 141, "metadata": {}, "output_type": "execute_result" } @@ -347,7 +428,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 207, "outputs": [ { "name": "stderr", @@ -362,14 +443,23 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", + "trip_mode_choice tour_type 'atwork' (27 trips)\n", + "trip_mode_choice tour_type 'eatout' (33 trips)\n", + "trip_mode_choice tour_type 'escort' (6 trips)\n", + "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", + "trip_mode_choice tour_type 'othmaint' (46 trips)\n", + "trip_mode_choice tour_type 'school' (37 trips)\n", + "trip_mode_choice tour_type 'shopping' (77 trips)\n", "trip_mode_choice tour_type 'social' (19 trips)\n", + "trip_mode_choice tour_type 'univ' (26 trips)\n", + "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n" ] } ], "source": [ "from fru_utils import get_stuff\n", - "t, ru, neu, nu, ns, nested_probs, base_probs = get_stuff(do_these_purposes=[\"social\"])" + "t, ru, neu, nu, ns, nested_probs, base_probs = get_stuff() #do_these_purposes=[\"social\"]) #social" ], "metadata": { "collapsed": false, @@ -380,29 +470,20 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 209, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] \n", - "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] \n", - "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] \n", - "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2'] \n", - "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3'] \n", - "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] \n", - "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] \n", - "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS'] \n", - "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS'] \n", - "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] \n" - ] + "data": { + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n106170305 323689 256660 work 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n106170305 1 1 10 13271288 work \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n106170305 NaN 11.0 WALK_LRF -0.011825 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
106170305323689256660work1True111013271288workNaN11.0WALK_LRF-0.011825
\n
" + }, + "execution_count": 209, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "nest_spec = ns[0]\n", - "for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", - " nest.print()" + "t.loc[t.index == 106170305]" ], "metadata": { "collapsed": false, @@ -413,20 +494,48 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 187, + "outputs": [], + "source": [ + "# nest_spec = ns[0]\n", + "# for nest in logit.each_nest(nest_spec, post_order=False):\n", + "# print(f\"{nest.level}, {nest.name}, {nest.coefficient}, parent scale: {nest.parent_scale}\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 188, "outputs": [ { "data": { "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" }, - "execution_count": 106, + "execution_count": 188, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "base_probs[0]" + "base_probs[0] #615236801" ], "metadata": { "collapsed": false, @@ -437,7 +546,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 189, "outputs": [ { "name": "stderr", @@ -481,7 +590,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 190, "id": "1fcdc0ff-c510-4878-8672-8365e046442c", "metadata": { "pycharm": { @@ -508,8 +617,8 @@ }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.016101 1.548643e-13 0.001499 0.000854 0.003359 \ntrip_mode 0.000000 0.000000e+00 0.000000 0.000000 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.046835 0.144256 0.787096 \ntrip_mode 0.012600 0.223600 0.763800 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0161011.548643e-130.0014990.0008540.0033590.0468350.1442560.787096
trip_mode0.0000000.000000e+000.0000000.0000000.0000000.0126000.2236000.763800
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED \\\nmode_share_obs 0.000022 2.084422e-16 1.325277e-07 3.433307e-07 \ntrip_mode 0.000000 0.000000e+00 0.000000e+00 0.000000e+00 \n\n TNC_SINGLE WALK WALK_LOC \nmode_share_obs 8.507634e-12 0.627973 0.372005 \ntrip_mode 0.000000e+00 0.673100 0.326900 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOC
mode_share_obs0.0000222.084422e-161.325277e-073.433307e-078.507634e-120.6279730.372005
trip_mode0.0000000.000000e+000.000000e+000.000000e+000.000000e+000.6731000.326900
\n
" }, "metadata": {}, "output_type": "display_data" @@ -518,11 +627,197 @@ "source": [ "choose_individual_max_utility = True\n", "num_samples = 10000 # 1e5 takes about 25s\n", - "trip_id_to_check = 1767182945 # 2464104885 # 642446345 # 1767182945\n", + "trip_id_to_check = 2464104885\n", + "# work 615236801\n", + "# social 2464104885 642446345 1767182945\n", "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)\n", "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ] }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 160, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# stuff\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 172, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [982875].\n" + ] + } + ], + "source": [ + "from activitysim.core import pipeline\n", + "\n", + "resume_after = \"trip_scheduling\"\n", + "model_name = \"trip_mode_choice\"\n", + "chunk_size = 0 # test_mtc means no chunking\n", + "\n", + "pipeline.open_pipeline(resume_after)\n", + "# preload any bulky injectables (e.g. skims) not in pipeline\n", + "inject.get_injectable('preload_injectables', None)\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "#step_name = model_name\n", + "args = {}\n", + "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 176, + "outputs": [], + "source": [ + "t_ = trips.to_frame()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 177, + "outputs": [ + { + "data": { + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n2464104885 7512514 2821179 social 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n2464104885 1 8 9 308013110 home \n\n destination_logsum depart \ntrip_id \n2464104885 NaN 16.0 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdepart
trip_id
246410488575125142821179social1False189308013110homeNaN16.0
\n
" + }, + "execution_count": 177, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t_.loc[t_.index == 2464104885]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 1286183cd2f34dda168baeaac2a6f84d25bc3ed6 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 6 May 2022 16:57:43 +1000 Subject: [PATCH 017/135] output clean up --- notebooks/fru_utils.py | 2 +- notebooks/validate_frozen_impl.ipynb | 219 ++++++++++----------------- 2 files changed, 82 insertions(+), 139 deletions(-) diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index 935e8b0289..5f616e8ecf 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -199,7 +199,7 @@ def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, tri sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0] #sim_mode_shares.columns = ["mode_share_sim"] - obs_probs = base_probs[0].loc[base_probs[0].index == trip_id_to_check].T + obs_probs = base_probs.loc[base_probs.index == trip_id_to_check].T obs_probs.columns = ["mode_share_obs"] ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how="outer").fillna(0) ms_comp["diff"] = ms_comp["trip_mode"] - ms_comp["mode_share_obs"] diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index ec136d150d..d7e2ab1905 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 224, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -87,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 225, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 226, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -126,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 227, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -145,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 228, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -172,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 198, + "execution_count": 229, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -207,7 +207,8 @@ "trip_mode_choice tour_type 'shopping' (77 trips)\n", "trip_mode_choice tour_type 'social' (19 trips)\n", "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n" + "trip_mode_choice tour_type 'work' (168 trips)\n", + "Done\n" ] }, { @@ -222,7 +223,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Done\n", "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", @@ -235,8 +235,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 23.4 s, sys: 2.09 s, total: 25.5 s\n", - "Wall time: 24 s\n" + "CPU times: user 25.2 s, sys: 2.31 s, total: 27.5 s\n", + "Wall time: 26.5 s\n" ] } ], @@ -248,14 +248,14 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 231, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 297 304\nWALK_LOC 100 85\nWALK_LRF 52 53\nBIKE 16 17\nDRIVEALONEFREE 7 4\nWALK_HVY 4 4\nTNC_SINGLE 3 4\nSHARED2FREE 2 4\nTNC_SHARED 1 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK297304
WALK_LOC10085
WALK_LRF5253
BIKE1617
DRIVEALONEFREE74
WALK_HVY44
TNC_SINGLE34
SHARED2FREE24
TNC_SHARED14
\n
" }, - "execution_count": 184, + "execution_count": 231, "metadata": {}, "output_type": "execute_result" } @@ -275,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 232, "outputs": [ { "data": { @@ -303,13 +303,13 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 233, "outputs": [ { "data": { "text/plain": "trip_id\n106170305 4.050638\n106170309 0.553496\n1767013721 0.482067\n535694461 0.251889\n943749469 0.203583\n ... \n535672921 0.000000\n535620049 0.000000\n535388485 0.000000\n535388481 0.000000\n943749471 0.000000\nLength: 482, dtype: float64" }, - "execution_count": 200, + "execution_count": 233, "metadata": {}, "output_type": "execute_result" } @@ -327,14 +327,14 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 234, "outputs": [ { "data": { - "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n106170305 WALK_LRF -0.002341 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n106170305 -0.011825 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
106170305WALK_LRF-0.002341WALK_LRF-0.011825
\n
" + "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n615236801 WALK_LOC 7.627291 WALK_LOC \n\n mode_choice_logsum_asim \ntrip_id \n615236801 7.627291 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
615236801WALK_LOC7.627291WALK_LOC7.627291
\n
" }, - "execution_count": 201, + "execution_count": 234, "metadata": {}, "output_type": "execute_result" } @@ -385,30 +385,6 @@ } } }, - { - "cell_type": "code", - "execution_count": 141, - "outputs": [ - { - "data": { - "text/plain": "trip_id\n535694417 0.143382\n943749469 0.109040\n1004301761 0.122286\n1767182865 0.119168\n1767183297 0.138448\n1767186581 0.111968\ndtype: float64" - }, - "execution_count": 141, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x_ = np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim)\n", - "x_.loc[x_ > 0.1]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "markdown", "source": [ @@ -428,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 207, + "execution_count": 235, "outputs": [ { "name": "stderr", @@ -470,20 +446,20 @@ }, { "cell_type": "code", - "execution_count": 209, + "execution_count": 236, "outputs": [ { "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n106170305 323689 256660 work 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n106170305 1 1 10 13271288 work \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n106170305 NaN 11.0 WALK_LRF -0.011825 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
106170305323689256660work1True111013271288workNaN11.0WALK_LRF-0.011825
\n
" + "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n86627409 WALK_LRF 2.435199 WALK_LRF \n86627413 WALK_LRF 2.665463 WALK_LRF \n86673661 WALK_LRF 1.842576 WALK_LRF \n106170305 WALK_LRF -0.002341 WALK_LRF \n106170309 WALK_LRF -0.017607 WALK_LRF \n... ... ... ... \n1767186253 WALK_LRF 0.976319 WALK_LRF \n1767186581 WALK_HVY 1.712704 WALK_LRF \n1768237161 WALK 6.262944 WALK \n1768237167 WALK_LRF 11.191330 WALK \n2473024477 WALK_LRF 0.637076 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n86627409 2.407248 \n86627413 2.642707 \n86673661 1.834538 \n106170305 -0.011825 \n106170309 -0.027353 \n... ... \n1767186253 0.880819 \n1767186581 1.600736 \n1768237161 6.226927 \n1768237167 11.144496 \n2473024477 0.624304 \n\n[74 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
86627409WALK_LRF2.435199WALK_LRF2.407248
86627413WALK_LRF2.665463WALK_LRF2.642707
86673661WALK_LRF1.842576WALK_LRF1.834538
106170305WALK_LRF-0.002341WALK_LRF-0.011825
106170309WALK_LRF-0.017607WALK_LRF-0.027353
...............
1767186253WALK_LRF0.976319WALK_LRF0.880819
1767186581WALK_HVY1.712704WALK_LRF1.600736
1768237161WALK6.262944WALK6.226927
1768237167WALK_LRF11.191330WALK11.144496
2473024477WALK_LRF0.637076WALK_LRF0.624304
\n

74 rows × 4 columns

\n
" }, - "execution_count": 209, + "execution_count": 236, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "t.loc[t.index == 106170305]" + "trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" ], "metadata": { "collapsed": false, @@ -494,24 +470,20 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" + "execution_count": 237, + "outputs": [ + { + "data": { + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n86627409 264107 226869 work 1 True \n86627413 264107 226869 work 1 False \n86673661 264248 226939 work 1 False \n106170305 323689 256660 work 1 True \n106170309 323689 256660 work 1 False \n... ... ... ... ... ... \n1767186253 5387762 2223027 work 1 False \n1767186581 5387763 2223027 work 1 False \n1768237161 5390966 2224629 work 1 True \n1768237167 5390966 2224629 work 3 False \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n86627409 1 24 9 10828426 work \n86627413 1 9 24 10828426 home \n86673661 1 10 22 10834207 home \n106170305 1 1 10 13271288 work \n106170309 1 10 1 13271288 home \n... ... ... ... ... ... \n1767186253 1 9 4 220898281 home \n1767186581 2 9 14 220898322 othmaint \n1768237161 1 16 17 221029645 work \n1768237167 3 17 7 221029645 home \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n86627409 NaN 7.0 WALK_LRF 2.407248 \n86627413 NaN 19.0 WALK_LRF 2.642707 \n86673661 NaN 19.0 WALK_LRF 1.834538 \n106170305 NaN 11.0 WALK_LRF -0.011825 \n106170309 NaN 18.0 WALK_LRF -0.027353 \n... ... ... ... ... \n1767186253 NaN 17.0 WALK_LRF 0.880819 \n1767186581 32.045096 17.0 WALK_LRF 1.600736 \n1768237161 NaN 8.0 WALK 6.226927 \n1768237167 NaN 19.0 WALK 11.144496 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[74 rows x 14 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86627409264107226869work1True124910828426workNaN7.0WALK_LRF2.407248
86627413264107226869work1False192410828426homeNaN19.0WALK_LRF2.642707
86673661264248226939work1False1102210834207homeNaN19.0WALK_LRF1.834538
106170305323689256660work1True111013271288workNaN11.0WALK_LRF-0.011825
106170309323689256660work1False110113271288homeNaN18.0WALK_LRF-0.027353
.............................................
176718625353877622223027work1False194220898281homeNaN17.0WALK_LRF0.880819
176718658153877632223027work1False2914220898322othmaint32.04509617.0WALK_LRF1.600736
176823716153909662224629work1True11617221029645workNaN8.0WALK6.226927
176823716753909662224629work3False3177221029645homeNaN19.0WALK11.144496
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

74 rows × 14 columns

\n
" + }, + "execution_count": 237, + "metadata": {}, + "output_type": "execute_result" } - } - }, - { - "cell_type": "code", - "execution_count": 187, - "outputs": [], + ], "source": [ - "# nest_spec = ns[0]\n", - "# for nest in logit.each_nest(nest_spec, post_order=False):\n", - "# print(f\"{nest.level}, {nest.name}, {nest.coefficient}, parent scale: {nest.parent_scale}\")" + "t.loc[t.index.isin(trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0].index.values)]" ], "metadata": { "collapsed": false, @@ -522,20 +494,36 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 238, "outputs": [ { "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n615236801 1875721 982875 work 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n615236801 1 10 16 76904600 work \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n615236801 NaN 8.0 WALK_LOC 7.627291 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
6152368011875721982875work1True1101676904600workNaN8.0WALK_LOC7.627291
\n
" }, - "execution_count": 188, + "execution_count": 238, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "base_probs[0] #615236801" + "t.loc[t.index == 615236801]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 187, + "outputs": [], + "source": [ + "# nest_spec = ns[0]\n", + "# for nest in logit.each_nest(nest_spec, post_order=False):\n", + "# print(f\"{nest.level}, {nest.name}, {nest.coefficient}, parent scale: {nest.parent_scale}\")" ], "metadata": { "collapsed": false, @@ -546,40 +534,20 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 220, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (10000 trips)\n", - "Done\n" - ] - }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED \\\nmode_share_obs 0.000022 2.084422e-16 1.325277e-07 3.433307e-07 \ntrip_mode 0.000000 0.000000e+00 0.000000e+00 0.000000e+00 \n\n TNC_SINGLE WALK WALK_LOC \nmode_share_obs 8.507634e-12 0.627973 0.372005 \ntrip_mode 0.000000e+00 0.620900 0.379100 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOC
mode_share_obs0.0000222.084422e-161.325277e-073.433307e-078.507634e-120.6279730.372005
trip_mode0.0000000.000000e+000.000000e+000.000000e+000.000000e+000.6209000.379100
\n
" + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" }, + "execution_count": 220, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "choose_individual_max_utility = False\n", - "num_samples = 10000\n", - "trip_id_to_check = 2464104885\n", - "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)\n", - "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" + "base_probs[7] #615236801" ], "metadata": { "collapsed": false, @@ -590,13 +558,7 @@ }, { "cell_type": "code", - "execution_count": 190, - "id": "1fcdc0ff-c510-4878-8672-8365e046442c", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "execution_count": null, "outputs": [ { "name": "stderr", @@ -611,50 +573,31 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (10000 trips)\n", - "Done\n" + "trip_mode_choice tour_type 'work' (10000 trips)\n" ] - }, - { - "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED \\\nmode_share_obs 0.000022 2.084422e-16 1.325277e-07 3.433307e-07 \ntrip_mode 0.000000 0.000000e+00 0.000000e+00 0.000000e+00 \n\n TNC_SINGLE WALK WALK_LOC \nmode_share_obs 8.507634e-12 0.627973 0.372005 \ntrip_mode 0.000000e+00 0.673100 0.326900 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOC
mode_share_obs0.0000222.084422e-161.325277e-073.433307e-078.507634e-120.6279730.372005
trip_mode0.0000000.000000e+000.000000e+000.000000e+000.000000e+000.6731000.326900
\n
" - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ - "choose_individual_max_utility = True\n", - "num_samples = 10000 # 1e5 takes about 25s\n", - "trip_id_to_check = 2464104885\n", - "# work 615236801\n", + "num_samples = 10000\n", + "trip_id_to_check = 615236801 # 1e5 takes about 25s\n", + "# work 615236801 also in trace log\n", "# social 2464104885 642446345 1767182945\n", - "ms_comp = comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check)\n", + "\n", + "bp = base_probs[9] # 7 for social, 9 for work. Check t[t.index == trip_id_to_check] to see which purpose, and\n", + "# output above to see which index\n", + "# choose_individual_max_utility = False\n", + "#ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", + "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", + "\n", + "choose_individual_max_utility = True\n", + "ms_comp = comp_mode_shares(base_probs[9], choose_individual_max_utility, num_samples, trip_id_to_check)\n", "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 160, - "outputs": [], - "source": [], + ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%%\n", + "is_executing": true } } }, From c95f68df5c08ffd04916fe0ea82c4f52904838f3 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 8 May 2022 21:15:41 +1000 Subject: [PATCH 018/135] fixes bug in choice for tree --- activitysim/core/logit.py | 67 ++-- notebooks/fru_utils.py | 2 - notebooks/validate_frozen_impl.ipynb | 445 ++++++++++++++++++++------- 3 files changed, 378 insertions(+), 136 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 0e7feb33a1..8166607945 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -192,37 +192,53 @@ def inverse_ev1_cdf(x, location=0.0, scale=1.0): return location - scale * np.log(-np.log(x)) -def add_ev1_random(df, nest_spec): - # TODO: generalise to logit for nest_spec==None by adding one rand (scale=1) to all columns +# def add_ev1_random(df, nest_spec): +# # TODO: generalise to logit for nest_spec==None by adding one rand (scale=1) to all columns +# nest_utils_for_choice = df.copy() +# for n in each_nest(nest_spec): +# if n.level == 1: +# continue # skip the root level, not needed +# uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice) +# rands = inverse_ev1_cdf(uniform_rands) +# nest_utils_for_choice.loc[:, n.name] += rands[:, 0] # inverse_ev1_cdf of single-row df adds dimension +# return nest_utils_for_choice +def add_ev1_random(df): nest_utils_for_choice = df.copy() - for n in each_nest(nest_spec): - if n.level == 1: - continue # skip the root level, not needed - uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice) - rands = inverse_ev1_cdf(uniform_rands) - nest_utils_for_choice.loc[:, n.name] += rands[:, 0] # inverse_ev1_cdf of single-row df adds dimension + uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice, n=df.shape[1]) + rands = inverse_ev1_cdf(uniform_rands) + nest_utils_for_choice += rands return nest_utils_for_choice -def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_choosers=None): - trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') - nest_utils_for_choice = add_ev1_random(nested_utilities, nest_spec) - all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, each_nest(nest_spec)))) - - def is_alternative(name): - return name in all_alternatives - - nest_utils_for_choice["choice"] = None - for level, alts in group_nests_by_level(nest_spec).items(): +def choose_from_tree(nest_utils, nest_spec): + all_alternatives = [nest.name for nest in each_nest(nest_spec, type='leaf')] + for level, nests_at_level in group_nests_by_level(nest_spec).items(): + nest_alts = [nest.name for nest in nests_at_level] if level == 1: + assert len(nests_at_level) == 1 + assert len(nest_alts) == 1 + next_level_alts = nests_at_level[0].alternatives continue - no_choices_made_yet = nest_utils_for_choice["choice"].isnull() - choice_this_level = nest_utils_for_choice.loc[no_choices_made_yet][alts].idxmax(1) - nest_utils_for_choice.loc[no_choices_made_yet, "choice"] = \ - np.where(choice_this_level.apply(is_alternative), choice_this_level, None) + # all alternatives from the previous level + alts_this_level = list(filter(lambda x: x in next_level_alts, nest_alts)) + choice_this_level = nest_utils[nest_utils.index.isin(alts_this_level)].idxmax() + if choice_this_level in all_alternatives: + return choice_this_level + chosen_nest = list(filter(lambda x: x.name == choice_this_level, nests_at_level)) + assert len(chosen_nest) == 1 + next_level_alts = chosen_nest[0].alternatives - assert not nest_utils_for_choice["choice"].isnull().any(), "No choice for XXX - implement reporting" - choices = pd.Series(nest_utils_for_choice["choice"], index=nested_utilities.index) + raise ValueError("This should never happen - no alternative found") + + +def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_choosers=None): + """ walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') + nest_utils_for_choice = add_ev1_random(nested_utilities) + # TODO: the following apply is slow, try to improve it + choices = nest_utils_for_choice.apply(lambda x: choose_from_tree(x, nest_spec), axis=1) + assert not choices.isnull().any(), "No choice for XXX - implement reporting" + choices = pd.Series(choices, index=nested_utilities.index) return choices @@ -523,6 +539,5 @@ def group_nests_by_level(nest_spec): depth = np.max([x.level for x in each_nest(nest_spec)]) nest_levels = {x: [] for x in range(1, depth+1)} for n in each_nest(nest_spec): - nest_levels[n.level].append(n.name) - assert len(nest_levels[1]) == 1 # only one root + nest_levels[n.level].append(n) return nest_levels diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index 5f616e8ecf..0f13679b52 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -1,11 +1,9 @@ import os -import argparse from datetime import datetime import numpy as np import pandas as pd -from activitysim.cli import run from activitysim.core import inject from activitysim.core import tracing from activitysim.core import config diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index d7e2ab1905..f7270bdf64 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 1, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "execution": { @@ -22,23 +22,14 @@ }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 2, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "execution": { @@ -57,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 224, + "execution_count": 19, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -87,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 225, + "execution_count": 20, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -106,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 226, + "execution_count": 21, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -126,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 227, + "execution_count": 23, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -145,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 228, + "execution_count": 24, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -172,7 +163,25 @@ }, { "cell_type": "code", - "execution_count": 229, + "execution_count": 25, + "outputs": [], + "source": [ + "# need to do this before loading checkpoint tables\n", + "trace_hh_ids = 1024353\n", + "#1024353 has trip id 642446345 (at least) where logsum and probs are different\n", + "#2821179 # has trip_ids 2464104881 and 2464104885\n", + "inject.add_injectable(\"trace_hh_id\", trace_hh_ids)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 26, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -189,7 +198,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", + "register joint_tour_participants: no rows with household_id in [1024353].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -207,15 +216,28 @@ "trip_mode_choice tour_type 'shopping' (77 trips)\n", "trip_mode_choice tour_type 'social' (19 trips)\n", "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n" + "trip_mode_choice tour_type 'work' (168 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "register joint_tour_participants: no rows with household_id in [1024353].\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "Running with %d trips 482\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -223,7 +245,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", @@ -235,8 +256,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 25.2 s, sys: 2.31 s, total: 27.5 s\n", - "Wall time: 26.5 s\n" + "CPU times: user 18.1 s, sys: 2.48 s, total: 20.6 s\n", + "Wall time: 17 s\n" ] } ], @@ -248,14 +269,14 @@ }, { "cell_type": "code", - "execution_count": 231, + "execution_count": 27, "outputs": [ { "data": { - "text/plain": " trip_mode_fru trip_mode_asim\nWALK 297 304\nWALK_LOC 100 85\nWALK_LRF 52 53\nBIKE 16 17\nDRIVEALONEFREE 7 4\nWALK_HVY 4 4\nTNC_SINGLE 3 4\nSHARED2FREE 2 4\nTNC_SHARED 1 4", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK297304
WALK_LOC10085
WALK_LRF5253
BIKE1617
DRIVEALONEFREE74
WALK_HVY44
TNC_SINGLE34
SHARED2FREE24
TNC_SHARED14
\n
" + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 304 304\nWALK_LOC 91 91\nWALK_LRF 49 47\nBIKE 15 17\nTNC_SINGLE 6 4\nDRIVEALONEFREE 5 4\nWALK_HVY 5 4\nSHARED2FREE 3 4\nTNC_SHARED 2 4\nTAXI 2 2", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK304304
WALK_LOC9191
WALK_LRF4947
BIKE1517
TNC_SINGLE64
DRIVEALONEFREE54
WALK_HVY54
SHARED2FREE34
TNC_SHARED24
TAXI22
\n
" }, - "execution_count": 231, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -275,17 +296,14 @@ }, { "cell_type": "code", - "execution_count": 232, + "execution_count": 28, "outputs": [ { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN+ElEQVR4nO3df2xd9XnH8fdTXCgiG2kb5rEkmpGIOiHc0WIBU/+YA/sRoFrYBBUdKkmXKZoEUqdmak2RVk3btFSIoXarKkWlWjqhGcRWgQjVygCr6h9hS1pK+NEOF4U1EQXR0nQGusnrsz984NqpzXXiY9/cx++XZOV8v9/jr5/7CH18cnLuJTITSVItb+t1AZKk9hnuklSQ4S5JBRnuklSQ4S5JBQ30ugCAdevW5dDQUCt7vfrqq5x11lmt7NXv7EWHveiwFx393ouDBw++nJnnzLd2SoT70NAQBw4caGWviYkJRkdHW9mr39mLDnvRYS86+r0XEfH8QmvelpGkggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekgk6Jd6hK0koYGts3Z7xreJrtY/s4vPvqHlW0fLxyl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKmjR4R4Rp0XEtyLigWZ8XkQ8FhGTEXF3RJzezJ/RjCeb9aFlql2StIATuXL/GPDMrPFngDsy83zgFWBHM78DeKWZv6M5T5K0ghYV7hGxAbga+GIzDuBy4N7mlL3ANc3x1mZMs35Fc74kaYVEZnY/KeJe4G+AXwD+DNgO7G+uzomIjcBXM/PCiHgS2JKZR5q17wGXZubLx+25E9gJMDg4ePH4+HgrL2hqaoo1a9a0sle/sxcd9qJjNffi0NFjc8aDZ8KLr8Pw+rN7VNHSbN68+WBmjsy3NtDtmyPig8BLmXkwIkbbKioz9wB7AEZGRnJ0tJ2tJyYmaGuvfmcvOuxFx2ruxfaxfXPGu4anuf3QAIdvGO1NQcuoa7gDHwB+LyKuAt4B/CLwWWBtRAxk5jSwATjanH8U2AgciYgB4Gzgh61XLklaUNd77pl5S2ZuyMwh4Hrgkcy8AXgUuLY5bRtwX3N8fzOmWX8kF3PvR5LUmqU85/5J4OMRMQm8G7izmb8TeHcz/3FgbGklSpJO1GJuy7wpMyeAieb4OeCSec75KXBdC7VJkk6S71CVpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIK6hntEvCMi/j0ivh0RT0XEXzTz50XEYxExGRF3R8TpzfwZzXiyWR9a5tcgSTrOYq7c/we4PDN/HbgI2BIRlwGfAe7IzPOBV4Adzfk7gFea+Tua8yRJK6hruOeMqWb49uYrgcuBe5v5vcA1zfHWZkyzfkVERFsFS5K6i8zsflLEacBB4Hzg88BtwP7m6pyI2Ah8NTMvjIgngS2ZeaRZ+x5waWa+fNyeO4GdAIODgxePj4+38oKmpqZYs2ZNK3v1O3vRYS86VnMvDh09Nmc8eCa8+DoMrz+7RxUtzebNmw9m5sh8awOL2SAz/w+4KCLWAl8Bfm2pRWXmHmAPwMjISI6Oji51SwAmJiZoa69+Zy867EXHau7F9rF9c8a7hqe5/dAAh28Y7U1By+iEnpbJzB8DjwK/AayNiDd+OWwAjjbHR4GNAM362cAP2yhWkrQ4i3la5pzmip2IOBP4beAZZkL+2ua0bcB9zfH9zZhm/ZFczL0fSVJrFnNb5lxgb3Pf/W3APZn5QEQ8DYxHxF8B3wLubM6/E/jHiJgEfgRcvwx1S5LeQtdwz8wngPfNM/8ccMk88z8FrmulOknSSfEdqpJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUUNdwj4iNEfFoRDwdEU9FxMea+XdFxEMR8Wzz5zub+YiIz0XEZEQ8ERHvX+4XIUmaazFX7tPArsy8ALgMuCkiLgDGgIczcxPwcDMGuBLY1HztBL7QetWSpLfUNdwz84XM/GZz/N/AM8B6YCuwtzltL3BNc7wV+HLO2A+sjYhz2y5ckrSwE7rnHhFDwPuAx4DBzHyhWfoBMNgcrwe+P+vbjjRzkqQVMrDYEyNiDfDPwJ9m5k8i4s21zMyIyBP5wRGxk5nbNgwODjIxMXEi376gqamp1vbqd/aiw150rOZe7BqenjMePHNmrmI/FhXuEfF2ZoL9rsz8l2b6xYg4NzNfaG67vNTMHwU2zvr2Dc3cHJm5B9gDMDIykqOjoyf3Co4zMTFBW3v1O3vRYS86VnMvto/tmzPeNTzN7YcGOHzDaG8KWkaLeVomgDuBZzLzb2ct3Q9sa463AffNmr+xeWrmMuDYrNs3kqQVsJgr9w8AHwEORcTjzdyngN3APRGxA3ge+FCz9iBwFTAJvAZ8tM2CJUnddQ33zPwGEAssXzHP+QnctMS6JElL4DtUJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCuoa7hHxpYh4KSKenDX3roh4KCKebf58ZzMfEfG5iJiMiCci4v3LWbwkaX6LuXL/B2DLcXNjwMOZuQl4uBkDXAlsar52Al9op0xJ0onoGu6Z+XXgR8dNbwX2Nsd7gWtmzX85Z+wH1kbEuS3VKklapMjM7idFDAEPZOaFzfjHmbm2OQ7glcxcGxEPALsz8xvN2sPAJzPzwDx77mTm6p7BwcGLx8fHW3lBU1NTrFmzppW9+p296LAXHau5F4eOHpszHjwTXnwdhtef3aOKlmbz5s0HM3NkvrWBpW6emRkR3X9D/Pz37QH2AIyMjOTo6OhSSwFgYmKCtvbqd/aiw150rOZebB/bN2e8a3ia2w8NcPiG0d4UtIxO9mmZF9+43dL8+VIzfxTYOOu8Dc2cJGkFnWy43w9sa463AffNmr+xeWrmMuBYZr6wxBolSSeo622ZiPgnYBRYFxFHgE8Du4F7ImIH8Dzwoeb0B4GrgEngNeCjy1CzJKmLruGemR9eYOmKec5N4KalFiVJWhrfoSpJBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBQ30ugBJ6rWhsX3zzh/effUKV9Ier9wlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaBlec49IrYAnwVOA76YmbuX4+dI0nLq5+ffW79yj4jTgM8DVwIXAB+OiAva/jmSpIUtx5X7JcBkZj4HEBHjwFbg6WX4WT/3m3XX8DTbx/b1xW9Wqbp+vvJ9Kyfzula6F5GZ7W4YcS2wJTP/uBl/BLg0M28+7rydwM5m+B7guy2VsA54uaW9+p296LAXHfaio9978auZec58Cz37bJnM3APsaXvfiDiQmSNt79uP7EWHveiwFx2Ve7EcT8scBTbOGm9o5iRJK2Q5wv0/gE0RcV5EnA5cD9y/DD9HkrSA1m/LZOZ0RNwM/Cszj0J+KTOfavvnvIXWb/X0MXvRYS867EVH2V60/g+qkqTe8x2qklSQ4S5JBZUN94jYFREZEet6XUsvRcRfRsQTEfF4RHwtIn6l1zX1SkTcFhHfafrxlYhY2+uaeiUirouIpyLiZxFR8lHAbiJiS0R8NyImI2Ks1/W0rWS4R8RG4HeA/+p1LaeA2zLzvZl5EfAA8Oc9rqeXHgIuzMz3Av8J3NLjenrpSeAPgK/3upBeWA0fk1Iy3IE7gE8Aq/5fizPzJ7OGZ7GKe5KZX8vM6Wa4n5n3YKxKmflMZrb1rvB+9ObHpGTm/wJvfExKGT17h+pyiYitwNHM/HZE9LqcU0JE/DVwI3AM2Nzjck4VfwTc3esi1DPrge/PGh8BLu1RLcuiL8M9Iv4N+OV5lm4FPsXMLZlV4636kZn3ZeatwK0RcQtwM/DpFS1wBXXrRXPOrcA0cNdK1rbSFtML1dWX4Z6ZvzXffEQMA+cBb1y1bwC+GRGXZOYPVrDEFbVQP+ZxF/AghcO9Wy8iYjvwQeCKLP4mjxP472I1Kv8xKX0Z7gvJzEPAL70xjojDwEhm9vOnvi1JRGzKzGeb4VbgO72sp5ea/4nMJ4DfzMzXel2PeurNj0lhJtSvB/6wtyW1q1S4a167I+I9wM+A54E/6XE9vfT3wBnAQ83f7PZn5qrsR0T8PvB3wDnAvoh4PDN/t8dlrZhT4GNSlp0fPyBJBVV9FFKSVjXDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaD/B/tepd0H3PK6AAAAAElFTkSuQmCC\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" + "name": "stdout", + "output_type": "stream", + "text": [ + "Logsums agree\n" + ] } ], "source": [ @@ -303,13 +321,13 @@ }, { "cell_type": "code", - "execution_count": 233, + "execution_count": 12, "outputs": [ { "data": { "text/plain": "trip_id\n106170305 4.050638\n106170309 0.553496\n1767013721 0.482067\n535694461 0.251889\n943749469 0.203583\n ... \n535672921 0.000000\n535620049 0.000000\n535388485 0.000000\n535388481 0.000000\n943749471 0.000000\nLength: 482, dtype: float64" }, - "execution_count": 233, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -327,20 +345,20 @@ }, { "cell_type": "code", - "execution_count": 234, + "execution_count": 13, "outputs": [ { "data": { - "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n615236801 WALK_LOC 7.627291 WALK_LOC \n\n mode_choice_logsum_asim \ntrip_id \n615236801 7.627291 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
615236801WALK_LOC7.627291WALK_LOC7.627291
\n
" + "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n642446345 WALK_LRF 1.634462 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n642446345 1.593075 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
642446345WALK_LRF1.634462WALK_LRF1.593075
\n
" }, - "execution_count": 234, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "trips.loc[trips.index == 106170305]" + "trips.loc[trips.index == 642446345]" ], "metadata": { "collapsed": false, @@ -351,20 +369,20 @@ }, { "cell_type": "code", - "execution_count": 210, + "execution_count": 14, "outputs": [ { "data": { "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n86627409 WALK_LRF 2.435199 WALK_LRF \n86627413 WALK_LRF 2.665463 WALK_LRF \n86673661 WALK_LRF 1.842576 WALK_LRF \n106170305 WALK_LRF -0.002341 WALK_LRF \n106170309 WALK_LRF -0.017607 WALK_LRF \n... ... ... ... \n1767186253 WALK_LRF 0.976319 WALK_LRF \n1767186581 WALK_HVY 1.712704 WALK_LRF \n1768237161 WALK 6.262944 WALK \n1768237167 WALK_LRF 11.191330 WALK \n2473024477 WALK_LRF 0.637076 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n86627409 2.407248 \n86627413 2.642707 \n86673661 1.834538 \n106170305 -0.011825 \n106170309 -0.027353 \n... ... \n1767186253 0.880819 \n1767186581 1.600736 \n1768237161 6.226927 \n1768237167 11.144496 \n2473024477 0.624304 \n\n[74 rows x 4 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
86627409WALK_LRF2.435199WALK_LRF2.407248
86627413WALK_LRF2.665463WALK_LRF2.642707
86673661WALK_LRF1.842576WALK_LRF1.834538
106170305WALK_LRF-0.002341WALK_LRF-0.011825
106170309WALK_LRF-0.017607WALK_LRF-0.027353
...............
1767186253WALK_LRF0.976319WALK_LRF0.880819
1767186581WALK_HVY1.712704WALK_LRF1.600736
1768237161WALK6.262944WALK6.226927
1768237167WALK_LRF11.191330WALK11.144496
2473024477WALK_LRF0.637076WALK_LRF0.624304
\n

74 rows × 4 columns

\n
" }, - "execution_count": 210, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" + "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" ], "metadata": { "collapsed": false, @@ -404,13 +422,13 @@ }, { "cell_type": "code", - "execution_count": 235, + "execution_count": 29, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", + "register joint_tour_participants: no rows with household_id in [1024353].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -446,20 +464,10 @@ }, { "cell_type": "code", - "execution_count": 236, - "outputs": [ - { - "data": { - "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n86627409 WALK_LRF 2.435199 WALK_LRF \n86627413 WALK_LRF 2.665463 WALK_LRF \n86673661 WALK_LRF 1.842576 WALK_LRF \n106170305 WALK_LRF -0.002341 WALK_LRF \n106170309 WALK_LRF -0.017607 WALK_LRF \n... ... ... ... \n1767186253 WALK_LRF 0.976319 WALK_LRF \n1767186581 WALK_HVY 1.712704 WALK_LRF \n1768237161 WALK 6.262944 WALK \n1768237167 WALK_LRF 11.191330 WALK \n2473024477 WALK_LRF 0.637076 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n86627409 2.407248 \n86627413 2.642707 \n86673661 1.834538 \n106170305 -0.011825 \n106170309 -0.027353 \n... ... \n1767186253 0.880819 \n1767186581 1.600736 \n1768237161 6.226927 \n1768237167 11.144496 \n2473024477 0.624304 \n\n[74 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
86627409WALK_LRF2.435199WALK_LRF2.407248
86627413WALK_LRF2.665463WALK_LRF2.642707
86673661WALK_LRF1.842576WALK_LRF1.834538
106170305WALK_LRF-0.002341WALK_LRF-0.011825
106170309WALK_LRF-0.017607WALK_LRF-0.027353
...............
1767186253WALK_LRF0.976319WALK_LRF0.880819
1767186581WALK_HVY1.712704WALK_LRF1.600736
1768237161WALK6.262944WALK6.226927
1768237167WALK_LRF11.191330WALK11.144496
2473024477WALK_LRF0.637076WALK_LRF0.624304
\n

74 rows × 4 columns

\n
" - }, - "execution_count": 236, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 30, + "outputs": [], "source": [ - "trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" + "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" ], "metadata": { "collapsed": false, @@ -470,20 +478,11 @@ }, { "cell_type": "code", - "execution_count": 237, - "outputs": [ - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n86627409 264107 226869 work 1 True \n86627413 264107 226869 work 1 False \n86673661 264248 226939 work 1 False \n106170305 323689 256660 work 1 True \n106170309 323689 256660 work 1 False \n... ... ... ... ... ... \n1767186253 5387762 2223027 work 1 False \n1767186581 5387763 2223027 work 1 False \n1768237161 5390966 2224629 work 1 True \n1768237167 5390966 2224629 work 3 False \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n86627409 1 24 9 10828426 work \n86627413 1 9 24 10828426 home \n86673661 1 10 22 10834207 home \n106170305 1 1 10 13271288 work \n106170309 1 10 1 13271288 home \n... ... ... ... ... ... \n1767186253 1 9 4 220898281 home \n1767186581 2 9 14 220898322 othmaint \n1768237161 1 16 17 221029645 work \n1768237167 3 17 7 221029645 home \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n86627409 NaN 7.0 WALK_LRF 2.407248 \n86627413 NaN 19.0 WALK_LRF 2.642707 \n86673661 NaN 19.0 WALK_LRF 1.834538 \n106170305 NaN 11.0 WALK_LRF -0.011825 \n106170309 NaN 18.0 WALK_LRF -0.027353 \n... ... ... ... ... \n1767186253 NaN 17.0 WALK_LRF 0.880819 \n1767186581 32.045096 17.0 WALK_LRF 1.600736 \n1768237161 NaN 8.0 WALK 6.226927 \n1768237167 NaN 19.0 WALK 11.144496 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[74 rows x 14 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86627409264107226869work1True124910828426workNaN7.0WALK_LRF2.407248
86627413264107226869work1False192410828426homeNaN19.0WALK_LRF2.642707
86673661264248226939work1False1102210834207homeNaN19.0WALK_LRF1.834538
106170305323689256660work1True111013271288workNaN11.0WALK_LRF-0.011825
106170309323689256660work1False110113271288homeNaN18.0WALK_LRF-0.027353
.............................................
176718625353877622223027work1False194220898281homeNaN17.0WALK_LRF0.880819
176718658153877632223027work1False2914220898322othmaint32.04509617.0WALK_LRF1.600736
176823716153909662224629work1True11617221029645workNaN8.0WALK6.226927
176823716753909662224629work3False3177221029645homeNaN19.0WALK11.144496
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

74 rows × 14 columns

\n
" - }, - "execution_count": 237, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 31, + "outputs": [], "source": [ - "t.loc[t.index.isin(trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0].index.values)]" + "#t.loc[t.index.isin(trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0].index.values)]\\\n", + "# .sort_values(by=[\"primary_purpose\"])" ], "metadata": { "collapsed": false, @@ -494,20 +493,12 @@ }, { "cell_type": "code", - "execution_count": 238, - "outputs": [ - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n615236801 1875721 982875 work 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n615236801 1 10 16 76904600 work \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n615236801 NaN 8.0 WALK_LOC 7.627291 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
6152368011875721982875work1True1101676904600workNaN8.0WALK_LOC7.627291
\n
" - }, - "execution_count": 238, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 32, + "outputs": [], "source": [ - "t.loc[t.index == 615236801]" + "# nest_spec = ns[0]\n", + "# for nest in logit.each_nest(nest_spec, post_order=False):\n", + "# print(f\"{nest.level}, {nest.name}, {nest.coefficient}, parent scale: {nest.parent_scale}\")" ], "metadata": { "collapsed": false, @@ -518,12 +509,10 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 33, "outputs": [], "source": [ - "# nest_spec = ns[0]\n", - "# for nest in logit.each_nest(nest_spec, post_order=False):\n", - "# print(f\"{nest.level}, {nest.name}, {nest.coefficient}, parent scale: {nest.parent_scale}\")" + "#base_probs[7] #615236801" ], "metadata": { "collapsed": false, @@ -534,20 +523,20 @@ }, { "cell_type": "code", - "execution_count": 220, + "execution_count": 34, "outputs": [ { "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n8805121 0.0 0.0 0.000000e+00 0.0 \n8805125 0.0 0.0 0.000000e+00 0.0 \n8805126 0.0 0.0 0.000000e+00 0.0 \n468119921 0.0 0.0 0.000000e+00 0.0 \n468119922 0.0 0.0 0.000000e+00 0.0 \n468119925 0.0 0.0 0.000000e+00 0.0 \n468119926 0.0 0.0 0.000000e+00 0.0 \n468119927 0.0 0.0 0.000000e+00 0.0 \n468119928 0.0 0.0 0.000000e+00 0.0 \n642446345 0.0 0.0 4.882838e-03 0.0 \n642446349 0.0 0.0 3.622165e-03 0.0 \n1179608009 0.0 0.0 5.572746e-05 0.0 \n1179608013 0.0 0.0 1.108122e-05 0.0 \n1658807153 0.0 0.0 4.252415e-07 0.0 \n1658807157 0.0 0.0 5.038020e-07 0.0 \n1767182945 0.0 0.0 1.610090e-02 0.0 \n1767182949 0.0 0.0 1.540381e-02 0.0 \n2464104881 0.0 0.0 2.228876e-05 0.0 \n2464104885 0.0 0.0 2.163505e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n8805121 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805125 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n8805126 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119921 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119922 0.000000e+00 0.0 0.999874 0.0 0.000000 0.000000 \n468119925 0.000000e+00 0.0 0.999875 0.0 0.000000 0.000000 \n468119926 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119927 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n468119928 0.000000e+00 0.0 1.000000 0.0 0.000000 0.000000 \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n642446349 3.950582e-14 0.0 0.043359 0.0 0.068362 0.884144 \n1179608009 1.964682e-05 0.0 0.969538 0.0 0.000000 0.000000 \n1179608013 7.469124e-06 0.0 0.969598 0.0 0.000000 0.000000 \n1658807153 1.499257e-07 0.0 0.048044 0.0 0.951445 0.000000 \n1658807157 1.907576e-07 0.0 0.065847 0.0 0.933430 0.000000 \n1767182945 1.548643e-13 0.0 0.046835 0.0 0.144256 0.787096 \n1767182949 1.509357e-13 0.0 0.041767 0.0 0.141232 0.795898 \n2464104881 2.147403e-16 0.0 0.646947 0.0 0.353030 0.000000 \n2464104885 2.084422e-16 0.0 0.627973 0.0 0.372005 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n8805121 0.0 0.0 0.0 0.0 0.0 0.0 \n8805125 0.0 0.0 0.0 0.0 0.0 0.0 \n8805126 0.0 0.0 0.0 0.0 0.0 0.0 \n468119921 0.0 0.0 0.0 0.0 0.0 0.0 \n468119922 0.0 0.0 0.0 0.0 0.0 0.0 \n468119925 0.0 0.0 0.0 0.0 0.0 0.0 \n468119926 0.0 0.0 0.0 0.0 0.0 0.0 \n468119927 0.0 0.0 0.0 0.0 0.0 0.0 \n468119928 0.0 0.0 0.0 0.0 0.0 0.0 \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n642446349 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608009 0.0 0.0 0.0 0.0 0.0 0.0 \n1179608013 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807153 0.0 0.0 0.0 0.0 0.0 0.0 \n1658807157 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182945 0.0 0.0 0.0 0.0 0.0 0.0 \n1767182949 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104881 0.0 0.0 0.0 0.0 0.0 0.0 \n2464104885 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n8805121 0.0 0.0 4.266799e-08 7.843494e-08 2.071998e-08 \n8805125 0.0 0.0 4.057327e-08 7.325707e-08 1.950799e-08 \n8805126 0.0 0.0 4.864169e-08 9.376443e-08 2.428075e-08 \n468119921 0.0 0.0 1.096638e-08 2.754998e-10 4.051502e-08 \n468119922 0.0 0.0 2.658716e-05 6.599176e-07 9.832967e-05 \n468119925 0.0 0.0 2.651144e-05 6.601311e-07 9.831370e-05 \n468119926 0.0 0.0 1.568041e-08 2.408144e-10 3.615344e-08 \n468119927 0.0 0.0 5.308505e-08 1.315547e-10 2.041405e-08 \n468119928 0.0 0.0 2.367465e-09 1.493582e-10 1.939597e-08 \n642446345 0.0 0.0 9.610763e-05 1.307549e-04 2.867554e-04 \n642446349 0.0 0.0 9.293281e-05 1.316643e-04 2.885385e-04 \n1179608009 0.0 0.0 5.225133e-06 2.766547e-02 2.716407e-03 \n1179608013 0.0 0.0 5.222635e-06 2.766274e-02 2.715479e-03 \n1658807153 0.0 0.0 7.091296e-08 4.695006e-04 4.115873e-05 \n1658807157 0.0 0.0 9.574501e-08 6.645671e-04 5.801899e-05 \n1767182945 0.0 0.0 1.498950e-03 3.358768e-03 8.536317e-04 \n1767182949 0.0 0.0 1.476552e-03 3.378934e-03 8.436653e-04 \n2464104881 0.0 0.0 1.365321e-07 8.764695e-12 3.537046e-07 \n2464104885 0.0 0.0 1.325277e-07 8.507634e-12 3.433307e-07 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
88051210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.266799e-087.843494e-082.071998e-08
88051250.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.057327e-087.325707e-081.950799e-08
88051260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.04.864169e-089.376443e-082.428075e-08
4681199210.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.096638e-082.754998e-104.051502e-08
4681199220.00.00.000000e+000.00.000000e+000.00.9998740.00.0000000.0000000.00.00.00.00.00.00.00.02.658716e-056.599176e-079.832967e-05
4681199250.00.00.000000e+000.00.000000e+000.00.9998750.00.0000000.0000000.00.00.00.00.00.00.00.02.651144e-056.601311e-079.831370e-05
4681199260.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.01.568041e-082.408144e-103.615344e-08
4681199270.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.05.308505e-081.315547e-102.041405e-08
4681199280.00.00.000000e+000.00.000000e+000.01.0000000.00.0000000.0000000.00.00.00.00.00.00.00.02.367465e-091.493582e-101.939597e-08
6424463450.00.04.882838e-030.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.09.610763e-051.307549e-042.867554e-04
6424463490.00.03.622165e-030.03.950582e-140.00.0433590.00.0683620.8841440.00.00.00.00.00.00.00.09.293281e-051.316643e-042.885385e-04
11796080090.00.05.572746e-050.01.964682e-050.00.9695380.00.0000000.0000000.00.00.00.00.00.00.00.05.225133e-062.766547e-022.716407e-03
11796080130.00.01.108122e-050.07.469124e-060.00.9695980.00.0000000.0000000.00.00.00.00.00.00.00.05.222635e-062.766274e-022.715479e-03
16588071530.00.04.252415e-070.01.499257e-070.00.0480440.00.9514450.0000000.00.00.00.00.00.00.00.07.091296e-084.695006e-044.115873e-05
16588071570.00.05.038020e-070.01.907576e-070.00.0658470.00.9334300.0000000.00.00.00.00.00.00.00.09.574501e-086.645671e-045.801899e-05
17671829450.00.01.610090e-020.01.548643e-130.00.0468350.00.1442560.7870960.00.00.00.00.00.00.00.01.498950e-033.358768e-038.536317e-04
17671829490.00.01.540381e-020.01.509357e-130.00.0417670.00.1412320.7958980.00.00.00.00.00.00.00.01.476552e-033.378934e-038.436653e-04
24641048810.00.02.228876e-050.02.147403e-160.00.6469470.00.3530300.0000000.00.00.00.00.00.00.00.01.365321e-078.764695e-123.537046e-07
24641048850.00.02.163505e-050.02.084422e-160.00.6279730.00.3720050.0000000.00.00.00.00.00.00.00.01.325277e-078.507634e-123.433307e-07
\n
" + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n642446345 1958677 1024353 social 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n642446345 1 14 9 80305793 social \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n642446345 NaN 10.0 WALK_LRF 1.729057 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
64244634519586771024353social1True114980305793socialNaN10.0WALK_LRF1.729057
\n
" }, - "execution_count": 220, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "base_probs[7] #615236801" + "t.loc[t.index == 642446345]" ], "metadata": { "collapsed": false, @@ -558,13 +547,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 183, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", + "register joint_tour_participants: no rows with household_id in [1024353].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -573,39 +562,279 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'work' (10000 trips)\n" + "trip_mode_choice tour_type 'social' (10000 trips)\n", + "Done\n" ] + }, + { + "data": { + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004262 4.103657e-14 0.000084 0.00025 0.000114 \ntrip_mode 0.003800 0.000000e+00 0.000000 0.00020 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.039196 0.19705 0.759044 \ntrip_mode 0.038000 0.20070 0.757300 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0042624.103657e-140.0000840.000250.0001140.0391960.197050.759044
trip_mode0.0038000.000000e+000.0000000.000200.0000000.0380000.200700.757300
\n
" + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ "num_samples = 10000\n", - "trip_id_to_check = 615236801 # 1e5 takes about 25s\n", + "trip_id_to_check = 642446345 # 1e5 takes about 25s\n", "# work 615236801 also in trace log\n", "# social 2464104885 642446345 1767182945\n", "\n", - "bp = base_probs[9] # 7 for social, 9 for work. Check t[t.index == trip_id_to_check] to see which purpose, and\n", + "bp = base_probs[7] # 7 for social, 9 for work. Check t[t.index == trip_id_to_check] to see which purpose, and\n", "# output above to see which index\n", "# choose_individual_max_utility = False\n", "#ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", "\n", "choose_individual_max_utility = True\n", - "ms_comp = comp_mode_shares(base_probs[9], choose_individual_max_utility, num_samples, trip_id_to_check)\n", + "ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n", - "is_executing": true + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# Investigate differing probs\n", + "\n", + "is it due to clipping, due to idxmax always returning the first and by coincidence having equal probs somewhere, or\n", + "bad methodology?" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" } } }, { "cell_type": "code", - "execution_count": null, + "execution_count": 121, "outputs": [], - "source": [], + "source": [ + "def compute_probs(nested_utils, nest_spec):\n", + " nested_exp_utils = compute_nested_probabilities(np.exp(nested_utils), nest_spec)\n", + " base_probabilities = pd.DataFrame(index=nested_exp_utils.index)\n", + " for nest in logit.each_nest(nest_spec, type='leaf', post_order=False):\n", + " # skip root: it has a prob of 1 but we didn't compute a nested probability column for it\n", + " ancestors = nest.ancestors[1:]\n", + " base_probabilities[nest.name] = nested_exp_utils[ancestors].prod(axis=1)\n", + " return base_probabilities\n", + "\n", + "def compute_nested_probabilities(nested_exp_utilities, nest_spec):\n", + " nested_probabilities = pd.DataFrame(index=nested_exp_utilities.index)\n", + " for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", + " probs = u_to_p(nested_exp_utilities[nest.alternatives])\n", + " nested_probabilities = pd.concat([nested_probabilities, probs], axis=1)\n", + " return nested_probabilities\n", + "\n", + "EXP_UTIL_MIN = 1e-300\n", + "EXP_UTIL_MAX = np.inf\n", + "\n", + "PROB_MIN = 0.0\n", + "PROB_MAX = 1.0\n", + "\n", + "def u_to_p(utils, trace_label=None):\n", + " utils_arr = utils.values\n", + " np.clip(utils_arr, EXP_UTIL_MIN, EXP_UTIL_MAX, out=utils_arr)\n", + " utils_arr = np.where(utils_arr == EXP_UTIL_MIN, 0.0, utils_arr)\n", + " arr_sum = utils_arr.sum(axis=1)\n", + " inf_utils = np.isinf(arr_sum)\n", + " if inf_utils.any():\n", + " print(\"INF\")\n", + " with np.errstate(invalid='ignore', divide='ignore'):\n", + " np.divide(utils_arr, arr_sum.reshape(len(utils_arr), 1), out=utils_arr)\n", + " ## if allow_zero_probs, this will cause EXP_UTIL_MIN util rows to have all zero probabilities\n", + " utils_arr[np.isnan(utils_arr)] = PROB_MIN\n", + " np.clip(utils_arr, PROB_MIN, PROB_MAX, out=utils_arr)\n", + " probs = pd.DataFrame(utils_arr, columns=utils.columns, index=utils.index)\n", + " return probs" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 177, + "outputs": [], + "source": [ + "from activitysim.core import logit\n", + "from numpy.random import default_rng\n", + "\n", + "rng = default_rng(999)\n", + "\n", + "def random_for_df(df):\n", + " return rng.random(size=df.shape)\n", + "\n", + "def add_ev1_random(df):\n", + " uniform_rands = random_for_df(df)\n", + " df = df + logit.inverse_ev1_cdf(uniform_rands)\n", + " return df\n", + "\n", + "def group_nests_by_level(nest_spec):\n", + " # group nests by level, returns {level: [nest.name at that level]}\n", + " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in logit.each_nest(nest_spec):\n", + " nest_levels[n.level].append(n.name)\n", + " assert len(nest_levels[1]) == 1 # only one root\n", + " return nest_levels\n", + "\n", + "def make_choices_ru_frozen(nested_utilities, nest_spec):\n", + " nest_utils_for_choice = add_ev1_random(nested_utilities)\n", + " all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, logit.each_nest(nest_spec))))\n", + "\n", + " def is_alternative(name):\n", + " return name in all_alternatives\n", + "\n", + " nest_utils_for_choice[\"choice\"] = None\n", + " for level, alts in group_nests_by_level(nest_spec).items():\n", + " if level == 1:\n", + " continue\n", + " no_choices_made_yet = nest_utils_for_choice[\"choice\"].isnull()\n", + " choice_this_level = nest_utils_for_choice.loc[no_choices_made_yet][alts].idxmax(1)\n", + " nest_utils_for_choice.loc[no_choices_made_yet, \"choice\"] = \\\n", + " np.where(choice_this_level.apply(is_alternative), choice_this_level, None)\n", + "\n", + " assert not nest_utils_for_choice[\"choice\"].isnull().any(), \"No choice for XXX - implement reporting\"\n", + " choices = pd.Series(nest_utils_for_choice[\"choice\"], index=nested_utilities.index)\n", + " return choices, nest_utils_for_choice" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 178, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n642446345 0.0 0.0 0.004262 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n642446345 4.103657e-14 0.0 0.039196 0.0 0.19705 0.759044 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n642446345 0.0 0.0 0.000084 0.000114 0.00025 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
6424463450.00.00.0042620.04.103657e-140.00.0391960.00.197050.7590440.00.00.00.00.00.00.00.00.0000840.0001140.00025
\n
" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "nest_spec = ns[7] # 7 is social\n", + "nest_util = nu[7].loc[nu[7].index == trip_id_to_check]\n", + "\n", + "n = 100000\n", + "\n", + "nest_util = nest_util.loc[nest_util.index.repeat(n)]\n", + "nest_util.index += np.arange(n)\n", + "\n", + "display(base_probs[7].loc[base_probs[7].index == trip_id_to_check])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 179, + "outputs": [ + { + "data": { + "text/plain": "WALK_LRF 0.78148\nWALK_LOC 0.20604\nWALK 0.01194\nSHARED2FREE 0.00054\nName: choice, dtype: float64" + }, + "execution_count": 179, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "choices, nu_choices = make_choices_ru_frozen(nest_util, nest_spec)\n", + "choices.value_counts() / n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 180, + "outputs": [], + "source": [ + "def group_nests_by_level(nest_spec):\n", + " # group nests by level, returns {level: [nest]}\n", + " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in logit.each_nest(nest_spec):\n", + " nest_levels[n.level].append(n)\n", + " assert len(nest_levels[1]) == 1 # only one root\n", + " return nest_levels\n", + "\n", + "def make_choices_ru_frozen_apply(nested_utilities, nest_spec):\n", + " nest_utils_for_choice = add_ev1_random(nested_utilities)\n", + " all_alternatives = [nest.name for nest in logit.each_nest(nest_spec, type='leaf')]\n", + "\n", + " for level, nests_at_level in group_nests_by_level(nest_spec).items():\n", + " nest_alts = [nest.name for nest in nests_at_level]\n", + " if level == 1:\n", + " assert len(nests_at_level) == 1\n", + " assert len(nest_alts) == 1\n", + " next_level_alts = nests_at_level[0].alternatives\n", + " continue\n", + "\n", + " # all alternatives from the previous level\n", + " alts_this_level = list(filter(lambda x: x in next_level_alts, nest_alts))\n", + " #print(f\"Level {level} alternatives are {alts_this_level}\")\n", + " choice_this_level = nest_utils_for_choice[nest_utils_for_choice.index.isin(alts_this_level)].idxmax()\n", + " #print(f\"choice is {choice_this_level}\")\n", + " if choice_this_level in all_alternatives:\n", + " return choice_this_level\n", + " chosen_nest = list(filter(lambda x: x.name == choice_this_level, nests_at_level))\n", + " assert len(chosen_nest) == 1\n", + " next_level_alts = chosen_nest[0].alternatives\n", + "\n", + " raise ValueError(\"This should never happen - no alternative found\")\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 182, + "outputs": [ + { + "data": { + "text/plain": "WALK_LRF 0.75972\nWALK_LOC 0.19677\nWALK 0.03901\nSHARED2FREE 0.00406\nTNC_SHARED 0.00025\nTNC_SINGLE 0.00010\nTAXI 0.00009\ndtype: float64" + }, + "execution_count": 182, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "choices_apply = nest_util.apply(lambda x: make_choices_ru_frozen_apply(x, nest_spec), axis=1)\n", + "choices_apply.value_counts() / n" + ], "metadata": { "collapsed": false, "pycharm": { From 9f4d2622b09f52e1cf33ac2389a5a04cd8a59e1f Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 8 May 2022 21:32:31 +1000 Subject: [PATCH 019/135] check --- notebooks/validate_frozen_impl.ipynb | 296 +++++++++------------------ 1 file changed, 96 insertions(+), 200 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index f7270bdf64..00fd6d8c7b 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -181,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 187, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -230,8 +230,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Done\n", - "Running with %d trips 482\n" + "Done\n" ] }, { @@ -245,6 +244,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", @@ -256,8 +256,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 18.1 s, sys: 2.48 s, total: 20.6 s\n", - "Wall time: 17 s\n" + "CPU times: user 18.1 s, sys: 2.36 s, total: 20.5 s\n", + "Wall time: 17.3 s\n" ] } ], @@ -269,14 +269,14 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 188, "outputs": [ { "data": { - "text/plain": " trip_mode_fru trip_mode_asim\nWALK 304 304\nWALK_LOC 91 91\nWALK_LRF 49 47\nBIKE 15 17\nTNC_SINGLE 6 4\nDRIVEALONEFREE 5 4\nWALK_HVY 5 4\nSHARED2FREE 3 4\nTNC_SHARED 2 4\nTAXI 2 2", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK304304
WALK_LOC9191
WALK_LRF4947
BIKE1517
TNC_SINGLE64
DRIVEALONEFREE54
WALK_HVY54
SHARED2FREE34
TNC_SHARED24
TAXI22
\n
" + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 308 304\nWALK_LOC 87 85\nWALK_LRF 54 53\nBIKE 15 17\nTNC_SINGLE 7 4\nDRIVEALONEFREE 5 4\nSHARED2FREE 3 4\nSHARED3FREE 2 1\nWALK_HVY 1 4", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK308304
WALK_LOC8785
WALK_LRF5453
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n
" }, - "execution_count": 27, + "execution_count": 188, "metadata": {}, "output_type": "execute_result" } @@ -296,14 +296,17 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 189, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logsums agree\n" - ] + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN+ElEQVR4nO3df2xd9XnH8fdTXCgiG2kb5rEkmpGIOiHc0WIBU/+YA/sRoFrYBBUdKkmXKZoEUqdmak2RVk3btFSIoXarKkWlWjqhGcRWgQjVygCr6h9hS1pK+NEOF4U1EQXR0nQGusnrsz984NqpzXXiY9/cx++XZOV8v9/jr5/7CH18cnLuJTITSVItb+t1AZKk9hnuklSQ4S5JBRnuklSQ4S5JBQ30ugCAdevW5dDQUCt7vfrqq5x11lmt7NXv7EWHveiwFx393ouDBw++nJnnzLd2SoT70NAQBw4caGWviYkJRkdHW9mr39mLDnvRYS86+r0XEfH8QmvelpGkggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekgk6Jd6hK0koYGts3Z7xreJrtY/s4vPvqHlW0fLxyl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKmjR4R4Rp0XEtyLigWZ8XkQ8FhGTEXF3RJzezJ/RjCeb9aFlql2StIATuXL/GPDMrPFngDsy83zgFWBHM78DeKWZv6M5T5K0ghYV7hGxAbga+GIzDuBy4N7mlL3ANc3x1mZMs35Fc74kaYVEZnY/KeJe4G+AXwD+DNgO7G+uzomIjcBXM/PCiHgS2JKZR5q17wGXZubLx+25E9gJMDg4ePH4+HgrL2hqaoo1a9a0sle/sxcd9qJjNffi0NFjc8aDZ8KLr8Pw+rN7VNHSbN68+WBmjsy3NtDtmyPig8BLmXkwIkbbKioz9wB7AEZGRnJ0tJ2tJyYmaGuvfmcvOuxFx2ruxfaxfXPGu4anuf3QAIdvGO1NQcuoa7gDHwB+LyKuAt4B/CLwWWBtRAxk5jSwATjanH8U2AgciYgB4Gzgh61XLklaUNd77pl5S2ZuyMwh4Hrgkcy8AXgUuLY5bRtwX3N8fzOmWX8kF3PvR5LUmqU85/5J4OMRMQm8G7izmb8TeHcz/3FgbGklSpJO1GJuy7wpMyeAieb4OeCSec75KXBdC7VJkk6S71CVpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIK6hntEvCMi/j0ivh0RT0XEXzTz50XEYxExGRF3R8TpzfwZzXiyWR9a5tcgSTrOYq7c/we4PDN/HbgI2BIRlwGfAe7IzPOBV4Adzfk7gFea+Tua8yRJK6hruOeMqWb49uYrgcuBe5v5vcA1zfHWZkyzfkVERFsFS5K6i8zsflLEacBB4Hzg88BtwP7m6pyI2Ah8NTMvjIgngS2ZeaRZ+x5waWa+fNyeO4GdAIODgxePj4+38oKmpqZYs2ZNK3v1O3vRYS86VnMvDh09Nmc8eCa8+DoMrz+7RxUtzebNmw9m5sh8awOL2SAz/w+4KCLWAl8Bfm2pRWXmHmAPwMjISI6Oji51SwAmJiZoa69+Zy867EXHau7F9rF9c8a7hqe5/dAAh28Y7U1By+iEnpbJzB8DjwK/AayNiDd+OWwAjjbHR4GNAM362cAP2yhWkrQ4i3la5pzmip2IOBP4beAZZkL+2ua0bcB9zfH9zZhm/ZFczL0fSVJrFnNb5lxgb3Pf/W3APZn5QEQ8DYxHxF8B3wLubM6/E/jHiJgEfgRcvwx1S5LeQtdwz8wngPfNM/8ccMk88z8FrmulOknSSfEdqpJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUUNdwj4iNEfFoRDwdEU9FxMea+XdFxEMR8Wzz5zub+YiIz0XEZEQ8ERHvX+4XIUmaazFX7tPArsy8ALgMuCkiLgDGgIczcxPwcDMGuBLY1HztBL7QetWSpLfUNdwz84XM/GZz/N/AM8B6YCuwtzltL3BNc7wV+HLO2A+sjYhz2y5ckrSwE7rnHhFDwPuAx4DBzHyhWfoBMNgcrwe+P+vbjjRzkqQVMrDYEyNiDfDPwJ9m5k8i4s21zMyIyBP5wRGxk5nbNgwODjIxMXEi376gqamp1vbqd/aiw150rOZe7BqenjMePHNmrmI/FhXuEfF2ZoL9rsz8l2b6xYg4NzNfaG67vNTMHwU2zvr2Dc3cHJm5B9gDMDIykqOjoyf3Co4zMTFBW3v1O3vRYS86VnMvto/tmzPeNTzN7YcGOHzDaG8KWkaLeVomgDuBZzLzb2ct3Q9sa463AffNmr+xeWrmMuDYrNs3kqQVsJgr9w8AHwEORcTjzdyngN3APRGxA3ge+FCz9iBwFTAJvAZ8tM2CJUnddQ33zPwGEAssXzHP+QnctMS6JElL4DtUJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCuoa7hHxpYh4KSKenDX3roh4KCKebf58ZzMfEfG5iJiMiCci4v3LWbwkaX6LuXL/B2DLcXNjwMOZuQl4uBkDXAlsar52Al9op0xJ0onoGu6Z+XXgR8dNbwX2Nsd7gWtmzX85Z+wH1kbEuS3VKklapMjM7idFDAEPZOaFzfjHmbm2OQ7glcxcGxEPALsz8xvN2sPAJzPzwDx77mTm6p7BwcGLx8fHW3lBU1NTrFmzppW9+p296LAXHau5F4eOHpszHjwTXnwdhtef3aOKlmbz5s0HM3NkvrWBpW6emRkR3X9D/Pz37QH2AIyMjOTo6OhSSwFgYmKCtvbqd/aiw150rOZebB/bN2e8a3ia2w8NcPiG0d4UtIxO9mmZF9+43dL8+VIzfxTYOOu8Dc2cJGkFnWy43w9sa463AffNmr+xeWrmMuBYZr6wxBolSSeo622ZiPgnYBRYFxFHgE8Du4F7ImIH8Dzwoeb0B4GrgEngNeCjy1CzJKmLruGemR9eYOmKec5N4KalFiVJWhrfoSpJBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBQ30ugBJ6rWhsX3zzh/effUKV9Ier9wlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaBlec49IrYAnwVOA76YmbuX4+dI0nLq5+ffW79yj4jTgM8DVwIXAB+OiAva/jmSpIUtx5X7JcBkZj4HEBHjwFbg6WX4WT/3m3XX8DTbx/b1xW9Wqbp+vvJ9Kyfzula6F5GZ7W4YcS2wJTP/uBl/BLg0M28+7rydwM5m+B7guy2VsA54uaW9+p296LAXHfaio9978auZec58Cz37bJnM3APsaXvfiDiQmSNt79uP7EWHveiwFx2Ve7EcT8scBTbOGm9o5iRJK2Q5wv0/gE0RcV5EnA5cD9y/DD9HkrSA1m/LZOZ0RNwM/Cszj0J+KTOfavvnvIXWb/X0MXvRYS867EVH2V60/g+qkqTe8x2qklSQ4S5JBZUN94jYFREZEet6XUsvRcRfRsQTEfF4RHwtIn6l1zX1SkTcFhHfafrxlYhY2+uaeiUirouIpyLiZxFR8lHAbiJiS0R8NyImI2Ks1/W0rWS4R8RG4HeA/+p1LaeA2zLzvZl5EfAA8Oc9rqeXHgIuzMz3Av8J3NLjenrpSeAPgK/3upBeWA0fk1Iy3IE7gE8Aq/5fizPzJ7OGZ7GKe5KZX8vM6Wa4n5n3YKxKmflMZrb1rvB+9ObHpGTm/wJvfExKGT17h+pyiYitwNHM/HZE9LqcU0JE/DVwI3AM2Nzjck4VfwTc3esi1DPrge/PGh8BLu1RLcuiL8M9Iv4N+OV5lm4FPsXMLZlV4636kZn3ZeatwK0RcQtwM/DpFS1wBXXrRXPOrcA0cNdK1rbSFtML1dWX4Z6ZvzXffEQMA+cBb1y1bwC+GRGXZOYPVrDEFbVQP+ZxF/AghcO9Wy8iYjvwQeCKLP4mjxP472I1Kv8xKX0Z7gvJzEPAL70xjojDwEhm9vOnvi1JRGzKzGeb4VbgO72sp5ea/4nMJ4DfzMzXel2PeurNj0lhJtSvB/6wtyW1q1S4a167I+I9wM+A54E/6XE9vfT3wBnAQ83f7PZn5qrsR0T8PvB3wDnAvoh4PDN/t8dlrZhT4GNSlp0fPyBJBVV9FFKSVjXDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaD/B/tepd0H3PK6AAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" } ], "source": [ @@ -321,13 +324,13 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 190, "outputs": [ { "data": { "text/plain": "trip_id\n106170305 4.050638\n106170309 0.553496\n1767013721 0.482067\n535694461 0.251889\n943749469 0.203583\n ... \n535672921 0.000000\n535620049 0.000000\n535388485 0.000000\n535388481 0.000000\n943749471 0.000000\nLength: 482, dtype: float64" }, - "execution_count": 12, + "execution_count": 190, "metadata": {}, "output_type": "execute_result" } @@ -345,14 +348,14 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 191, "outputs": [ { "data": { "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n642446345 WALK_LRF 1.634462 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n642446345 1.593075 ", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
642446345WALK_LRF1.634462WALK_LRF1.593075
\n
" }, - "execution_count": 13, + "execution_count": 191, "metadata": {}, "output_type": "execute_result" } @@ -369,18 +372,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "outputs": [ - { - "data": { - "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n86627409 WALK_LRF 2.435199 WALK_LRF \n86627413 WALK_LRF 2.665463 WALK_LRF \n86673661 WALK_LRF 1.842576 WALK_LRF \n106170305 WALK_LRF -0.002341 WALK_LRF \n106170309 WALK_LRF -0.017607 WALK_LRF \n... ... ... ... \n1767186253 WALK_LRF 0.976319 WALK_LRF \n1767186581 WALK_HVY 1.712704 WALK_LRF \n1768237161 WALK 6.262944 WALK \n1768237167 WALK_LRF 11.191330 WALK \n2473024477 WALK_LRF 0.637076 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n86627409 2.407248 \n86627413 2.642707 \n86673661 1.834538 \n106170305 -0.011825 \n106170309 -0.027353 \n... ... \n1767186253 0.880819 \n1767186581 1.600736 \n1768237161 6.226927 \n1768237167 11.144496 \n2473024477 0.624304 \n\n[74 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
86627409WALK_LRF2.435199WALK_LRF2.407248
86627413WALK_LRF2.665463WALK_LRF2.642707
86673661WALK_LRF1.842576WALK_LRF1.834538
106170305WALK_LRF-0.002341WALK_LRF-0.011825
106170309WALK_LRF-0.017607WALK_LRF-0.027353
...............
1767186253WALK_LRF0.976319WALK_LRF0.880819
1767186581WALK_HVY1.712704WALK_LRF1.600736
1768237161WALK6.262944WALK6.226927
1768237167WALK_LRF11.191330WALK11.144496
2473024477WALK_LRF0.637076WALK_LRF0.624304
\n

74 rows × 4 columns

\n
" - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 192, + "outputs": [], "source": [ "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" ], @@ -393,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 192, "outputs": [], "source": [], "metadata": { @@ -422,7 +415,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 193, "outputs": [ { "name": "stderr", @@ -464,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 194, "outputs": [], "source": [ "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" @@ -478,7 +471,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 195, "outputs": [], "source": [ "#t.loc[t.index.isin(trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0].index.values)]\\\n", @@ -493,7 +486,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 196, "outputs": [], "source": [ "# nest_spec = ns[0]\n", @@ -509,7 +502,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 197, "outputs": [], "source": [ "#base_probs[7] #615236801" @@ -523,14 +516,14 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 198, "outputs": [ { "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n642446345 1958677 1024353 social 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n642446345 1 14 9 80305793 social \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n642446345 NaN 10.0 WALK_LRF 1.729057 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
64244634519586771024353social1True114980305793socialNaN10.0WALK_LRF1.729057
\n
" + "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n642446345 1958677 1024353 social 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n642446345 1 14 9 80305793 social \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n642446345 NaN 10.0 WALK_LRF 1.593075 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
64244634519586771024353social1True114980305793socialNaN10.0WALK_LRF1.593075
\n
" }, - "execution_count": 34, + "execution_count": 198, "metadata": {}, "output_type": "execute_result" } @@ -547,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": 183, + "execution_count": null, "outputs": [ { "name": "stderr", @@ -562,22 +555,15 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (10000 trips)\n", - "Done\n" + "trip_mode_choice tour_type 'social' (100000 trips)\n" ] - }, - { - "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004262 4.103657e-14 0.000084 0.00025 0.000114 \ntrip_mode 0.003800 0.000000e+00 0.000000 0.00020 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.039196 0.19705 0.759044 \ntrip_mode 0.038000 0.20070 0.757300 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0042624.103657e-140.0000840.000250.0001140.0391960.197050.759044
trip_mode0.0038000.000000e+000.0000000.000200.0000000.0380000.200700.757300
\n
" - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ - "num_samples = 10000\n", - "trip_id_to_check = 642446345 # 1e5 takes about 25s\n", + "%%time\n", + "\n", + "num_samples = 100000\n", + "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like XXX and 1e4 takes 1m21s\n", "# work 615236801 also in trace log\n", "# social 2464104885 642446345 1767182945\n", "\n", @@ -594,78 +580,52 @@ "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%%\n", + "is_executing": true } } }, { - "cell_type": "markdown", - "source": [ - "# Investigate differing probs\n", - "\n", - "is it due to clipping, due to idxmax always returning the first and by coincidence having equal probs somewhere, or\n", - "bad methodology?" - ], + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { - "name": "#%% md\n" + "name": "#%%\n" } } }, { "cell_type": "code", - "execution_count": 121, + "execution_count": null, "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", "source": [ - "def compute_probs(nested_utils, nest_spec):\n", - " nested_exp_utils = compute_nested_probabilities(np.exp(nested_utils), nest_spec)\n", - " base_probabilities = pd.DataFrame(index=nested_exp_utils.index)\n", - " for nest in logit.each_nest(nest_spec, type='leaf', post_order=False):\n", - " # skip root: it has a prob of 1 but we didn't compute a nested probability column for it\n", - " ancestors = nest.ancestors[1:]\n", - " base_probabilities[nest.name] = nested_exp_utils[ancestors].prod(axis=1)\n", - " return base_probabilities\n", - "\n", - "def compute_nested_probabilities(nested_exp_utilities, nest_spec):\n", - " nested_probabilities = pd.DataFrame(index=nested_exp_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, type='node', post_order=False):\n", - " probs = u_to_p(nested_exp_utilities[nest.alternatives])\n", - " nested_probabilities = pd.concat([nested_probabilities, probs], axis=1)\n", - " return nested_probabilities\n", - "\n", - "EXP_UTIL_MIN = 1e-300\n", - "EXP_UTIL_MAX = np.inf\n", - "\n", - "PROB_MIN = 0.0\n", - "PROB_MAX = 1.0\n", + "## try to improve runtime of apply\n", "\n", - "def u_to_p(utils, trace_label=None):\n", - " utils_arr = utils.values\n", - " np.clip(utils_arr, EXP_UTIL_MIN, EXP_UTIL_MAX, out=utils_arr)\n", - " utils_arr = np.where(utils_arr == EXP_UTIL_MIN, 0.0, utils_arr)\n", - " arr_sum = utils_arr.sum(axis=1)\n", - " inf_utils = np.isinf(arr_sum)\n", - " if inf_utils.any():\n", - " print(\"INF\")\n", - " with np.errstate(invalid='ignore', divide='ignore'):\n", - " np.divide(utils_arr, arr_sum.reshape(len(utils_arr), 1), out=utils_arr)\n", - " ## if allow_zero_probs, this will cause EXP_UTIL_MIN util rows to have all zero probabilities\n", - " utils_arr[np.isnan(utils_arr)] = PROB_MIN\n", - " np.clip(utils_arr, PROB_MIN, PROB_MAX, out=utils_arr)\n", - " probs = pd.DataFrame(utils_arr, columns=utils.columns, index=utils.index)\n", - " return probs" + "try caching {nest_name: nest alternatives}\n" ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } } }, { "cell_type": "code", - "execution_count": 177, + "execution_count": 184, "outputs": [], "source": [ "from activitysim.core import logit\n", @@ -681,34 +641,29 @@ " df = df + logit.inverse_ev1_cdf(uniform_rands)\n", " return df\n", "\n", - "def group_nests_by_level(nest_spec):\n", - " # group nests by level, returns {level: [nest.name at that level]}\n", - " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in logit.each_nest(nest_spec):\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1 # only one root\n", - " return nest_levels\n", - "\n", - "def make_choices_ru_frozen(nested_utilities, nest_spec):\n", - " nest_utils_for_choice = add_ev1_random(nested_utilities)\n", - " all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, logit.each_nest(nest_spec))))\n", - "\n", - " def is_alternative(name):\n", - " return name in all_alternatives\n", + "def make_choices_ru_frozen_apply(nested_utilities, nest_spec):\n", + " all_alternatives = [nest.name for nest in logit.each_nest(nest_spec, type='leaf')]\n", "\n", - " nest_utils_for_choice[\"choice\"] = None\n", - " for level, alts in group_nests_by_level(nest_spec).items():\n", + " for level, nests_at_level in logit.group_nests_by_level(nest_spec).items():\n", + " nest_alts = [nest.name for nest in nests_at_level]\n", " if level == 1:\n", + " assert len(nests_at_level) == 1\n", + " assert len(nest_alts) == 1\n", + " next_level_alts = nests_at_level[0].alternatives\n", " continue\n", - " no_choices_made_yet = nest_utils_for_choice[\"choice\"].isnull()\n", - " choice_this_level = nest_utils_for_choice.loc[no_choices_made_yet][alts].idxmax(1)\n", - " nest_utils_for_choice.loc[no_choices_made_yet, \"choice\"] = \\\n", - " np.where(choice_this_level.apply(is_alternative), choice_this_level, None)\n", "\n", - " assert not nest_utils_for_choice[\"choice\"].isnull().any(), \"No choice for XXX - implement reporting\"\n", - " choices = pd.Series(nest_utils_for_choice[\"choice\"], index=nested_utilities.index)\n", - " return choices, nest_utils_for_choice" + " # all alternatives from the previous level\n", + " alts_this_level = list(filter(lambda x: x in next_level_alts, nest_alts))\n", + " #print(f\"Level {level} alternatives are {alts_this_level}\")\n", + " choice_this_level = nested_utilities[nested_utilities.index.isin(alts_this_level)].idxmax()\n", + " #print(f\"choice is {choice_this_level}\")\n", + " if choice_this_level in all_alternatives:\n", + " return choice_this_level\n", + " chosen_nest = list(filter(lambda x: x.name == choice_this_level, nests_at_level))\n", + " assert len(chosen_nest) == 1\n", + " next_level_alts = chosen_nest[0].alternatives\n", + "\n", + " raise ValueError(\"This should never happen - no alternative found\")" ], "metadata": { "collapsed": false, @@ -719,7 +674,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 185, "outputs": [ { "data": { @@ -750,89 +705,30 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 186, "outputs": [ { - "data": { - "text/plain": "WALK_LRF 0.78148\nWALK_LOC 0.20604\nWALK 0.01194\nSHARED2FREE 0.00054\nName: choice, dtype: float64" - }, - "execution_count": 179, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "choices, nu_choices = make_choices_ru_frozen(nest_util, nest_spec)\n", - "choices.value_counts() / n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 180, - "outputs": [], - "source": [ - "def group_nests_by_level(nest_spec):\n", - " # group nests by level, returns {level: [nest]}\n", - " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in logit.each_nest(nest_spec):\n", - " nest_levels[n.level].append(n)\n", - " assert len(nest_levels[1]) == 1 # only one root\n", - " return nest_levels\n", - "\n", - "def make_choices_ru_frozen_apply(nested_utilities, nest_spec):\n", - " nest_utils_for_choice = add_ev1_random(nested_utilities)\n", - " all_alternatives = [nest.name for nest in logit.each_nest(nest_spec, type='leaf')]\n", - "\n", - " for level, nests_at_level in group_nests_by_level(nest_spec).items():\n", - " nest_alts = [nest.name for nest in nests_at_level]\n", - " if level == 1:\n", - " assert len(nests_at_level) == 1\n", - " assert len(nest_alts) == 1\n", - " next_level_alts = nests_at_level[0].alternatives\n", - " continue\n", - "\n", - " # all alternatives from the previous level\n", - " alts_this_level = list(filter(lambda x: x in next_level_alts, nest_alts))\n", - " #print(f\"Level {level} alternatives are {alts_this_level}\")\n", - " choice_this_level = nest_utils_for_choice[nest_utils_for_choice.index.isin(alts_this_level)].idxmax()\n", - " #print(f\"choice is {choice_this_level}\")\n", - " if choice_this_level in all_alternatives:\n", - " return choice_this_level\n", - " chosen_nest = list(filter(lambda x: x.name == choice_this_level, nests_at_level))\n", - " assert len(chosen_nest) == 1\n", - " next_level_alts = chosen_nest[0].alternatives\n", - "\n", - " raise ValueError(\"This should never happen - no alternative found\")\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 182, - "outputs": [ + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1min 8s, sys: 1.91 s, total: 1min 10s\n", + "Wall time: 1min 9s\n" + ] + }, { "data": { - "text/plain": "WALK_LRF 0.75972\nWALK_LOC 0.19677\nWALK 0.03901\nSHARED2FREE 0.00406\nTNC_SHARED 0.00025\nTNC_SINGLE 0.00010\nTAXI 0.00009\ndtype: float64" + "text/plain": "WALK_LRF 0.75759\nWALK_LOC 0.19934\nWALK 0.03826\nSHARED2FREE 0.00428\nTNC_SHARED 0.00027\nTNC_SINGLE 0.00017\nTAXI 0.00009\ndtype: float64" }, - "execution_count": 182, + "execution_count": 186, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "choices_apply = nest_util.apply(lambda x: make_choices_ru_frozen_apply(x, nest_spec), axis=1)\n", + "%%time\n", + "\n", + "nest_utils_for_choice = add_ev1_random(nest_util)\n", + "choices_apply = nest_utils_for_choice.apply(lambda x: make_choices_ru_frozen_apply(x, nest_spec), axis=1)\n", "choices_apply.value_counts() / n" ], "metadata": { From 6c17d89bace65edff58d5426cbc34a6c8128518f Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 8 May 2022 22:26:14 +1000 Subject: [PATCH 020/135] caching nest level names and alternatives --- activitysim/core/logit.py | 34 +++--- notebooks/validate_frozen_impl.ipynb | 159 ++++++++++++++------------- 2 files changed, 98 insertions(+), 95 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 8166607945..8e563ac3d7 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -210,24 +210,15 @@ def add_ev1_random(df): return nest_utils_for_choice -def choose_from_tree(nest_utils, nest_spec): - all_alternatives = [nest.name for nest in each_nest(nest_spec, type='leaf')] - for level, nests_at_level in group_nests_by_level(nest_spec).items(): - nest_alts = [nest.name for nest in nests_at_level] +def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alternatives_by_name): + for level, nest_names in logit_nest_groups.items(): if level == 1: - assert len(nests_at_level) == 1 - assert len(nest_alts) == 1 - next_level_alts = nests_at_level[0].alternatives + next_level_alts = nest_alternatives_by_name[nest_names[0]] continue - # all alternatives from the previous level - alts_this_level = list(filter(lambda x: x in next_level_alts, nest_alts)) - choice_this_level = nest_utils[nest_utils.index.isin(alts_this_level)].idxmax() + choice_this_level = nest_utils[nest_utils.index.isin(next_level_alts)].idxmax() if choice_this_level in all_alternatives: return choice_this_level - chosen_nest = list(filter(lambda x: x.name == choice_this_level, nests_at_level)) - assert len(chosen_nest) == 1 - next_level_alts = chosen_nest[0].alternatives - + next_level_alts = nest_alternatives_by_name[choice_this_level] raise ValueError("This should never happen - no alternative found") @@ -235,8 +226,15 @@ def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_ """ walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') nest_utils_for_choice = add_ev1_random(nested_utilities) - # TODO: the following apply is slow, try to improve it - choices = nest_utils_for_choice.apply(lambda x: choose_from_tree(x, nest_spec), axis=1) + + all_alternatives = set(nest.name for nest in each_nest(nest_spec, type='leaf')) + logit_nest_groups = group_nest_names_by_level(nest_spec) + nest_alternatives_by_name = {n.name: n.alternatives for n in each_nest(nest_spec)} + + choices = nest_utils_for_choice.apply( + lambda x: choose_from_tree(x, all_alternatives, logit_nest_groups, nest_alternatives_by_name), + axis=1 + ) assert not choices.isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=nested_utilities.index) return choices @@ -534,10 +532,10 @@ def count_each_nest(spec, count): return count_each_nest(nest_spec, 0) if nest_spec is not None else 0 -def group_nests_by_level(nest_spec): +def group_nest_names_by_level(nest_spec): # group nests by level, returns {level: [nest.name at that level]} depth = np.max([x.level for x in each_nest(nest_spec)]) nest_levels = {x: [] for x in range(1, depth+1)} for n in each_nest(nest_spec): - nest_levels[n.level].append(n) + nest_levels[n.level].append(n.name) return nest_levels diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 00fd6d8c7b..0a00395558 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -555,14 +555,14 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (100000 trips)\n" + "trip_mode_choice tour_type 'social' (100 trips)\n" ] } ], "source": [ "%%time\n", "\n", - "num_samples = 100000\n", + "num_samples = 100\n", "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like XXX and 1e4 takes 1m21s\n", "# work 615236801 also in trace log\n", "# social 2464104885 642446345 1767182945\n", @@ -598,38 +598,68 @@ } }, { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "cell_type": "markdown", + "source": [ + "## try to improve runtime of apply\n", + "\n", + "try caching {nest_name: nest alternatives}\n" + ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } } }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 239, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n642446345 0.0 0.0 0.004883 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n642446345 0.0 0.0 0.000096 0.000131 0.000287 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
6424463450.00.00.0048830.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.00.0000960.0001310.000287
\n
" + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "## try to improve runtime of apply\n", + "nest_spec = ns[7] # 7 is social\n", + "nest_util = nu[7].loc[nu[7].index == trip_id_to_check]\n", "\n", - "try caching {nest_name: nest alternatives}\n" + "n = 10000\n", + "\n", + "nest_util = nest_util.loc[nest_util.index.repeat(n)]\n", + "nest_util.index += np.arange(n)\n", + "\n", + "display(base_probs[7].loc[base_probs[7].index == trip_id_to_check])" ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%% md\n" + "name": "#%%\n" } } }, { "cell_type": "code", - "execution_count": 184, - "outputs": [], + "execution_count": 240, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 31.2 ms, sys: 0 ns, total: 31.2 ms\n", + "Wall time: 21.1 ms\n" + ] + } + ], "source": [ - "from activitysim.core import logit\n", + "%%time\n", + "\n", "from numpy.random import default_rng\n", + "from activitysim.core import logit\n", "\n", "rng = default_rng(999)\n", "\n", @@ -641,28 +671,37 @@ " df = df + logit.inverse_ev1_cdf(uniform_rands)\n", " return df\n", "\n", - "def make_choices_ru_frozen_apply(nested_utilities, nest_spec):\n", - " all_alternatives = [nest.name for nest in logit.each_nest(nest_spec, type='leaf')]\n", + "nest_utils_for_choice = add_ev1_random(nest_util)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 241, + "outputs": [], + "source": [ + "def group_nests_by_name(nest_spec):\n", + " # group nests by level, returns {level: [nest.name at that level]}\n", + " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", + " nest_levels = {x: [] for x in range(1, depth+1)}\n", + " for n in logit.each_nest(nest_spec):\n", + " nest_levels[n.level].append(n.name)\n", + " return nest_levels\n", "\n", - " for level, nests_at_level in logit.group_nests_by_level(nest_spec).items():\n", - " nest_alts = [nest.name for nest in nests_at_level]\n", + "def make_choices_apply(nested_utilities, all_alternatives, logit_nest_groups, nest_alternatives_by_name):\n", + " for level, nest_names in logit_nest_groups.items():\n", " if level == 1:\n", - " assert len(nests_at_level) == 1\n", - " assert len(nest_alts) == 1\n", - " next_level_alts = nests_at_level[0].alternatives\n", + " next_level_alts = nest_alternatives_by_name[nest_names[0]]\n", " continue\n", - "\n", - " # all alternatives from the previous level\n", - " alts_this_level = list(filter(lambda x: x in next_level_alts, nest_alts))\n", - " #print(f\"Level {level} alternatives are {alts_this_level}\")\n", - " choice_this_level = nested_utilities[nested_utilities.index.isin(alts_this_level)].idxmax()\n", - " #print(f\"choice is {choice_this_level}\")\n", + " choice_this_level = nested_utilities[nested_utilities.index.isin(next_level_alts)].idxmax()\n", " if choice_this_level in all_alternatives:\n", " return choice_this_level\n", - " chosen_nest = list(filter(lambda x: x.name == choice_this_level, nests_at_level))\n", - " assert len(chosen_nest) == 1\n", - " next_level_alts = chosen_nest[0].alternatives\n", - "\n", + " next_level_alts = nest_alternatives_by_name[choice_this_level]\n", " raise ValueError(\"This should never happen - no alternative found\")" ], "metadata": { @@ -674,27 +713,17 @@ }, { "cell_type": "code", - "execution_count": 185, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n642446345 0.0 0.0 0.004262 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n642446345 4.103657e-14 0.0 0.039196 0.0 0.19705 0.759044 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n642446345 0.0 0.0 0.000084 0.000114 0.00025 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
6424463450.00.00.0042620.04.103657e-140.00.0391960.00.197050.7590440.00.00.00.00.00.00.00.00.0000840.0001140.00025
\n
" - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": 242, + "outputs": [], "source": [ - "nest_spec = ns[7] # 7 is social\n", - "nest_util = nu[7].loc[nu[7].index == trip_id_to_check]\n", - "\n", - "n = 100000\n", + "all_alternatives = set(nest.name for nest in logit.each_nest(nest_spec, type='leaf'))\n", + "logit_nest_groups = group_nests_by_name(nest_spec)\n", + "nest_alternatives_by_name = {n.name: n.alternatives for n in logit.each_nest(nest_spec)}\n", "\n", - "nest_util = nest_util.loc[nest_util.index.repeat(n)]\n", - "nest_util.index += np.arange(n)\n", - "\n", - "display(base_probs[7].loc[base_probs[7].index == trip_id_to_check])" + "choices_apply = nest_utils_for_choice.apply(\n", + " lambda x: make_choices_apply(x, all_alternatives, logit_nest_groups, nest_alternatives_by_name),\n", + " axis=1\n", + ")" ], "metadata": { "collapsed": false, @@ -705,30 +734,18 @@ }, { "cell_type": "code", - "execution_count": 186, + "execution_count": 243, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1min 8s, sys: 1.91 s, total: 1min 10s\n", - "Wall time: 1min 9s\n" - ] - }, { "data": { - "text/plain": "WALK_LRF 0.75759\nWALK_LOC 0.19934\nWALK 0.03826\nSHARED2FREE 0.00428\nTNC_SHARED 0.00027\nTNC_SINGLE 0.00017\nTAXI 0.00009\ndtype: float64" + "text/plain": "WALK_LRF 0.8383\nWALK_LOC 0.1158\nWALK 0.0405\nSHARED2FREE 0.0048\nTNC_SHARED 0.0003\nTAXI 0.0002\nTNC_SINGLE 0.0001\ndtype: float64" }, - "execution_count": 186, + "execution_count": 243, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "\n", - "nest_utils_for_choice = add_ev1_random(nest_util)\n", - "choices_apply = nest_utils_for_choice.apply(lambda x: make_choices_ru_frozen_apply(x, nest_spec), axis=1)\n", "choices_apply.value_counts() / n" ], "metadata": { @@ -774,18 +791,6 @@ } } }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "markdown", "source": [ From 798439208bf923919432f28bf5ae0f59e19412ec Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 8 May 2022 22:45:27 +1000 Subject: [PATCH 021/135] removes trace in runtime tests --- activitysim/core/logit.py | 2 +- notebooks/validate_frozen_impl.ipynb | 220 +++++---------------------- 2 files changed, 35 insertions(+), 187 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 8e563ac3d7..40a3a8671f 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -236,7 +236,7 @@ def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_ axis=1 ) assert not choices.isnull().any(), "No choice for XXX - implement reporting" - choices = pd.Series(choices, index=nested_utilities.index) + choices = pd.Series(choices, index=nest_utils_for_choice.index) return choices diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 0a00395558..130f805135 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 3, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -78,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 4, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -97,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 5, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -117,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 6, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -136,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 7, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -163,14 +163,14 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 8, "outputs": [], "source": [ - "# need to do this before loading checkpoint tables\n", - "trace_hh_ids = 1024353\n", - "#1024353 has trip id 642446345 (at least) where logsum and probs are different\n", - "#2821179 # has trip_ids 2464104881 and 2464104885\n", - "inject.add_injectable(\"trace_hh_id\", trace_hh_ids)" + "## need to do this before loading checkpoint tables\n", + "#trace_hh_ids = 1024353\n", + "##1024353 has trip id 642446345 (at least) where logsum and probs are different\n", + "##2821179 # has trip_ids 2464104881 and 2464104885\n", + "#inject.add_injectable(\"trace_hh_id\", trace_hh_ids)" ], "metadata": { "collapsed": false, @@ -181,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 9, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -198,7 +198,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [1024353].\n", + "setting trace_hh_id is wrong type, should be an int, but was \n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -219,20 +219,6 @@ "trip_mode_choice tour_type 'work' (168 trips)\n" ] }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [1024353].\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n" - ] - }, { "name": "stderr", "output_type": "stream", @@ -244,6 +230,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Done\n", "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", @@ -256,8 +243,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 18.1 s, sys: 2.36 s, total: 20.5 s\n", - "Wall time: 17.3 s\n" + "CPU times: user 16.9 s, sys: 2.78 s, total: 19.7 s\n", + "Wall time: 14.7 s\n" ] } ], @@ -269,14 +256,14 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 10, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 308 304\nWALK_LOC 87 85\nWALK_LRF 54 53\nBIKE 15 17\nTNC_SINGLE 7 4\nDRIVEALONEFREE 5 4\nSHARED2FREE 3 4\nSHARED3FREE 2 1\nWALK_HVY 1 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK308304
WALK_LOC8785
WALK_LRF5453
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n
" }, - "execution_count": 188, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -296,7 +283,7 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 11, "outputs": [ { "data": { @@ -324,57 +311,12 @@ }, { "cell_type": "code", - "execution_count": 190, - "outputs": [ - { - "data": { - "text/plain": "trip_id\n106170305 4.050638\n106170309 0.553496\n1767013721 0.482067\n535694461 0.251889\n943749469 0.203583\n ... \n535672921 0.000000\n535620049 0.000000\n535388485 0.000000\n535388481 0.000000\n943749471 0.000000\nLength: 482, dtype: float64" - }, - "execution_count": 190, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ls_ = np.abs(((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru))\n", - "ls_.sort_values(ascending=False)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 191, - "outputs": [ - { - "data": { - "text/plain": " trip_mode_fru mode_choice_logsum_fru trip_mode_asim \\\ntrip_id \n642446345 WALK_LRF 1.634462 WALK_LRF \n\n mode_choice_logsum_asim \ntrip_id \n642446345 1.593075 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frumode_choice_logsum_frutrip_mode_asimmode_choice_logsum_asim
trip_id
642446345WALK_LRF1.634462WALK_LRF1.593075
\n
" - }, - "execution_count": 191, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trips.loc[trips.index == 642446345]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 192, + "execution_count": 12, "outputs": [], "source": [ + "#ls_ = np.abs(((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru))\n", + "#ls_.sort_values(ascending=False)\n", + "#trips.loc[trips.index == 642446345]\n", "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" ], "metadata": { @@ -384,18 +326,6 @@ } } }, - { - "cell_type": "code", - "execution_count": 192, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "markdown", "source": [ @@ -415,13 +345,12 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 13, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [1024353].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -457,96 +386,12 @@ }, { "cell_type": "code", - "execution_count": 194, - "outputs": [], - "source": [ - "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 195, - "outputs": [], - "source": [ - "#t.loc[t.index.isin(trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0].index.values)]\\\n", - "# .sort_values(by=[\"primary_purpose\"])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 196, - "outputs": [], - "source": [ - "# nest_spec = ns[0]\n", - "# for nest in logit.each_nest(nest_spec, post_order=False):\n", - "# print(f\"{nest.level}, {nest.name}, {nest.coefficient}, parent scale: {nest.parent_scale}\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 197, - "outputs": [], - "source": [ - "#base_probs[7] #615236801" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 198, - "outputs": [ - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n642446345 1958677 1024353 social 1 True \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n642446345 1 14 9 80305793 social \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n642446345 NaN 10.0 WALK_LRF 1.593075 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
64244634519586771024353social1True114980305793socialNaN10.0WALK_LRF1.593075
\n
" - }, - "execution_count": 198, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t.loc[t.index == 642446345]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 20, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "register joint_tour_participants: no rows with household_id in [1024353].\n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -555,15 +400,19 @@ "output_type": "stream", "text": [ "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (100 trips)\n" + "trip_mode_choice tour_type 'social' (100000 trips)\n", + "Done\n", + "CPU times: user 1min 1s, sys: 3.78 s, total: 1min 5s\n", + "Wall time: 1min 2s\n" ] } ], "source": [ "%%time\n", + "#prun\n", "\n", - "num_samples = 100\n", - "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like XXX and 1e4 takes 1m21s\n", + "num_samples = 100000\n", + "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like 65s and 1e4 takes 8s\n", "# work 615236801 also in trace log\n", "# social 2464104885 642446345 1767182945\n", "\n", @@ -575,13 +424,12 @@ "\n", "choose_individual_max_utility = True\n", "ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", - "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" + "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n", - "is_executing": true + "name": "#%%\n" } } }, From ebf64c6a238aee78283d4816ab28bf00d27bfc13 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 9 May 2022 17:23:50 +1000 Subject: [PATCH 022/135] larch notebook to calculate probs a different way --- activitysim/core/simulate.py | 5 - .../larch_probabilities_via_estimation.ipynb | 3044 +++++++++++++++++ notebooks/validate_frozen_impl.ipynb | 12 +- 3 files changed, 3052 insertions(+), 9 deletions(-) create mode 100644 notebooks/larch_probabilities_via_estimation.ipynb diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 7bdb5ed7e2..be03116968 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1044,11 +1044,6 @@ def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estima compute_nested_probabilities(np.exp(nested_utilities), nest_spec, trace_label=trace_label) chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) - if want_logsums: - # logsum of nest root - logsums = pd.Series(nested_utilities.root, index=choosers.index) - chunk.log_df(trace_label, "logsums", logsums) - if have_trace_targets: tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, column_labels=['alternative', 'probability']) diff --git a/notebooks/larch_probabilities_via_estimation.ipynb b/notebooks/larch_probabilities_via_estimation.ipynb new file mode 100644 index 0000000000..0bb5f0a04d --- /dev/null +++ b/notebooks/larch_probabilities_via_estimation.ipynb @@ -0,0 +1,3044 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": true, + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Probabilities for three level nests differ\n", + "\n", + "Asim scales nested utilties differently, for two level and RU2 it's identical to what we do (which agrees with the\n", + "larch paper and Zenith), but for three levels there are small differences of probabilities and logsums. I\n", + "quadruple-checked the scaling and leaf and note utility calculation, let's use larch to see if we get the same as\n", + "Asim or if the results agree with my implementation. The idea is to use estimation mode to load a small data set, and\n", + " then use my Asim setup in validate_frozen_impl.ipynb to calculate Asim's and my probabilities by sampling, and check\n", + " what larch has to say by using m.probabilities() (no need to estimate, use parameters as given by asim model).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "# I installed larch deps (which I collected in a requirements file, xmle and sharrow are not on pypi)\n", + "# python -m pip install -r ../larch/requirements.txt\n", + "\n", + "larch_path = \"/mnt/c/Users/jan.zill/code/larch\"\n", + "if larch_path not in sys.path:\n", + " sys.path.append(larch_path)\n", + "import larch" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 105, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_estimation_sf\")\n", + "os.chdir(example_dir)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 106, + "outputs": [], + "source": [ + "# create estimation test example\n", + "#!activitysim create -e example_estimation_sf -d test_estimation_sf" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 13, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Configured logging using basicConfig\r\n", + "INFO:activitysim:Configured logging using basicConfig\r\n", + "INFO - Read logging configuration from: configs_estimation/configs/logging.yaml\r\n", + "INFO - SETTING configs_dir: ['configs_estimation/configs', 'configs']\r\n", + "INFO - SETTING settings_file_name: settings.yaml\r\n", + "INFO - SETTING data_dir: ['data_sf']\r\n", + "INFO - SETTING output_dir: output\r\n", + "INFO - SETTING households_sample_size: 0\r\n", + "INFO - SETTING chunk_size: 0\r\n", + "INFO - SETTING chunk_method: hybrid_uss\r\n", + "INFO - SETTING chunk_training_mode: disabled\r\n", + "INFO - SETTING multiprocess: None\r\n", + "INFO - SETTING num_processes: None\r\n", + "INFO - SETTING resume_after: None\r\n", + "INFO - SETTING trace_hh_id: None\r\n", + "INFO - ENV MKL_NUM_THREADS: None\r\n", + "INFO - ENV OMP_NUM_THREADS: None\r\n", + "INFO - ENV OPENBLAS_NUM_THREADS: None\r\n", + "INFO - NUMPY blas_opt_info libraries: ['openblas', 'openblas']\r\n", + "INFO - NUMPY lapack_opt_info libraries: ['openblas', 'openblas']\r\n", + "INFO - run single process simulation\r\n", + "INFO - Time to execute open_pipeline : 0.029 seconds (0.0 minutes)\r\n", + "INFO - preload_injectables\r\n", + "INFO - Time to execute preload_injectables : 0.036 seconds (0.0 minutes)\r\n", + "INFO - #run_model running step initialize_landuse\r\n", + "Running step 'initialize_landuse'\r\n", + "INFO - Reading CSV file data_sf/land_use.csv\r\n", + "INFO - loaded land_use (190, 24)\r\n", + "INFO - initialize_landuse.annotate_tables - annotating land_use SPEC annotate_landuse\r\n", + "INFO - Network_LOS using skim_dict_factory: NumpyArraySkimFactory\r\n", + "INFO - allocate_skim_buffer shared False taz shape (826, 190, 190) total size: 119_274_400 (119.3 MB)\r\n", + "INFO - _read_skims_from_omx data_sf/skims.omx\r\n", + "INFO - _read_skims_from_omx loaded 826 skims from data_sf/skims.omx\r\n", + "INFO - writing skim cache taz (826, 190, 190) to output/cache/cached_taz.mmap\r\n", + "INFO - load_skims_to_buffer taz shape (826, 190, 190)\r\n", + "INFO - get_skim_data taz SkimData shape (826, 190, 190)\r\n", + "INFO - SkimDict init taz\r\n", + "INFO - SkimDict.build_3d_skim_block_offset_table registered 167 3d keys\r\n", + "Time to execute step 'initialize_landuse': 3.69 s\r\n", + "Total time to execute iteration 1 with iteration value None: 3.69 s\r\n", + "INFO - #run_model running step initialize_households\r\n", + "Running step 'initialize_households'\r\n", + "INFO - Reading CSV file data_sf/survey_data/override_households.csv\r\n", + "INFO - full household list contains 2000 households\r\n", + "INFO - loaded households (2000, 7)\r\n", + "INFO - Reading CSV file data_sf/survey_data/override_persons.csv\r\n", + "INFO - loaded persons (4405, 7)\r\n", + "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons\r\n", + "INFO - initialize_households.annotate_tables - annotating households SPEC annotate_households\r\n", + "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons_after_hh\r\n", + "Time to execute step 'initialize_households': 0.39 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.39 s\r\n", + "INFO - #run_model running step compute_accessibility\r\n", + "Running step 'compute_accessibility'\r\n", + "INFO - Running compute_accessibility with 190 orig zones 190 dest zones\r\n", + "INFO - compute_accessibility Running adaptive_chunked_choosers with 190 choosers\r\n", + "INFO - Running chunk 1 of 1 with 190 of 190 choosers\r\n", + "INFO - Running compute_accessibility with 190 orig zones 190 dest zones\r\n", + "INFO - compute_accessibility computed accessibilities (190, 10)\r\n", + "Time to execute step 'compute_accessibility': 0.12 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.12 s\r\n", + "INFO - #run_model running step school_location\r\n", + "Running step 'school_location'\r\n", + "INFO - Initialize Estimator for'school_location'\r\n", + "DEBUG - school_location: estimate.write_spec: output/estimation_data_bundle/school_location/school_location_SPEC.csv\r\n", + "DEBUG - school_location: write_table write: school_location_coefficients.csv\r\n", + "DEBUG - school_location: write_table write: size_terms\r\n", + "DEBUG - school_location: write_table write: landuse\r\n", + "INFO - Running school_location.i1.sample.university with 329 persons\r\n", + "INFO - Estimation mode for school_location.i1.sample.university using unsampled alternatives short_circuit_choices\r\n", + "INFO - school_location.i1.sample.university.interaction_sample Running adaptive_chunked_choosers with 329 choosers\r\n", + "INFO - Running chunk 1 of 1 with 329 of 329 choosers\r\n", + "INFO - Running eval_interaction_utilities on 5593 rows\r\n", + "INFO - Estimation mode for school_location.i1.sample.university.interaction_sample using unsampled alternatives\r\n", + "INFO - Running school_location.i1.logsums.university with 5593 rows\r\n", + "INFO - school_location.i1.logsums.university.compute_logsums Running adaptive_chunked_choosers with 5593 choosers\r\n", + "INFO - Running chunk 1 of 1 with 5593 of 5593 choosers\r\n", + "INFO - Running school_location.i1.simulate.university with 329 persons\r\n", + "DEBUG - school_location: write_table cache: choosers\r\n", + "DEBUG - school_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - school_location.i1.simulate.university.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 329 choosers and 5593 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 329 of 329 choosers\r\n", + "INFO - Running eval_interaction_utilities on 5593 rows\r\n", + "INFO - school_location: eval_interaction_utilities write_interaction_expression_values school_location.i1.simulate.university.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - school_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - school_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - school_location: write_table cache: override_choices\r\n", + "INFO - Running school_location.i1.sample.highschool with 111 persons\r\n", + "INFO - Estimation mode for school_location.i1.sample.highschool using unsampled alternatives short_circuit_choices\r\n", + "INFO - school_location.i1.sample.highschool.interaction_sample Running adaptive_chunked_choosers with 111 choosers\r\n", + "INFO - Running chunk 1 of 1 with 111 of 111 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2775 rows\r\n", + "INFO - Estimation mode for school_location.i1.sample.highschool.interaction_sample using unsampled alternatives\r\n", + "INFO - Running school_location.i1.logsums.highschool with 2775 rows\r\n", + "INFO - school_location.i1.logsums.highschool.compute_logsums Running adaptive_chunked_choosers with 2775 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2775 of 2775 choosers\r\n", + "INFO - Running school_location.i1.simulate.highschool with 111 persons\r\n", + "DEBUG - school_location: write_table cache: choosers\r\n", + "DEBUG - school_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - school_location.i1.simulate.highschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 111 choosers and 2775 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 111 of 111 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2775 rows\r\n", + "INFO - school_location: eval_interaction_utilities write_interaction_expression_values school_location.i1.simulate.highschool.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - school_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - school_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - school_location: write_table cache: override_choices\r\n", + "INFO - Running school_location.i1.sample.gradeschool with 544 persons\r\n", + "INFO - Estimation mode for school_location.i1.sample.gradeschool using unsampled alternatives short_circuit_choices\r\n", + "INFO - school_location.i1.sample.gradeschool.interaction_sample Running adaptive_chunked_choosers with 544 choosers\r\n", + "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", + "INFO - Running eval_interaction_utilities on 103360 rows\r\n", + "INFO - Estimation mode for school_location.i1.sample.gradeschool.interaction_sample using unsampled alternatives\r\n", + "INFO - Running school_location.i1.logsums.gradeschool with 103360 rows\r\n", + "INFO - school_location.i1.logsums.gradeschool.compute_logsums Running adaptive_chunked_choosers with 103360 choosers\r\n", + "INFO - Running chunk 1 of 1 with 103360 of 103360 choosers\r\n", + "INFO - Running school_location.i1.simulate.gradeschool with 544 persons\r\n", + "DEBUG - school_location: write_table cache: choosers\r\n", + "DEBUG - school_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - school_location.i1.simulate.gradeschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 544 choosers and 103360 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", + "INFO - Running eval_interaction_utilities on 103360 rows\r\n", + "INFO - school_location: eval_interaction_utilities write_interaction_expression_values school_location.i1.simulate.gradeschool.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - school_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - school_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - school_location: write_table cache: override_choices\r\n", + "INFO - write_trace_files iteration 1\r\n", + "INFO - school_location_logsum top 10 value counts:\r\n", + "-1.824183 2\r\n", + "-0.909839 2\r\n", + "-1.366984 2\r\n", + "-1.278060 1\r\n", + "-1.180527 1\r\n", + "-1.674889 1\r\n", + "-1.291101 1\r\n", + "-1.813244 1\r\n", + "-1.069701 1\r\n", + " 0.053658 1\r\n", + "Name: logsum, dtype: int64\r\n", + "DEBUG - school_location: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - school_location: write_omnibus_choosers: output/estimation_data_bundle/school_location/school_location_choosers_combined.csv\r\n", + "DEBUG - school_location: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - school_location: write_omnibus_choosers: output/estimation_data_bundle/school_location/school_location_alternatives_combined.csv\r\n", + "INFO - school_location: end estimation\r\n", + "Time to execute step 'school_location': 15.20 s\r\n", + "Total time to execute iteration 1 with iteration value None: 15.21 s\r\n", + "INFO - #run_model running step workplace_location\r\n", + "Running step 'workplace_location'\r\n", + "INFO - Initialize Estimator for'workplace_location'\r\n", + "DEBUG - workplace_location: estimate.write_spec: output/estimation_data_bundle/workplace_location/workplace_location_SPEC.csv\r\n", + "DEBUG - workplace_location: write_table write: workplace_location_coefficients.csv\r\n", + "DEBUG - workplace_location: write_table write: size_terms\r\n", + "DEBUG - workplace_location: write_table write: landuse\r\n", + "INFO - Running workplace_location.i1.sample.work_low with 518 persons\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_low using unsampled alternatives short_circuit_choices\r\n", + "INFO - workplace_location.i1.sample.work_low.interaction_sample Running adaptive_chunked_choosers with 518 choosers\r\n", + "INFO - Running chunk 1 of 1 with 518 of 518 choosers\r\n", + "INFO - Running eval_interaction_utilities on 98420 rows\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_low.interaction_sample using unsampled alternatives\r\n", + "INFO - Running workplace_location.i1.logsums.work_low with 98420 rows\r\n", + "INFO - workplace_location.i1.logsums.work_low.compute_logsums Running adaptive_chunked_choosers with 98420 choosers\r\n", + "INFO - Running chunk 1 of 1 with 98420 of 98420 choosers\r\n", + "INFO - Running workplace_location.i1.simulate.work_low with 518 persons\r\n", + "DEBUG - workplace_location: write_table cache: choosers\r\n", + "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - workplace_location.i1.simulate.work_low.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 518 choosers and 98420 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 518 of 518 choosers\r\n", + "INFO - Running eval_interaction_utilities on 98420 rows\r\n", + "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_low.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - workplace_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - workplace_location: write_table cache: override_choices\r\n", + "INFO - Running workplace_location.i1.sample.work_med with 545 persons\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_med using unsampled alternatives short_circuit_choices\r\n", + "INFO - workplace_location.i1.sample.work_med.interaction_sample Running adaptive_chunked_choosers with 545 choosers\r\n", + "INFO - Running chunk 1 of 1 with 545 of 545 choosers\r\n", + "INFO - Running eval_interaction_utilities on 103550 rows\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_med.interaction_sample using unsampled alternatives\r\n", + "INFO - Running workplace_location.i1.logsums.work_med with 103550 rows\r\n", + "INFO - workplace_location.i1.logsums.work_med.compute_logsums Running adaptive_chunked_choosers with 103550 choosers\r\n", + "INFO - Running chunk 1 of 1 with 103550 of 103550 choosers\r\n", + "INFO - Running workplace_location.i1.simulate.work_med with 545 persons\r\n", + "DEBUG - workplace_location: write_table cache: choosers\r\n", + "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - workplace_location.i1.simulate.work_med.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 545 choosers and 103550 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 545 of 545 choosers\r\n", + "INFO - Running eval_interaction_utilities on 103550 rows\r\n", + "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_med.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - workplace_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - workplace_location: write_table cache: override_choices\r\n", + "INFO - Running workplace_location.i1.sample.work_high with 614 persons\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_high using unsampled alternatives short_circuit_choices\r\n", + "INFO - workplace_location.i1.sample.work_high.interaction_sample Running adaptive_chunked_choosers with 614 choosers\r\n", + "INFO - Running chunk 1 of 1 with 614 of 614 choosers\r\n", + "INFO - Running eval_interaction_utilities on 116660 rows\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_high.interaction_sample using unsampled alternatives\r\n", + "INFO - Running workplace_location.i1.logsums.work_high with 116660 rows\r\n", + "INFO - workplace_location.i1.logsums.work_high.compute_logsums Running adaptive_chunked_choosers with 116660 choosers\r\n", + "INFO - Running chunk 1 of 1 with 116660 of 116660 choosers\r\n", + "INFO - Running workplace_location.i1.simulate.work_high with 614 persons\r\n", + "DEBUG - workplace_location: write_table cache: choosers\r\n", + "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - workplace_location.i1.simulate.work_high.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 614 choosers and 116660 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 614 of 614 choosers\r\n", + "INFO - Running eval_interaction_utilities on 116660 rows\r\n", + "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_high.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - workplace_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - workplace_location: write_table cache: override_choices\r\n", + "INFO - Running workplace_location.i1.sample.work_veryhigh with 905 persons\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_veryhigh using unsampled alternatives short_circuit_choices\r\n", + "INFO - workplace_location.i1.sample.work_veryhigh.interaction_sample Running adaptive_chunked_choosers with 905 choosers\r\n", + "INFO - Running chunk 1 of 1 with 905 of 905 choosers\r\n", + "INFO - Running eval_interaction_utilities on 171950 rows\r\n", + "INFO - Estimation mode for workplace_location.i1.sample.work_veryhigh.interaction_sample using unsampled alternatives\r\n", + "INFO - Running workplace_location.i1.logsums.work_veryhigh with 171950 rows\r\n", + "INFO - workplace_location.i1.logsums.work_veryhigh.compute_logsums Running adaptive_chunked_choosers with 171950 choosers\r\n", + "INFO - Running chunk 1 of 1 with 171950 of 171950 choosers\r\n", + "INFO - Running workplace_location.i1.simulate.work_veryhigh with 905 persons\r\n", + "DEBUG - workplace_location: write_table cache: choosers\r\n", + "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", + "INFO - workplace_location.i1.simulate.work_veryhigh.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 905 choosers and 171950 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 905 of 905 choosers\r\n", + "INFO - Running eval_interaction_utilities on 171950 rows\r\n", + "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_veryhigh.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", + "DEBUG - workplace_location: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - workplace_location: write_table cache: override_choices\r\n", + "INFO - write_trace_files iteration 1\r\n", + "INFO - workplace_location_logsum top 10 value counts:\r\n", + "-0.007794 2\r\n", + " 1.586426 2\r\n", + "-1.938350 2\r\n", + " 0.491470 2\r\n", + " 0.267120 2\r\n", + " 0.047726 2\r\n", + " 0.657742 1\r\n", + "-0.051882 1\r\n", + " 6.017921 1\r\n", + "-0.325755 1\r\n", + "Name: logsum, dtype: int64\r\n", + "DEBUG - workplace_location: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - workplace_location: write_omnibus_choosers: output/estimation_data_bundle/workplace_location/workplace_location_choosers_combined.csv\r\n", + "DEBUG - workplace_location: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - workplace_location: write_omnibus_choosers: output/estimation_data_bundle/workplace_location/workplace_location_alternatives_combined.csv\r\n", + "INFO - workplace_location: end estimation\r\n", + "Time to execute step 'workplace_location': 51.96 s\r\n", + "Total time to execute iteration 1 with iteration value None: 51.96 s\r\n", + "INFO - #run_model running step auto_ownership_simulate\r\n", + "Running step 'auto_ownership_simulate'\r\n", + "INFO - Initialize Estimator for'auto_ownership'\r\n", + "INFO - Running auto_ownership_simulate with 2000 households\r\n", + "DEBUG - auto_ownership: estimate.write_spec: output/estimation_data_bundle/auto_ownership/auto_ownership_SPEC.csv\r\n", + "DEBUG - auto_ownership: write_table write: auto_ownership_coefficients.csv\r\n", + "DEBUG - auto_ownership: write_table cache: choosers\r\n", + "INFO - auto_ownership_simulate.simple_simulate Running adaptive_chunked_choosers with 2000 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2000 of 2000 choosers\r\n", + "DEBUG - auto_ownership: write_table cache: expression_values\r\n", + "DEBUG - auto_ownership: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using households.index\r\n", + "DEBUG - auto_ownership: write_table cache: override_choices\r\n", + "DEBUG - auto_ownership: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - auto_ownership: write_omnibus_choosers: output/estimation_data_bundle/auto_ownership/auto_ownership_values_combined.csv\r\n", + "INFO - auto_ownership: end estimation\r\n", + "INFO - auto_ownership top 10 value counts:\r\n", + "1 883\r\n", + "0 615\r\n", + "2 390\r\n", + "3 74\r\n", + "4 38\r\n", + "Name: auto_ownership, dtype: int64\r\n", + "Time to execute step 'auto_ownership_simulate': 0.41 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.41 s\r\n", + "INFO - #run_model running step free_parking\r\n", + "Running step 'free_parking'\r\n", + "INFO - Running free_parking with 2582 persons\r\n", + "INFO - Initialize Estimator for'free_parking'\r\n", + "DEBUG - free_parking: estimate.write_spec: output/estimation_data_bundle/free_parking/free_parking_SPEC.csv\r\n", + "DEBUG - free_parking: write_table write: free_parking_coefficients.csv\r\n", + "DEBUG - free_parking: write_table cache: choosers\r\n", + "INFO - free_parking.simple_simulate Running adaptive_chunked_choosers with 2582 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2582 of 2582 choosers\r\n", + "DEBUG - free_parking: write_table cache: expression_values\r\n", + "DEBUG - free_parking: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - free_parking: write_table cache: override_choices\r\n", + "DEBUG - free_parking: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - free_parking: write_omnibus_choosers: output/estimation_data_bundle/free_parking/free_parking_values_combined.csv\r\n", + "INFO - free_parking: end estimation\r\n", + "INFO - free_parking top 10 value counts:\r\n", + "False 4272\r\n", + "True 133\r\n", + "Name: free_parking_at_work, dtype: int64\r\n", + "Time to execute step 'free_parking': 0.31 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.31 s\r\n", + "INFO - #run_model running step cdap_simulate\r\n", + "Running step 'cdap_simulate'\r\n", + "INFO - Initialize Estimator for'cdap'\r\n", + "INFO - Pre-building cdap specs\r\n", + "INFO - Time to execute build_cdap_spec hh_size 2 : 0.214 seconds (0.0 minutes)\r\n", + "INFO - Time to execute build_cdap_spec hh_size 3 : 0.5 seconds (0.0 minutes)\r\n", + "INFO - Time to execute build_cdap_spec hh_size 4 : 1.268 seconds (0.0 minutes)\r\n", + "INFO - Time to execute build_cdap_spec hh_size 5 : 3.119 seconds (0.1 minutes)\r\n", + "DEBUG - cdap: estimate.write_spec: output/estimation_data_bundle/cdap/cdap_INDIV_AND_HHSIZE1_SPEC.csv\r\n", + "DEBUG - cdap: estimate.write_spec: output/estimation_data_bundle/cdap/cdap_FIXED_RELATIVE_PROPORTIONS_SPEC.csv\r\n", + "DEBUG - cdap: write_table write: cdap_coefficients.csv\r\n", + "DEBUG - cdap: write_table write: interaction_coefficients\r\n", + "DEBUG - cdap: write_table cache: choosers\r\n", + "DEBUG - cdap: write_table write: spec_2\r\n", + "DEBUG - cdap: write_table write: spec_3\r\n", + "DEBUG - cdap: write_table write: spec_4\r\n", + "DEBUG - cdap: write_table write: spec_5\r\n", + "INFO - Running cdap_simulate with 4405 persons\r\n", + "INFO - cdap.cdap Running chunk 1 of 1 with 2000 of 2000 choosers\r\n", + "DEBUG - cdap: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - cdap: write_table cache: override_choices\r\n", + "DEBUG - cdap: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - cdap: write_omnibus_choosers: output/estimation_data_bundle/cdap/cdap_values_combined.csv\r\n", + "INFO - cdap: end estimation\r\n", + "INFO - cdap_activity top 10 value counts:\r\n", + "M 2620\r\n", + "N 1213\r\n", + "H 572\r\n", + "Name: cdap_activity, dtype: int64\r\n", + "INFO - cdap crosstabs:\r\n", + "cdap_activity H M N All\r\n", + "ptype \r\n", + "1 156 1606 158 1920\r\n", + "2 47 331 118 496\r\n", + "3 33 230 66 329\r\n", + "4 114 0 420 534\r\n", + "5 114 0 357 471\r\n", + "6 22 42 8 72\r\n", + "7 39 274 39 352\r\n", + "8 47 137 47 231\r\n", + "All 572 2620 1213 4405\r\n", + "Time to execute step 'cdap_simulate': 9.84 s\r\n", + "Total time to execute iteration 1 with iteration value None: 9.84 s\r\n", + "INFO - #run_model running step mandatory_tour_frequency\r\n", + "Running step 'mandatory_tour_frequency'\r\n", + "INFO - Running mandatory_tour_frequency with 2620 persons\r\n", + "INFO - Initialize Estimator for'mandatory_tour_frequency'\r\n", + "DEBUG - mandatory_tour_frequency: estimate.write_spec: output/estimation_data_bundle/mandatory_tour_frequency/mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - mandatory_tour_frequency: write_table write: mandatory_tour_frequency_coefficients.csv\r\n", + "DEBUG - mandatory_tour_frequency: write_table cache: choosers\r\n", + "INFO - mandatory_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 2620 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2620 of 2620 choosers\r\n", + "DEBUG - mandatory_tour_frequency: write_table cache: expression_values\r\n", + "DEBUG - mandatory_tour_frequency: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - mandatory_tour_frequency: write_table cache: override_choices\r\n", + "DEBUG - mandatory_tour_frequency: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - mandatory_tour_frequency: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_frequency/mandatory_tour_frequency_values_combined.csv\r\n", + "INFO - mandatory_tour_frequency: end estimation\r\n", + "INFO - mandatory_tour_frequency top 10 value counts:\r\n", + "work1 1941\r\n", + " 1785\r\n", + "school1 566\r\n", + "work2 81\r\n", + "work_and_school 21\r\n", + "school2 11\r\n", + "Name: mandatory_tour_frequency, dtype: int64\r\n", + "Time to execute step 'mandatory_tour_frequency': 1.21 s\r\n", + "Total time to execute iteration 1 with iteration value None: 1.21 s\r\n", + "INFO - #run_model running step mandatory_tour_scheduling\r\n", + "Running step 'mandatory_tour_scheduling'\r\n", + "DEBUG - @inject timetable\r\n", + "INFO - Initialize Estimator for'mandatory_tour_scheduling_work'\r\n", + "DEBUG - mandatory_tour_scheduling_work: estimate.write_spec: output/estimation_data_bundle/mandatory_tour_scheduling_work/mandatory_tour_scheduling_work_SPEC.csv\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table write: tour_scheduling_work_coefficients.csv\r\n", + "INFO - Initialize Estimator for'mandatory_tour_scheduling_school'\r\n", + "DEBUG - mandatory_tour_scheduling_school: estimate.write_spec: output/estimation_data_bundle/mandatory_tour_scheduling_school/mandatory_tour_scheduling_school_SPEC.csv\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table write: tour_scheduling_school_coefficients.csv\r\n", + "INFO - mandatory_tour_scheduling_work: timetable.begin_transaction person_windows\r\n", + "INFO - mandatory_tour_scheduling_school: timetable.begin_transaction person_windows\r\n", + "INFO - Running mandatory_tour_scheduling with 2733 tours\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 2043 tour choices\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work Running adaptive_chunked_choosers with 2043 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2043 of 2043 choosers\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 2043 tour choices\r\n", + "INFO - tdd_alt_segments specified for representative logsums\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 388170 to 30645 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums compute_logsums for 30645 choosers 30645 alts\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums Running adaptive_chunked_choosers with 30645 choosers\r\n", + "INFO - Running chunk 1 of 1 with 30645 of 30645 choosers\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: choosers\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_sample_alternatives\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 2043 choosers and 388170 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 2043 of 2043 choosers\r\n", + "INFO - Running eval_interaction_utilities on 388170 rows\r\n", + "INFO - mandatory_tour_scheduling_work: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_expression_values\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 451 tour choices\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school Running adaptive_chunked_choosers with 451 choosers\r\n", + "INFO - Running chunk 1 of 1 with 451 of 451 choosers\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 451 tour choices\r\n", + "INFO - tdd_alt_segments specified for representative logsums\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 85690 to 6765 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums compute_logsums for 6765 choosers 6765 alts\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums Running adaptive_chunked_choosers with 6765 choosers\r\n", + "INFO - Running chunk 1 of 1 with 6765 of 6765 choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 451 choosers and 85690 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 451 of 451 choosers\r\n", + "INFO - Running eval_interaction_utilities on 85690 rows\r\n", + "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 126 tour choices\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ Running adaptive_chunked_choosers with 126 choosers\r\n", + "INFO - Running chunk 1 of 1 with 126 of 126 choosers\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 126 tour choices\r\n", + "INFO - tdd_alt_segments specified for representative logsums\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 23940 to 1890 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums compute_logsums for 1890 choosers 1890 alts\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums Running adaptive_chunked_choosers with 1890 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1890 of 1890 choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 126 choosers and 23940 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 126 of 126 choosers\r\n", + "INFO - Running eval_interaction_utilities on 23940 rows\r\n", + "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work schedule_tours running 81 tour choices\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work Running adaptive_chunked_choosers with 81 choosers\r\n", + "INFO - Running chunk 1 of 1 with 81 of 81 choosers\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work schedule_tours running 81 tour choices\r\n", + "INFO - tdd_alt_segments specified for representative logsums\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 89.13% from 6074 to 660 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.compute_logsums.logsums compute_logsums for 660 choosers 660 alts\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.compute_logsums.logsums Running adaptive_chunked_choosers with 660 choosers\r\n", + "INFO - Running chunk 1 of 1 with 660 of 660 choosers\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: choosers\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_sample_alternatives\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 81 choosers and 6074 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 81 of 81 choosers\r\n", + "INFO - Running eval_interaction_utilities on 6074 rows\r\n", + "INFO - mandatory_tour_scheduling_work: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_expression_values\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school schedule_tours running 9 tour choices\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school Running adaptive_chunked_choosers with 9 choosers\r\n", + "INFO - Running chunk 1 of 1 with 9 of 9 choosers\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school schedule_tours running 9 tour choices\r\n", + "INFO - tdd_alt_segments specified for representative logsums\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 89.84% from 817 to 83 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.compute_logsums.logsums compute_logsums for 83 choosers 83 alts\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.compute_logsums.logsums Running adaptive_chunked_choosers with 83 choosers\r\n", + "INFO - Running chunk 1 of 1 with 83 of 83 choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 9 choosers and 817 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 9 of 9 choosers\r\n", + "INFO - Running eval_interaction_utilities on 817 rows\r\n", + "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ schedule_tours running 23 tour choices\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ Running adaptive_chunked_choosers with 23 choosers\r\n", + "INFO - Running chunk 1 of 1 with 23 of 23 choosers\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ schedule_tours running 23 tour choices\r\n", + "INFO - tdd_alt_segments specified for representative logsums\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 90.26% from 2176 to 212 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.compute_logsums.logsums compute_logsums for 212 choosers 212 alts\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.compute_logsums.logsums Running adaptive_chunked_choosers with 212 choosers\r\n", + "INFO - Running chunk 1 of 1 with 212 of 212 choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 23 choosers and 2176 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 23 of 23 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2176 rows\r\n", + "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_table cache: override_choices\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_work/mandatory_tour_scheduling_work_choosers_combined.csv\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - mandatory_tour_scheduling_work: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_work/mandatory_tour_scheduling_work_alternatives_combined.csv\r\n", + "INFO - mandatory_tour_scheduling_work: end estimation\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_table cache: override_choices\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_school/mandatory_tour_scheduling_school_choosers_combined.csv\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - mandatory_tour_scheduling_school: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_school/mandatory_tour_scheduling_school_alternatives_combined.csv\r\n", + "INFO - mandatory_tour_scheduling_school: end estimation\r\n", + "INFO - mandatory_tour_scheduling_work: timetable.rollback person_windows\r\n", + "INFO - mandatory_tour_scheduling_school: timetable.rollback person_windows\r\n", + "Time to execute step 'mandatory_tour_scheduling': 84.87 s\r\n", + "Total time to execute iteration 1 with iteration value None: 84.87 s\r\n", + "INFO - #run_model running step joint_tour_frequency\r\n", + "Running step 'joint_tour_frequency'\r\n", + "INFO - Initialize Estimator for'joint_tour_frequency'\r\n", + "INFO - Running joint_tour_frequency with 1028 multi-person households\r\n", + "DEBUG - @inject timetable\r\n", + "DEBUG - joint_tour_frequency: estimate.write_spec: output/estimation_data_bundle/joint_tour_frequency/joint_tour_frequency_SPEC.csv\r\n", + "DEBUG - joint_tour_frequency: write_table write: joint_tour_frequency_coefficients.csv\r\n", + "DEBUG - joint_tour_frequency: write_table cache: choosers\r\n", + "INFO - joint_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 1028 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1028 of 1028 choosers\r\n", + "DEBUG - joint_tour_frequency: write_table cache: expression_values\r\n", + "DEBUG - joint_tour_frequency: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using households.index\r\n", + "DEBUG - joint_tour_frequency: write_table cache: override_choices\r\n", + "DEBUG - joint_tour_frequency: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - joint_tour_frequency: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_frequency/joint_tour_frequency_values_combined.csv\r\n", + "INFO - joint_tour_frequency: end estimation\r\n", + "INFO - joint_tour_frequency top 10 value counts:\r\n", + "0_tours 1910\r\n", + "1_Main 26\r\n", + "1_Disc 20\r\n", + "1_Shop 16\r\n", + "1_Eat 15\r\n", + "1_Visit 12\r\n", + "2_SS 1\r\n", + "Name: joint_tour_frequency, dtype: int64\r\n", + "len(survey_tours) 91\r\n", + "len(joint_tours) 91\r\n", + "Time to execute step 'joint_tour_frequency': 0.68 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.68 s\r\n", + "INFO - #run_model running step joint_tour_composition\r\n", + "Running step 'joint_tour_composition'\r\n", + "INFO - Initialize Estimator for'joint_tour_composition'\r\n", + "INFO - Running joint_tour_composition with 91 joint tours\r\n", + "DEBUG - @inject timetable\r\n", + "DEBUG - joint_tour_composition: estimate.write_spec: output/estimation_data_bundle/joint_tour_composition/joint_tour_composition_SPEC.csv\r\n", + "DEBUG - joint_tour_composition: write_table write: joint_tour_composition_coefficients.csv\r\n", + "DEBUG - joint_tour_composition: write_table cache: choosers\r\n", + "INFO - joint_tour_composition.simple_simulate Running adaptive_chunked_choosers with 91 choosers\r\n", + "INFO - Running chunk 1 of 1 with 91 of 91 choosers\r\n", + "DEBUG - joint_tour_composition: write_table cache: expression_values\r\n", + "DEBUG - joint_tour_composition: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - joint_tour_composition: write_table cache: override_choices\r\n", + "DEBUG - joint_tour_composition: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - joint_tour_composition: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_composition/joint_tour_composition_values_combined.csv\r\n", + "INFO - joint_tour_composition: end estimation\r\n", + "INFO - joint_tour_composition top 10 value counts:\r\n", + "adults 54\r\n", + "mixed 32\r\n", + "children 5\r\n", + "Name: composition, dtype: int64\r\n", + "Time to execute step 'joint_tour_composition': 0.31 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.31 s\r\n", + "INFO - #run_model running step joint_tour_participation\r\n", + "Running step 'joint_tour_participation'\r\n", + "INFO - Running joint_tours_participation with 304 potential participants (candidates)\r\n", + "DEBUG - @inject timetable\r\n", + "INFO - Initialize Estimator for'joint_tour_participation'\r\n", + "DEBUG - joint_tour_participation: estimate.write_spec: output/estimation_data_bundle/joint_tour_participation/joint_tour_participation_SPEC.csv\r\n", + "DEBUG - joint_tour_participation: write_table write: joint_tour_participation_coefficients.csv\r\n", + "DEBUG - joint_tour_participation: write_table cache: choosers\r\n", + "INFO - joint_tour_participation Running chunk 1 of 1 with 90 of 90 choosers\r\n", + "DEBUG - joint_tour_participation: write_table cache: expression_values\r\n", + "INFO - joint_tour_participation.eval_mnl.participants_chooser 91 joint tours to satisfy.\r\n", + "INFO - joint_tour_participation.eval_mnl.participants_chooser 8 iterations to satisfy all joint tours.\r\n", + "DEBUG - joint_tour_participation: write_table cache: choices\r\n", + "DEBUG - joint_tour_participation: write_table cache: override_choices\r\n", + "DEBUG - joint_tour_participation: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - joint_tour_participation: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_participation/joint_tour_participation_values_combined.csv\r\n", + "INFO - joint_tour_participation: end estimation\r\n", + "Time to execute step 'joint_tour_participation': 0.99 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.99 s\r\n", + "INFO - #run_model running step joint_tour_destination\r\n", + "Running step 'joint_tour_destination'\r\n", + "INFO - Initialize Estimator for'joint_tour_destination'\r\n", + "DEBUG - joint_tour_destination: write_table write: non_mandatory_tour_destination_coefficients.csv\r\n", + "DEBUG - joint_tour_destination: estimate.write_spec: output/estimation_data_bundle/joint_tour_destination/joint_tour_destination_SPEC.csv\r\n", + "DEBUG - joint_tour_destination: write_table write: size_terms\r\n", + "DEBUG - joint_tour_destination: write_table write: landuse\r\n", + "INFO - running joint_tour_destination.shopping.sample with 18 tours\r\n", + "INFO - Estimation mode for joint_tour_destination.shopping.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - joint_tour_destination.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 18 choosers\r\n", + "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3420 rows\r\n", + "INFO - Estimation mode for joint_tour_destination.shopping.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running joint_tour_destination.shopping.logsums with 3420 rows\r\n", + "INFO - joint_tour_destination.shopping.logsums.compute_logsums Running adaptive_chunked_choosers with 3420 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3420 of 3420 choosers\r\n", + "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 18 persons\r\n", + "INFO - joint_tour_destination.shopping.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 18 choosers and 3420 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3420 rows\r\n", + "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.shopping.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running joint_tour_destination.othmaint.sample with 26 tours\r\n", + "INFO - Estimation mode for joint_tour_destination.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - joint_tour_destination.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 26 choosers\r\n", + "INFO - Running chunk 1 of 1 with 26 of 26 choosers\r\n", + "INFO - Running eval_interaction_utilities on 4940 rows\r\n", + "INFO - Estimation mode for joint_tour_destination.othmaint.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running joint_tour_destination.othmaint.logsums with 4940 rows\r\n", + "INFO - joint_tour_destination.othmaint.logsums.compute_logsums Running adaptive_chunked_choosers with 4940 choosers\r\n", + "INFO - Running chunk 1 of 1 with 4940 of 4940 choosers\r\n", + "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 26 persons\r\n", + "INFO - joint_tour_destination.othmaint.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 26 choosers and 4940 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 26 of 26 choosers\r\n", + "INFO - Running eval_interaction_utilities on 4940 rows\r\n", + "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.othmaint.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running joint_tour_destination.othdiscr.sample with 20 tours\r\n", + "INFO - Estimation mode for joint_tour_destination.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - joint_tour_destination.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 20 choosers\r\n", + "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3800 rows\r\n", + "INFO - Estimation mode for joint_tour_destination.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running joint_tour_destination.othdiscr.logsums with 3800 rows\r\n", + "INFO - joint_tour_destination.othdiscr.logsums.compute_logsums Running adaptive_chunked_choosers with 3800 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3800 of 3800 choosers\r\n", + "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 20 persons\r\n", + "INFO - joint_tour_destination.othdiscr.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 20 choosers and 3800 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3800 rows\r\n", + "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.othdiscr.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running joint_tour_destination.eatout.sample with 15 tours\r\n", + "INFO - Estimation mode for joint_tour_destination.eatout.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - joint_tour_destination.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 15 choosers\r\n", + "INFO - Running chunk 1 of 1 with 15 of 15 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2850 rows\r\n", + "INFO - Estimation mode for joint_tour_destination.eatout.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running joint_tour_destination.eatout.logsums with 2850 rows\r\n", + "INFO - joint_tour_destination.eatout.logsums.compute_logsums Running adaptive_chunked_choosers with 2850 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2850 of 2850 choosers\r\n", + "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 15 persons\r\n", + "INFO - joint_tour_destination.eatout.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 15 choosers and 2850 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 15 of 15 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2850 rows\r\n", + "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.eatout.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running joint_tour_destination.social.sample with 12 tours\r\n", + "INFO - Estimation mode for joint_tour_destination.social.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - joint_tour_destination.social.sample.interaction_sample Running adaptive_chunked_choosers with 12 choosers\r\n", + "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2280 rows\r\n", + "INFO - Estimation mode for joint_tour_destination.social.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running joint_tour_destination.social.logsums with 2280 rows\r\n", + "INFO - joint_tour_destination.social.logsums.compute_logsums Running adaptive_chunked_choosers with 2280 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2280 of 2280 choosers\r\n", + "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 12 persons\r\n", + "INFO - joint_tour_destination.social.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 12 choosers and 2280 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2280 rows\r\n", + "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.social.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - joint_tour_destination skipping segment escort: no choosers\r\n", + "DEBUG - joint_tour_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - joint_tour_destination: write_table cache: override_choices\r\n", + "DEBUG - joint_tour_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - joint_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_destination/joint_tour_destination_choosers_combined.csv\r\n", + "DEBUG - joint_tour_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "WARNING - joint_tour_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", + "DEBUG - joint_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_destination/joint_tour_destination_alternatives_combined.csv\r\n", + "INFO - joint_tour_destination: end estimation\r\n", + "INFO - destination summary:\r\n", + "count 91.000000\r\n", + "mean 101.439560\r\n", + "std 56.893899\r\n", + "min 4.000000\r\n", + "25% 45.000000\r\n", + "50% 114.000000\r\n", + "75% 149.000000\r\n", + "max 190.000000\r\n", + "Name: destination, dtype: float64\r\n", + "Time to execute step 'joint_tour_destination': 7.54 s\r\n", + "Total time to execute iteration 1 with iteration value None: 7.54 s\r\n", + "INFO - #run_model running step joint_tour_scheduling\r\n", + "Running step 'joint_tour_scheduling'\r\n", + "INFO - Running joint_tour_scheduling with 91 joint tours\r\n", + "DEBUG - @inject timetable\r\n", + "INFO - Initialize Estimator for'joint_tour_scheduling'\r\n", + "DEBUG - joint_tour_scheduling: estimate.write_spec: output/estimation_data_bundle/joint_tour_scheduling/joint_tour_scheduling_SPEC.csv\r\n", + "DEBUG - joint_tour_scheduling: write_table write: tour_scheduling_joint_coefficients.csv\r\n", + "INFO - joint_tour_scheduling: timetable.begin_transaction person_windows\r\n", + "INFO - schedule_tours %s tours not monotonic_increasing - sorting df\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1 schedule_tours running 90 tour choices\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1 Running adaptive_chunked_choosers with 90 choosers\r\n", + "INFO - Running chunk 1 of 1 with 90 of 90 choosers\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1 schedule_tours running 90 tour choices\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 90 choosers and 6458 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 90 of 90 choosers\r\n", + "INFO - Running eval_interaction_utilities on 6458 rows\r\n", + "INFO - joint_tour_scheduling: eval_interaction_utilities write_interaction_expression_values joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2 schedule_tours running 1 tour choices\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2 Running adaptive_chunked_choosers with 1 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1 of 1 choosers\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2 schedule_tours running 1 tour choices\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 1 choosers and 73 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 1 of 1 choosers\r\n", + "INFO - Running eval_interaction_utilities on 73 rows\r\n", + "INFO - joint_tour_scheduling: eval_interaction_utilities write_interaction_expression_values joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - joint_tour_scheduling: write_table cache: override_choices\r\n", + "DEBUG - joint_tour_scheduling: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - joint_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_scheduling/joint_tour_scheduling_choosers_combined.csv\r\n", + "DEBUG - joint_tour_scheduling: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - joint_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_scheduling/joint_tour_scheduling_alternatives_combined.csv\r\n", + "INFO - joint_tour_scheduling: end estimation\r\n", + "INFO - joint_tour_scheduling: timetable.rollback person_windows\r\n", + "INFO - joint_tour_scheduling: assign timetable for 224 participants in 90 tours with tour_num 1\r\n", + "INFO - joint_tour_scheduling: assign timetable for 4 participants in 1 tours with tour_num 2\r\n", + "Time to execute step 'joint_tour_scheduling': 1.89 s\r\n", + "Total time to execute iteration 1 with iteration value None: 1.89 s\r\n", + "INFO - #run_model running step non_mandatory_tour_frequency\r\n", + "Running step 'non_mandatory_tour_frequency'\r\n", + "DEBUG - @inject timetable\r\n", + "INFO - Running non_mandatory_tour_frequency with 3833 persons\r\n", + "INFO - Running segment 'PTYPE_FULL' of size 1764\r\n", + "INFO - Initialize Estimator for'PTYPE_FULL'\r\n", + "DEBUG - PTYPE_FULL: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_FULL: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_FULL.csv\r\n", + "DEBUG - PTYPE_FULL: write_table cache: choosers\r\n", + "DEBUG - PTYPE_FULL: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_FULL.interaction_simulate Running adaptive_chunked_choosers with 1764 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1764 of 1764 choosers\r\n", + "WARNING - /mnt/c/Users/jan.zill/code/activitysim/activitysim/core/logit.py:358: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", + " alts_sample[c_chooser] = np.repeat(choosers[c].values, sample_size)\r\n", + "\r\n", + "INFO - Running eval_interaction_utilities on 169344 rows\r\n", + "WARNING - /mnt/c/Users/jan.zill/code/activitysim/activitysim/core/interaction_simulate.py:482: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", + " choices = _interaction_simulate(chooser_chunk, alternatives, spec,\r\n", + "\r\n", + "INFO - PTYPE_FULL: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_FULL.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_FULL: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_FULL: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_FULL: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_FULL: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_FULL: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_FULL/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_FULL: end estimation\r\n", + "INFO - Running segment 'PTYPE_PART' of size 449\r\n", + "INFO - Initialize Estimator for'PTYPE_PART'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_PART: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_PART: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_PART.csv\r\n", + "DEBUG - PTYPE_PART: write_table cache: choosers\r\n", + "DEBUG - PTYPE_PART: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_PART.interaction_simulate Running adaptive_chunked_choosers with 449 choosers\r\n", + "INFO - Running chunk 1 of 1 with 449 of 449 choosers\r\n", + "INFO - Running eval_interaction_utilities on 43104 rows\r\n", + "INFO - PTYPE_PART: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_PART.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_PART: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_PART: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_PART: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_PART: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_PART: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_PART/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_PART: end estimation\r\n", + "INFO - Running segment 'PTYPE_UNIVERSITY' of size 296\r\n", + "INFO - Initialize Estimator for'PTYPE_UNIVERSITY'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_UNIVERSITY: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_UNIVERSITY.csv\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_table cache: choosers\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_UNIVERSITY.interaction_simulate Running adaptive_chunked_choosers with 296 choosers\r\n", + "INFO - Running chunk 1 of 1 with 296 of 296 choosers\r\n", + "INFO - Running eval_interaction_utilities on 28416 rows\r\n", + "INFO - PTYPE_UNIVERSITY: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_UNIVERSITY.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_UNIVERSITY: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_UNIVERSITY/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_UNIVERSITY: end estimation\r\n", + "INFO - Running segment 'PTYPE_NONWORK' of size 420\r\n", + "INFO - Initialize Estimator for'PTYPE_NONWORK'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_NONWORK: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_NONWORK: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_NONWORK.csv\r\n", + "DEBUG - PTYPE_NONWORK: write_table cache: choosers\r\n", + "DEBUG - PTYPE_NONWORK: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_NONWORK.interaction_simulate Running adaptive_chunked_choosers with 420 choosers\r\n", + "INFO - Running chunk 1 of 1 with 420 of 420 choosers\r\n", + "INFO - Running eval_interaction_utilities on 40320 rows\r\n", + "INFO - PTYPE_NONWORK: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_NONWORK.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_NONWORK: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_NONWORK: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_NONWORK: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_NONWORK: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_NONWORK: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_NONWORK/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_NONWORK: end estimation\r\n", + "INFO - Running segment 'PTYPE_RETIRED' of size 357\r\n", + "INFO - Initialize Estimator for'PTYPE_RETIRED'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_RETIRED: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_RETIRED: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_RETIRED.csv\r\n", + "DEBUG - PTYPE_RETIRED: write_table cache: choosers\r\n", + "DEBUG - PTYPE_RETIRED: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_RETIRED.interaction_simulate Running adaptive_chunked_choosers with 357 choosers\r\n", + "INFO - Running chunk 1 of 1 with 357 of 357 choosers\r\n", + "INFO - Running eval_interaction_utilities on 34272 rows\r\n", + "INFO - PTYPE_RETIRED: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_RETIRED.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_RETIRED: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_RETIRED: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_RETIRED: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_RETIRED: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_RETIRED: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_RETIRED/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_RETIRED: end estimation\r\n", + "INFO - Running segment 'PTYPE_DRIVING' of size 50\r\n", + "INFO - Initialize Estimator for'PTYPE_DRIVING'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_DRIVING: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_DRIVING: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_DRIVING.csv\r\n", + "DEBUG - PTYPE_DRIVING: write_table cache: choosers\r\n", + "DEBUG - PTYPE_DRIVING: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_DRIVING.interaction_simulate Running adaptive_chunked_choosers with 50 choosers\r\n", + "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", + "INFO - Running eval_interaction_utilities on 4800 rows\r\n", + "INFO - PTYPE_DRIVING: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_DRIVING.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_DRIVING: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_DRIVING: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_DRIVING: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_DRIVING: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_DRIVING: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_DRIVING/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_DRIVING: end estimation\r\n", + "INFO - Running segment 'PTYPE_SCHOOL' of size 313\r\n", + "INFO - Initialize Estimator for'PTYPE_SCHOOL'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_SCHOOL: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_SCHOOL: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_SCHOOL.csv\r\n", + "DEBUG - PTYPE_SCHOOL: write_table cache: choosers\r\n", + "DEBUG - PTYPE_SCHOOL: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_SCHOOL.interaction_simulate Running adaptive_chunked_choosers with 313 choosers\r\n", + "INFO - Running chunk 1 of 1 with 313 of 313 choosers\r\n", + "INFO - Running eval_interaction_utilities on 30048 rows\r\n", + "INFO - PTYPE_SCHOOL: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_SCHOOL.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_SCHOOL: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_SCHOOL: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_SCHOOL: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_SCHOOL: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_SCHOOL: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_SCHOOL/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_SCHOOL: end estimation\r\n", + "INFO - Running segment 'PTYPE_PRESCHOOL' of size 184\r\n", + "INFO - Initialize Estimator for'PTYPE_PRESCHOOL'\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", + "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_PRESCHOOL: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_PRESCHOOL.csv\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_table cache: choosers\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_table write: alternatives\r\n", + "INFO - non_mandatory_tour_frequency.PTYPE_PRESCHOOL.interaction_simulate Running adaptive_chunked_choosers with 184 choosers\r\n", + "INFO - Running chunk 1 of 1 with 184 of 184 choosers\r\n", + "INFO - Running eval_interaction_utilities on 17664 rows\r\n", + "INFO - PTYPE_PRESCHOOL: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_PRESCHOOL.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_table write: interaction_expression_values\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_table cache: override_choices\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - PTYPE_PRESCHOOL: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_PRESCHOOL/non_mandatory_tour_frequency_choosers_combined.csv\r\n", + "INFO - PTYPE_PRESCHOOL: end estimation\r\n", + "INFO - extend_tour_counts increased tour count by 166 from 2324 to 2490\r\n", + "DEBUG - get_survey_values: reindexing using persons.index\r\n", + "INFO - estimation get_survey_values override_tour_counts 0 changed cells\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "INFO - non_mandatory_tour_frequency top 10 value counts:\r\n", + "0 2718\r\n", + "16 374\r\n", + "1 282\r\n", + "8 174\r\n", + "4 154\r\n", + "32 132\r\n", + "2 84\r\n", + "17 61\r\n", + "24 50\r\n", + "9 32\r\n", + "Name: non_mandatory_tour_frequency, dtype: int64\r\n", + "Time to execute step 'non_mandatory_tour_frequency': 188.21 s\r\n", + "Total time to execute iteration 1 with iteration value None: 188.21 s\r\n", + "INFO - #run_model running step non_mandatory_tour_destination\r\n", + "Running step 'non_mandatory_tour_destination'\r\n", + "INFO - Initialize Estimator for'non_mandatory_tour_destination'\r\n", + "DEBUG - non_mandatory_tour_destination: write_table write: non_mandatory_tour_destination_coefficients.csv\r\n", + "DEBUG - non_mandatory_tour_destination: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_destination/non_mandatory_tour_destination_SPEC.csv\r\n", + "DEBUG - non_mandatory_tour_destination: write_table write: size_terms\r\n", + "DEBUG - non_mandatory_tour_destination: write_table write: landuse\r\n", + "INFO - running non_mandatory_tour_destination.shopping.sample with 705 tours\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.shopping.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - non_mandatory_tour_destination.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 705 choosers\r\n", + "INFO - Running chunk 1 of 1 with 705 of 705 choosers\r\n", + "INFO - Running eval_interaction_utilities on 133950 rows\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.shopping.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running non_mandatory_tour_destination.shopping.logsums with 133950 rows\r\n", + "INFO - non_mandatory_tour_destination.shopping.logsums.compute_logsums Running adaptive_chunked_choosers with 133950 choosers\r\n", + "INFO - Running chunk 1 of 1 with 133950 of 133950 choosers\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 705 persons\r\n", + "INFO - non_mandatory_tour_destination.shopping.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 705 choosers and 133950 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 705 of 705 choosers\r\n", + "INFO - Running eval_interaction_utilities on 133950 rows\r\n", + "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.shopping.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running non_mandatory_tour_destination.othmaint.sample with 367 tours\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - non_mandatory_tour_destination.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 367 choosers\r\n", + "INFO - Running chunk 1 of 1 with 367 of 367 choosers\r\n", + "INFO - Running eval_interaction_utilities on 69730 rows\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.othmaint.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running non_mandatory_tour_destination.othmaint.logsums with 69730 rows\r\n", + "INFO - non_mandatory_tour_destination.othmaint.logsums.compute_logsums Running adaptive_chunked_choosers with 69730 choosers\r\n", + "INFO - Running chunk 1 of 1 with 69730 of 69730 choosers\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 367 persons\r\n", + "INFO - non_mandatory_tour_destination.othmaint.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 367 choosers and 69730 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 367 of 367 choosers\r\n", + "INFO - Running eval_interaction_utilities on 69730 rows\r\n", + "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.othmaint.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running non_mandatory_tour_destination.othdiscr.sample with 544 tours\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - non_mandatory_tour_destination.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 544 choosers\r\n", + "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", + "INFO - Running eval_interaction_utilities on 103360 rows\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running non_mandatory_tour_destination.othdiscr.logsums with 103360 rows\r\n", + "INFO - non_mandatory_tour_destination.othdiscr.logsums.compute_logsums Running adaptive_chunked_choosers with 103360 choosers\r\n", + "INFO - Running chunk 1 of 1 with 103360 of 103360 choosers\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 544 persons\r\n", + "INFO - non_mandatory_tour_destination.othdiscr.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 544 choosers and 103360 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", + "INFO - Running eval_interaction_utilities on 103360 rows\r\n", + "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.othdiscr.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running non_mandatory_tour_destination.eatout.sample with 301 tours\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.eatout.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - non_mandatory_tour_destination.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 301 choosers\r\n", + "INFO - Running chunk 1 of 1 with 301 of 301 choosers\r\n", + "INFO - Running eval_interaction_utilities on 57190 rows\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.eatout.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running non_mandatory_tour_destination.eatout.logsums with 57190 rows\r\n", + "INFO - non_mandatory_tour_destination.eatout.logsums.compute_logsums Running adaptive_chunked_choosers with 57190 choosers\r\n", + "INFO - Running chunk 1 of 1 with 57190 of 57190 choosers\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 301 persons\r\n", + "INFO - non_mandatory_tour_destination.eatout.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 301 choosers and 57190 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 301 of 301 choosers\r\n", + "INFO - Running eval_interaction_utilities on 57190 rows\r\n", + "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.eatout.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running non_mandatory_tour_destination.social.sample with 170 tours\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.social.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - non_mandatory_tour_destination.social.sample.interaction_sample Running adaptive_chunked_choosers with 170 choosers\r\n", + "INFO - Running chunk 1 of 1 with 170 of 170 choosers\r\n", + "INFO - Running eval_interaction_utilities on 32300 rows\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.social.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running non_mandatory_tour_destination.social.logsums with 32300 rows\r\n", + "INFO - non_mandatory_tour_destination.social.logsums.compute_logsums Running adaptive_chunked_choosers with 32300 choosers\r\n", + "INFO - Running chunk 1 of 1 with 32300 of 32300 choosers\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 170 persons\r\n", + "INFO - non_mandatory_tour_destination.social.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 170 choosers and 32300 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 170 of 170 choosers\r\n", + "INFO - Running eval_interaction_utilities on 32300 rows\r\n", + "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.social.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", + "INFO - running non_mandatory_tour_destination.escort.sample with 403 tours\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.escort.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - non_mandatory_tour_destination.escort.sample.interaction_sample Running adaptive_chunked_choosers with 403 choosers\r\n", + "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", + "INFO - Running eval_interaction_utilities on 76570 rows\r\n", + "INFO - Estimation mode for non_mandatory_tour_destination.escort.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running non_mandatory_tour_destination.escort.logsums with 76570 rows\r\n", + "INFO - non_mandatory_tour_destination.escort.logsums.compute_logsums Running adaptive_chunked_choosers with 76570 choosers\r\n", + "INFO - Running chunk 1 of 1 with 76570 of 76570 choosers\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 403 persons\r\n", + "INFO - non_mandatory_tour_destination.escort.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 403 choosers and 76570 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", + "INFO - Running eval_interaction_utilities on 76570 rows\r\n", + "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.escort.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - non_mandatory_tour_destination: write_table cache: override_choices\r\n", + "DEBUG - non_mandatory_tour_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - non_mandatory_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_destination/non_mandatory_tour_destination_choosers_combined.csv\r\n", + "DEBUG - non_mandatory_tour_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "WARNING - non_mandatory_tour_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", + "DEBUG - non_mandatory_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_destination/non_mandatory_tour_destination_alternatives_combined.csv\r\n", + "INFO - non_mandatory_tour_destination: end estimation\r\n", + "Time to execute step 'non_mandatory_tour_destination': 38.82 s\r\n", + "Total time to execute iteration 1 with iteration value None: 38.82 s\r\n", + "INFO - #run_model running step non_mandatory_tour_scheduling\r\n", + "Running step 'non_mandatory_tour_scheduling'\r\n", + "DEBUG - @inject timetable\r\n", + "INFO - Initialize Estimator for'non_mandatory_tour_scheduling'\r\n", + "DEBUG - non_mandatory_tour_scheduling: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_scheduling/non_mandatory_tour_scheduling_SPEC.csv\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table write: tour_scheduling_nonmandatory_coefficients.csv\r\n", + "INFO - non_mandatory_tour_scheduling: timetable.begin_transaction person_windows\r\n", + "INFO - Running non_mandatory_tour_scheduling with 2490 tours\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1 schedule_tours running 1687 tour choices\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1 Running adaptive_chunked_choosers with 1687 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1687 of 1687 choosers\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1 schedule_tours running 1687 tour choices\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 1687 choosers and 241779 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 1687 of 1687 choosers\r\n", + "INFO - Running eval_interaction_utilities on 241779 rows\r\n", + "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2 schedule_tours running 571 tour choices\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2 Running adaptive_chunked_choosers with 571 choosers\r\n", + "INFO - Running chunk 1 of 1 with 571 of 571 choosers\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2 schedule_tours running 571 tour choices\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 571 choosers and 47267 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 571 of 571 choosers\r\n", + "INFO - Running eval_interaction_utilities on 47267 rows\r\n", + "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3 schedule_tours running 179 tour choices\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3 Running adaptive_chunked_choosers with 179 choosers\r\n", + "INFO - Running chunk 1 of 1 with 179 of 179 choosers\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3 schedule_tours running 179 tour choices\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 179 choosers and 12032 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 179 of 179 choosers\r\n", + "INFO - Running eval_interaction_utilities on 12032 rows\r\n", + "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4 schedule_tours running 50 tour choices\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4 Running adaptive_chunked_choosers with 50 choosers\r\n", + "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4 schedule_tours running 50 tour choices\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 50 choosers and 2697 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2697 rows\r\n", + "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5 schedule_tours running 3 tour choices\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5 Running adaptive_chunked_choosers with 3 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5 schedule_tours running 3 tour choices\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 3 choosers and 146 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", + "INFO - Running eval_interaction_utilities on 146 rows\r\n", + "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_table cache: override_choices\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_scheduling/non_mandatory_tour_scheduling_choosers_combined.csv\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - non_mandatory_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_scheduling/non_mandatory_tour_scheduling_alternatives_combined.csv\r\n", + "INFO - non_mandatory_tour_scheduling: end estimation\r\n", + "INFO - non_mandatory_tour_scheduling: timetable.rollback person_windows\r\n", + "Time to execute step 'non_mandatory_tour_scheduling': 75.98 s\r\n", + "Total time to execute iteration 1 with iteration value None: 75.98 s\r\n", + "INFO - #run_model running step tour_mode_choice_simulate\r\n", + "Running step 'tour_mode_choice_simulate'\r\n", + "INFO - Running tour_mode_choice with 5314 tours\r\n", + "INFO - tour_types top 10 value counts:\r\n", + "work 2124\r\n", + "shopping 723\r\n", + "school 609\r\n", + "othdiscr 564\r\n", + "escort 403\r\n", + "othmaint 393\r\n", + "eatout 316\r\n", + "social 182\r\n", + "Name: tour_type, dtype: int64\r\n", + "INFO - Initialize Estimator for'tour_mode_choice'\r\n", + "DEBUG - tour_mode_choice: write_table write: tour_mode_choice_coefficients.csv\r\n", + "DEBUG - tour_mode_choice: write_table write: coefficients_template\r\n", + "DEBUG - tour_mode_choice: estimate.write_spec: output/estimation_data_bundle/tour_mode_choice/tour_mode_choice_SPEC.csv\r\n", + "INFO - tour_mode_choice_simulate tour_type 'eatout' (316 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.eatout.simple_simulate Running adaptive_chunked_choosers with 316 choosers\r\n", + "INFO - Running chunk 1 of 1 with 316 of 316 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate eatout choices_df top 10 value counts:\r\n", + "DRIVEALONEFREE 77\r\n", + "WALK 73\r\n", + "SHARED3FREE 53\r\n", + "SHARED2FREE 53\r\n", + "WALK_LOC 24\r\n", + "WALK_LRF 13\r\n", + "WALK_HVY 7\r\n", + "BIKE 6\r\n", + "TNC_SINGLE 5\r\n", + "DRIVE_LOC 3\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'escort' (403 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.escort.simple_simulate Running adaptive_chunked_choosers with 403 choosers\r\n", + "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate escort choices_df top 10 value counts:\r\n", + "SHARED2FREE 175\r\n", + "SHARED3FREE 151\r\n", + "TNC_SINGLE 45\r\n", + "TNC_SHARED 15\r\n", + "TAXI 6\r\n", + "BIKE 4\r\n", + "WALK 4\r\n", + "DRIVE_HVY 1\r\n", + "WALK_HVY 1\r\n", + "WALK_LRF 1\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'othdiscr' (564 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.othdiscr.simple_simulate Running adaptive_chunked_choosers with 564 choosers\r\n", + "INFO - Running chunk 1 of 1 with 564 of 564 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate othdiscr choices_df top 10 value counts:\r\n", + "DRIVEALONEFREE 151\r\n", + "SHARED3FREE 104\r\n", + "SHARED2FREE 83\r\n", + "WALK 73\r\n", + "WALK_LOC 65\r\n", + "WALK_LRF 31\r\n", + "WALK_HVY 19\r\n", + "BIKE 19\r\n", + "TNC_SINGLE 12\r\n", + "TAXI 2\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'othmaint' (393 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.othmaint.simple_simulate Running adaptive_chunked_choosers with 393 choosers\r\n", + "INFO - Running chunk 1 of 1 with 393 of 393 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate othmaint choices_df top 10 value counts:\r\n", + "DRIVEALONEFREE 141\r\n", + "SHARED2FREE 62\r\n", + "WALK_LOC 36\r\n", + "SHARED3FREE 35\r\n", + "BIKE 28\r\n", + "WALK 23\r\n", + "WALK_LRF 23\r\n", + "TNC_SINGLE 21\r\n", + "TNC_SHARED 11\r\n", + "WALK_HVY 9\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'school' (460 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.school.simple_simulate Running adaptive_chunked_choosers with 460 choosers\r\n", + "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate school choices_df top 10 value counts:\r\n", + "SHARED3FREE 126\r\n", + "WALK_LRF 101\r\n", + "WALK_LOC 92\r\n", + "SHARED2FREE 59\r\n", + "WALK 48\r\n", + "WALK_HVY 23\r\n", + "BIKE 7\r\n", + "DRIVEALONEFREE 2\r\n", + "TNC_SHARED 1\r\n", + "TAXI 1\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'shopping' (723 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.shopping.simple_simulate Running adaptive_chunked_choosers with 723 choosers\r\n", + "INFO - Running chunk 1 of 1 with 723 of 723 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate shopping choices_df top 10 value counts:\r\n", + "DRIVEALONEFREE 270\r\n", + "SHARED2FREE 107\r\n", + "WALK 82\r\n", + "SHARED3FREE 75\r\n", + "WALK_LOC 46\r\n", + "TNC_SINGLE 44\r\n", + "WALK_LRF 35\r\n", + "BIKE 27\r\n", + "TNC_SHARED 17\r\n", + "WALK_HVY 9\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'social' (182 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.social.simple_simulate Running adaptive_chunked_choosers with 182 choosers\r\n", + "INFO - Running chunk 1 of 1 with 182 of 182 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate social choices_df top 10 value counts:\r\n", + "SHARED2FREE 43\r\n", + "DRIVEALONEFREE 29\r\n", + "SHARED3FREE 28\r\n", + "WALK 23\r\n", + "WALK_LRF 17\r\n", + "WALK_LOC 16\r\n", + "BIKE 15\r\n", + "TNC_SINGLE 8\r\n", + "WALK_HVY 2\r\n", + "TNC_SHARED 1\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'univ' (149 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.univ.simple_simulate Running adaptive_chunked_choosers with 149 choosers\r\n", + "INFO - Running chunk 1 of 1 with 149 of 149 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate univ choices_df top 10 value counts:\r\n", + "DRIVEALONEFREE 40\r\n", + "WALK_LRF 38\r\n", + "WALK_LOC 38\r\n", + "WALK_HVY 9\r\n", + "SHARED3FREE 7\r\n", + "TNC_SHARED 6\r\n", + "TAXI 4\r\n", + "SHARED2FREE 3\r\n", + "TNC_SINGLE 2\r\n", + "BIKE 1\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - tour_mode_choice_simulate tour_type 'work' (2124 tours)\r\n", + "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", + "INFO - tour_mode_choice.work.simple_simulate Running adaptive_chunked_choosers with 2124 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2124 of 2124 choosers\r\n", + "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", + "INFO - tour_mode_choice_simulate work choices_df top 10 value counts:\r\n", + "DRIVEALONEFREE 665\r\n", + "WALK_LOC 324\r\n", + "WALK_LRF 254\r\n", + "SHARED2FREE 216\r\n", + "TNC_SINGLE 177\r\n", + "SHARED3FREE 138\r\n", + "WALK 137\r\n", + "BIKE 109\r\n", + "WALK_HVY 82\r\n", + "TAXI 10\r\n", + "Name: tour_mode, dtype: int64\r\n", + "DEBUG - tour_mode_choice: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - tour_mode_choice: write_table cache: override_choices\r\n", + "DEBUG - tour_mode_choice: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - tour_mode_choice: write_omnibus_choosers: output/estimation_data_bundle/tour_mode_choice/tour_mode_choice_values_combined.csv\r\n", + "INFO - tour_mode_choice: end estimation\r\n", + "INFO - tour_mode_choice_simulate all tour type choices top 10 value counts:\r\n", + "DRIVEALONEFREE 1357\r\n", + "SHARED2FREE 801\r\n", + "SHARED3FREE 701\r\n", + "WALK_LOC 601\r\n", + "WALK_LRF 593\r\n", + "WALK 456\r\n", + "TNC_SINGLE 312\r\n", + "BIKE 200\r\n", + "WALK_HVY 194\r\n", + "TNC_SHARED 53\r\n", + "Name: tour_mode, dtype: int64\r\n", + "Time to execute step 'tour_mode_choice_simulate': 16.74 s\r\n", + "Total time to execute iteration 1 with iteration value None: 16.74 s\r\n", + "INFO - #run_model running step atwork_subtour_frequency\r\n", + "Running step 'atwork_subtour_frequency'\r\n", + "INFO - Initialize Estimator for'atwork_subtour_frequency'\r\n", + "INFO - Running atwork_subtour_frequency with 2124 work tours\r\n", + "DEBUG - atwork_subtour_frequency: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_frequency/atwork_subtour_frequency_SPEC.csv\r\n", + "DEBUG - atwork_subtour_frequency: write_table write: atwork_subtour_frequency_coefficients.csv\r\n", + "DEBUG - atwork_subtour_frequency: write_table cache: choosers\r\n", + "INFO - atwork_subtour_frequency.simple_simulate Running adaptive_chunked_choosers with 2124 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2124 of 2124 choosers\r\n", + "DEBUG - atwork_subtour_frequency: write_table cache: expression_values\r\n", + "DEBUG - atwork_subtour_frequency: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - atwork_subtour_frequency: write_table cache: override_choices\r\n", + "DEBUG - atwork_subtour_frequency: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - atwork_subtour_frequency: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_frequency/atwork_subtour_frequency_values_combined.csv\r\n", + "INFO - atwork_subtour_frequency: end estimation\r\n", + "INFO - atwork_subtour_frequency top 10 value counts:\r\n", + " 3654\r\n", + "no_subtours 1664\r\n", + "eat 346\r\n", + "maint 62\r\n", + "business1 48\r\n", + "eat_business 3\r\n", + "business2 1\r\n", + "Name: atwork_subtour_frequency, dtype: int64\r\n", + "Time to execute step 'atwork_subtour_frequency': 0.75 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.75 s\r\n", + "INFO - #run_model running step atwork_subtour_destination\r\n", + "Running step 'atwork_subtour_destination'\r\n", + "INFO - Initialize Estimator for'atwork_subtour_destination'\r\n", + "DEBUG - atwork_subtour_destination: write_table write: atwork_subtour_destination_coefficients.csv\r\n", + "DEBUG - atwork_subtour_destination: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_destination/atwork_subtour_destination_SPEC.csv\r\n", + "DEBUG - atwork_subtour_destination: write_table write: size_terms\r\n", + "DEBUG - atwork_subtour_destination: write_table write: landuse\r\n", + "INFO - running atwork_subtour_destination.atwork.sample with 464 tours\r\n", + "INFO - Estimation mode for atwork_subtour_destination.atwork.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - atwork_subtour_destination.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 464 choosers\r\n", + "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", + "INFO - Running eval_interaction_utilities on 88160 rows\r\n", + "INFO - Estimation mode for atwork_subtour_destination.atwork.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Running atwork_subtour_destination.atwork.logsums with 88160 rows\r\n", + "INFO - atwork_subtour_destination.atwork.logsums.compute_logsums Running adaptive_chunked_choosers with 88160 choosers\r\n", + "INFO - Running chunk 1 of 1 with 88160 of 88160 choosers\r\n", + "DEBUG - atwork_subtour_destination: write_table cache: choosers\r\n", + "INFO - Running tour_destination_simulate with 464 persons\r\n", + "INFO - atwork_subtour_destination.atwork.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 464 choosers and 88160 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", + "INFO - Running eval_interaction_utilities on 88160 rows\r\n", + "INFO - atwork_subtour_destination: eval_interaction_utilities write_interaction_expression_values atwork_subtour_destination.atwork.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - atwork_subtour_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - atwork_subtour_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - atwork_subtour_destination: write_table cache: override_choices\r\n", + "DEBUG - atwork_subtour_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - atwork_subtour_destination: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_destination/atwork_subtour_destination_choosers_combined.csv\r\n", + "DEBUG - atwork_subtour_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "WARNING - atwork_subtour_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", + "DEBUG - atwork_subtour_destination: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_destination/atwork_subtour_destination_alternatives_combined.csv\r\n", + "INFO - atwork_subtour_destination: end estimation\r\n", + "INFO - destination summary:\r\n", + "count 464.000000\r\n", + "mean 92.616379\r\n", + "std 54.698856\r\n", + "min 1.000000\r\n", + "25% 44.750000\r\n", + "50% 90.000000\r\n", + "75% 139.250000\r\n", + "max 190.000000\r\n", + "Name: destination, dtype: float64\r\n", + "Time to execute step 'atwork_subtour_destination': 9.38 s\r\n", + "Total time to execute iteration 1 with iteration value None: 9.38 s\r\n", + "INFO - #run_model running step atwork_subtour_scheduling\r\n", + "Running step 'atwork_subtour_scheduling'\r\n", + "INFO - Initialize Estimator for'atwork_subtour_scheduling'\r\n", + "INFO - Running atwork_subtour_scheduling with 464 tours\r\n", + "DEBUG - atwork_subtour_scheduling: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_scheduling/atwork_subtour_scheduling_SPEC.csv\r\n", + "DEBUG - atwork_subtour_scheduling: write_table write: tour_scheduling_atwork_coefficients.csv\r\n", + "INFO - atwork_subtour_scheduling.tour_1 schedule_tours running 460 tour choices\r\n", + "INFO - atwork_subtour_scheduling.tour_1 Running adaptive_chunked_choosers with 460 choosers\r\n", + "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", + "INFO - atwork_subtour_scheduling.tour_1 schedule_tours running 460 tour choices\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: choosers\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - atwork_subtour_scheduling.tour_1.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 460 choosers and 33700 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", + "INFO - Running eval_interaction_utilities on 33700 rows\r\n", + "INFO - atwork_subtour_scheduling: eval_interaction_utilities write_interaction_expression_values atwork_subtour_scheduling.tour_1.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_expression_values\r\n", + "INFO - atwork_subtour_scheduling.tour_2 schedule_tours running 4 tour choices\r\n", + "INFO - atwork_subtour_scheduling.tour_2 Running adaptive_chunked_choosers with 4 choosers\r\n", + "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", + "INFO - atwork_subtour_scheduling.tour_2 schedule_tours running 4 tour choices\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: choosers\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_sample_alternatives\r\n", + "INFO - atwork_subtour_scheduling.tour_2.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 4 choosers and 223 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", + "INFO - Running eval_interaction_utilities on 223 rows\r\n", + "INFO - atwork_subtour_scheduling: eval_interaction_utilities write_interaction_expression_values atwork_subtour_scheduling.tour_2.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_expression_values\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - atwork_subtour_scheduling: write_table cache: override_choices\r\n", + "DEBUG - atwork_subtour_scheduling: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - atwork_subtour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_scheduling/atwork_subtour_scheduling_choosers_combined.csv\r\n", + "DEBUG - atwork_subtour_scheduling: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "DEBUG - atwork_subtour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_scheduling/atwork_subtour_scheduling_alternatives_combined.csv\r\n", + "INFO - atwork_subtour_scheduling: end estimation\r\n", + "Time to execute step 'atwork_subtour_scheduling': 5.79 s\r\n", + "Total time to execute iteration 1 with iteration value None: 5.79 s\r\n", + "INFO - #run_model running step atwork_subtour_mode_choice\r\n", + "Running step 'atwork_subtour_mode_choice'\r\n", + "INFO - Running atwork_subtour_mode_choice with 464 subtours\r\n", + "INFO - atwork_subtour_mode_choice tour_type top 10 value counts:\r\n", + "eat 349\r\n", + "maint 62\r\n", + "business 53\r\n", + "Name: tour_type, dtype: int64\r\n", + "INFO - Initialize Estimator for'atwork_subtour_mode_choice'\r\n", + "DEBUG - atwork_subtour_mode_choice: write_table write: tour_mode_choice_coefficients.csv\r\n", + "DEBUG - atwork_subtour_mode_choice: write_table write: coefficients_template\r\n", + "DEBUG - atwork_subtour_mode_choice: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_mode_choice/atwork_subtour_mode_choice_SPEC.csv\r\n", + "WARNING - /mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/pandas/core/frame.py:3636: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", + " self[k1] = value[k2]\r\n", + "\r\n", + "WARNING - /mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/util/mode.py:129: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", + " choosers[trace_column_names] = choosers.index\r\n", + "\r\n", + "DEBUG - atwork_subtour_mode_choice: write_table cache: choosers\r\n", + "INFO - atwork_subtour_mode_choice.simple_simulate Running adaptive_chunked_choosers with 464 choosers\r\n", + "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", + "DEBUG - atwork_subtour_mode_choice: write_table cache: expression_values\r\n", + "DEBUG - atwork_subtour_mode_choice: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - atwork_subtour_mode_choice: write_table cache: override_choices\r\n", + "DEBUG - atwork_subtour_mode_choice: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - atwork_subtour_mode_choice: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_mode_choice/atwork_subtour_mode_choice_values_combined.csv\r\n", + "INFO - atwork_subtour_mode_choice: end estimation\r\n", + "INFO - atwork_subtour_mode_choice choices top 10 value counts:\r\n", + "DRIVEALONEFREE 133\r\n", + "WALK 99\r\n", + "SHARED2FREE 95\r\n", + "SHARED3FREE 67\r\n", + "TNC_SINGLE 27\r\n", + "WALK_LOC 13\r\n", + "TNC_SHARED 11\r\n", + "BIKE 10\r\n", + "WALK_LRF 5\r\n", + "TAXI 2\r\n", + "Name: tour_mode, dtype: int64\r\n", + "Time to execute step 'atwork_subtour_mode_choice': 2.23 s\r\n", + "Total time to execute iteration 1 with iteration value None: 2.23 s\r\n", + "INFO - #run_model running step stop_frequency\r\n", + "Running step 'stop_frequency'\r\n", + "INFO - stop_frequency segments top 10 value counts:\r\n", + "work 2124\r\n", + "shopping 723\r\n", + "othdiscr 564\r\n", + "atwork 464\r\n", + "school 460\r\n", + "escort 403\r\n", + "othmaint 393\r\n", + "eatout 316\r\n", + "social 182\r\n", + "univ 149\r\n", + "Name: primary_purpose, dtype: int64\r\n", + "INFO - stop_frequency running segment work with 2124 chooser rows\r\n", + "INFO - Initialize Estimator for'work'\r\n", + "DEBUG - work: estimate.write_spec: output/estimation_data_bundle/stop_frequency/work/stop_frequency_SPEC.csv\r\n", + "DEBUG - work: write_table write: stop_frequency_coefficients_work.csv\r\n", + "DEBUG - work: write_table cache: choosers\r\n", + "INFO - stop_frequency.work.simple_simulate Running adaptive_chunked_choosers with 2124 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2124 of 2124 choosers\r\n", + "DEBUG - work: write_table cache: expression_values\r\n", + "DEBUG - work: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - work: write_table cache: override_choices\r\n", + "DEBUG - work: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - work: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/work/stop_frequency_values_combined.csv\r\n", + "INFO - work: end estimation\r\n", + "INFO - stop_frequency running segment school with 460 chooser rows\r\n", + "INFO - Initialize Estimator for'school'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - school: estimate.write_spec: output/estimation_data_bundle/stop_frequency/school/stop_frequency_SPEC.csv\r\n", + "DEBUG - school: write_table write: stop_frequency_coefficients_school.csv\r\n", + "DEBUG - school: write_table cache: choosers\r\n", + "INFO - stop_frequency.school.simple_simulate Running adaptive_chunked_choosers with 460 choosers\r\n", + "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", + "DEBUG - school: write_table cache: expression_values\r\n", + "DEBUG - school: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - school: write_table cache: override_choices\r\n", + "DEBUG - school: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - school: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/school/stop_frequency_values_combined.csv\r\n", + "INFO - school: end estimation\r\n", + "INFO - stop_frequency running segment univ with 149 chooser rows\r\n", + "INFO - Initialize Estimator for'univ'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - univ: estimate.write_spec: output/estimation_data_bundle/stop_frequency/univ/stop_frequency_SPEC.csv\r\n", + "DEBUG - univ: write_table write: stop_frequency_coefficients_univ.csv\r\n", + "DEBUG - univ: write_table cache: choosers\r\n", + "INFO - stop_frequency.univ.simple_simulate Running adaptive_chunked_choosers with 149 choosers\r\n", + "INFO - Running chunk 1 of 1 with 149 of 149 choosers\r\n", + "DEBUG - univ: write_table cache: expression_values\r\n", + "DEBUG - univ: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - univ: write_table cache: override_choices\r\n", + "DEBUG - univ: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - univ: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/univ/stop_frequency_values_combined.csv\r\n", + "INFO - univ: end estimation\r\n", + "INFO - stop_frequency running segment social with 182 chooser rows\r\n", + "INFO - Initialize Estimator for'social'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - social: estimate.write_spec: output/estimation_data_bundle/stop_frequency/social/stop_frequency_SPEC.csv\r\n", + "DEBUG - social: write_table write: stop_frequency_coefficients_social.csv\r\n", + "DEBUG - social: write_table cache: choosers\r\n", + "INFO - stop_frequency.social.simple_simulate Running adaptive_chunked_choosers with 182 choosers\r\n", + "INFO - Running chunk 1 of 1 with 182 of 182 choosers\r\n", + "DEBUG - social: write_table cache: expression_values\r\n", + "DEBUG - social: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - social: write_table cache: override_choices\r\n", + "DEBUG - social: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - social: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/social/stop_frequency_values_combined.csv\r\n", + "INFO - social: end estimation\r\n", + "INFO - stop_frequency running segment shopping with 723 chooser rows\r\n", + "INFO - Initialize Estimator for'shopping'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - shopping: estimate.write_spec: output/estimation_data_bundle/stop_frequency/shopping/stop_frequency_SPEC.csv\r\n", + "DEBUG - shopping: write_table write: stop_frequency_coefficients_shopping.csv\r\n", + "DEBUG - shopping: write_table cache: choosers\r\n", + "INFO - stop_frequency.shopping.simple_simulate Running adaptive_chunked_choosers with 723 choosers\r\n", + "INFO - Running chunk 1 of 1 with 723 of 723 choosers\r\n", + "DEBUG - shopping: write_table cache: expression_values\r\n", + "DEBUG - shopping: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - shopping: write_table cache: override_choices\r\n", + "DEBUG - shopping: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - shopping: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/shopping/stop_frequency_values_combined.csv\r\n", + "INFO - shopping: end estimation\r\n", + "INFO - stop_frequency running segment eatout with 316 chooser rows\r\n", + "INFO - Initialize Estimator for'eatout'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - eatout: estimate.write_spec: output/estimation_data_bundle/stop_frequency/eatout/stop_frequency_SPEC.csv\r\n", + "DEBUG - eatout: write_table write: stop_frequency_coefficients_eatout.csv\r\n", + "DEBUG - eatout: write_table cache: choosers\r\n", + "INFO - stop_frequency.eatout.simple_simulate Running adaptive_chunked_choosers with 316 choosers\r\n", + "INFO - Running chunk 1 of 1 with 316 of 316 choosers\r\n", + "DEBUG - eatout: write_table cache: expression_values\r\n", + "DEBUG - eatout: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - eatout: write_table cache: override_choices\r\n", + "DEBUG - eatout: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - eatout: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/eatout/stop_frequency_values_combined.csv\r\n", + "INFO - eatout: end estimation\r\n", + "INFO - stop_frequency running segment escort with 403 chooser rows\r\n", + "INFO - Initialize Estimator for'escort'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - escort: estimate.write_spec: output/estimation_data_bundle/stop_frequency/escort/stop_frequency_SPEC.csv\r\n", + "DEBUG - escort: write_table write: stop_frequency_coefficients_escort.csv\r\n", + "DEBUG - escort: write_table cache: choosers\r\n", + "INFO - stop_frequency.escort.simple_simulate Running adaptive_chunked_choosers with 403 choosers\r\n", + "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", + "DEBUG - escort: write_table cache: expression_values\r\n", + "DEBUG - escort: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - escort: write_table cache: override_choices\r\n", + "DEBUG - escort: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - escort: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/escort/stop_frequency_values_combined.csv\r\n", + "INFO - escort: end estimation\r\n", + "INFO - stop_frequency running segment othmaint with 393 chooser rows\r\n", + "INFO - Initialize Estimator for'othmaint'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - othmaint: estimate.write_spec: output/estimation_data_bundle/stop_frequency/othmaint/stop_frequency_SPEC.csv\r\n", + "DEBUG - othmaint: write_table write: stop_frequency_coefficients_othmaint.csv\r\n", + "DEBUG - othmaint: write_table cache: choosers\r\n", + "INFO - stop_frequency.othmaint.simple_simulate Running adaptive_chunked_choosers with 393 choosers\r\n", + "INFO - Running chunk 1 of 1 with 393 of 393 choosers\r\n", + "DEBUG - othmaint: write_table cache: expression_values\r\n", + "DEBUG - othmaint: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - othmaint: write_table cache: override_choices\r\n", + "DEBUG - othmaint: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - othmaint: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/othmaint/stop_frequency_values_combined.csv\r\n", + "INFO - othmaint: end estimation\r\n", + "INFO - stop_frequency running segment othdiscr with 564 chooser rows\r\n", + "INFO - Initialize Estimator for'othdiscr'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - othdiscr: estimate.write_spec: output/estimation_data_bundle/stop_frequency/othdiscr/stop_frequency_SPEC.csv\r\n", + "DEBUG - othdiscr: write_table write: stop_frequency_coefficients_othdiscr.csv\r\n", + "DEBUG - othdiscr: write_table cache: choosers\r\n", + "INFO - stop_frequency.othdiscr.simple_simulate Running adaptive_chunked_choosers with 564 choosers\r\n", + "INFO - Running chunk 1 of 1 with 564 of 564 choosers\r\n", + "DEBUG - othdiscr: write_table cache: expression_values\r\n", + "DEBUG - othdiscr: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - othdiscr: write_table cache: override_choices\r\n", + "DEBUG - othdiscr: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - othdiscr: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/othdiscr/stop_frequency_values_combined.csv\r\n", + "INFO - othdiscr: end estimation\r\n", + "INFO - stop_frequency running segment atwork with 464 chooser rows\r\n", + "INFO - Initialize Estimator for'atwork'\r\n", + "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", + "DEBUG - atwork: estimate.write_spec: output/estimation_data_bundle/stop_frequency/atwork/stop_frequency_SPEC.csv\r\n", + "DEBUG - atwork: write_table write: stop_frequency_coefficients_atwork.csv\r\n", + "DEBUG - atwork: write_table cache: choosers\r\n", + "INFO - stop_frequency.atwork.simple_simulate Running adaptive_chunked_choosers with 464 choosers\r\n", + "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", + "DEBUG - atwork: write_table cache: expression_values\r\n", + "DEBUG - atwork: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using tours.index\r\n", + "DEBUG - atwork: write_table cache: override_choices\r\n", + "DEBUG - atwork: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - atwork: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/atwork/stop_frequency_values_combined.csv\r\n", + "INFO - atwork: end estimation\r\n", + "INFO - stop_frequency top 10 value counts:\r\n", + "0out_0in 4040\r\n", + "0out_1in 636\r\n", + "1out_0in 421\r\n", + "0out_2in 178\r\n", + "1out_1in 164\r\n", + "0out_3in 91\r\n", + "2out_0in 62\r\n", + "1out_3in 50\r\n", + "1out_2in 44\r\n", + "2out_1in 31\r\n", + "Name: stop_frequency, dtype: int64\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "Time to execute step 'stop_frequency': 6.05 s\r\n", + "Total time to execute iteration 1 with iteration value None: 6.05 s\r\n", + "INFO - #run_model running step trip_purpose\r\n", + "Running step 'trip_purpose'\r\n", + "INFO - Initialize Estimator for'trip_purpose'\r\n", + "DEBUG - trip_purpose: write_table cache: choosers\r\n", + "DEBUG - trip_purpose: estimate.write_spec: output/estimation_data_bundle/trip_purpose/trip_purpose_PROBS_SPEC.csv\r\n", + "INFO - assign purpose to 5778 last outbound trips\r\n", + "INFO - assign purpose to 5778 last inbound trips\r\n", + "INFO - assign purpose to 2796 intermediate trips\r\n", + "INFO - trip_purpose Running adaptive_chunked_choosers with 2796 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2796 of 2796 choosers\r\n", + " primary_purpose outbound ... social othdiscr\r\n", + "trip_id ... \r\n", + "206561 univ True ... 0.048 0.050\r\n", + "9188185 univ True ... 0.048 0.050\r\n", + "9668377 univ True ... 0.048 0.050\r\n", + "9806797 univ False ... 0.067 0.113\r\n", + "10849837 univ False ... 0.067 0.113\r\n", + "... ... ... ... ... ...\r\n", + "2414259747 atwork True ... 0.004 0.014\r\n", + "2414259749 atwork False ... 0.019 0.018\r\n", + "2414488393 atwork True ... 0.004 0.014\r\n", + "2423389333 atwork False ... 0.019 0.018\r\n", + "2440963725 atwork False ... 0.019 0.018\r\n", + "\r\n", + "[2796 rows x 14 columns]\r\n", + "DEBUG - trip_purpose: write_table cache: probs\r\n", + "DEBUG - trip_purpose: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_purpose: write_table cache: override_choices\r\n", + "DEBUG - trip_purpose: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'choosers', 'probs']\r\n", + "DEBUG - trip_purpose: write_omnibus_choosers: output/estimation_data_bundle/trip_purpose/trip_purpose_values_combined.csv\r\n", + "INFO - trip_purpose: end estimation\r\n", + "Time to execute step 'trip_purpose': 0.33 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.33 s\r\n", + "INFO - #run_model running step trip_destination\r\n", + "Running step 'trip_destination'\r\n", + "INFO - Initialize Estimator for'trip_destination'\r\n", + "DEBUG - trip_destination: write_table write: trip_destination_coefficients.csv\r\n", + "DEBUG - trip_destination: estimate.write_spec: output/estimation_data_bundle/trip_destination/trip_destination_SPEC.csv\r\n", + "DEBUG - trip_destination: write_table write: size_terms\r\n", + "DEBUG - trip_destination: write_table write: landuse\r\n", + "INFO - Running trip_destination with 14352 trips\r\n", + "INFO - Running trip_destination.trip_num_1 with 2061 trips\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.atwork with 124 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.atwork.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 124 choosers\r\n", + "INFO - Running chunk 1 of 1 with 124 of 124 choosers\r\n", + "INFO - Running eval_interaction_utilities on 23560 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.atwork.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.atwork.trip_destination_sample : 0.09 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.atwork.compute_logsums with 23560 samples\r\n", + "INFO - trip_destination.trip_num_1.atwork.compute_logsums.od Running adaptive_chunked_choosers with 23560 choosers\r\n", + "INFO - Running chunk 1 of 1 with 23560 of 23560 choosers\r\n", + "INFO - trip_destination.trip_num_1.atwork.compute_logsums.dp Running adaptive_chunked_choosers with 23560 choosers\r\n", + "INFO - Running chunk 1 of 1 with 23560 of 23560 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.atwork.compute_logsums : 4.409 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 124 trips\r\n", + "INFO - trip_destination.trip_num_1.atwork.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 124 choosers and 23560 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 124 of 124 choosers\r\n", + "INFO - Running eval_interaction_utilities on 23560 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.atwork.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.atwork.trip_destination_simulate : 0.372 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.eatout with 66 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.eatout.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 66 choosers\r\n", + "INFO - Running chunk 1 of 1 with 66 of 66 choosers\r\n", + "INFO - Running eval_interaction_utilities on 12540 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.eatout.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.eatout.trip_destination_sample : 0.086 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.eatout.compute_logsums with 12540 samples\r\n", + "INFO - trip_destination.trip_num_1.eatout.compute_logsums.od Running adaptive_chunked_choosers with 12540 choosers\r\n", + "INFO - Running chunk 1 of 1 with 12540 of 12540 choosers\r\n", + "INFO - trip_destination.trip_num_1.eatout.compute_logsums.dp Running adaptive_chunked_choosers with 12540 choosers\r\n", + "INFO - Running chunk 1 of 1 with 12540 of 12540 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.eatout.compute_logsums : 3.345 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 66 trips\r\n", + "INFO - trip_destination.trip_num_1.eatout.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 66 choosers and 12540 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 66 of 66 choosers\r\n", + "INFO - Running eval_interaction_utilities on 12540 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.eatout.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.eatout.trip_destination_simulate : 0.266 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.escort with 117 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.escort.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.escort.sample.interaction_sample Running adaptive_chunked_choosers with 117 choosers\r\n", + "INFO - Running chunk 1 of 1 with 117 of 117 choosers\r\n", + "INFO - Running eval_interaction_utilities on 22230 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.escort.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.escort.trip_destination_sample : 0.119 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.escort.compute_logsums with 22230 samples\r\n", + "INFO - trip_destination.trip_num_1.escort.compute_logsums.od Running adaptive_chunked_choosers with 22230 choosers\r\n", + "INFO - Running chunk 1 of 1 with 22230 of 22230 choosers\r\n", + "INFO - trip_destination.trip_num_1.escort.compute_logsums.dp Running adaptive_chunked_choosers with 22230 choosers\r\n", + "INFO - Running chunk 1 of 1 with 22230 of 22230 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.escort.compute_logsums : 4.465 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 117 trips\r\n", + "INFO - trip_destination.trip_num_1.escort.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 117 choosers and 22230 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 117 of 117 choosers\r\n", + "INFO - Running eval_interaction_utilities on 22230 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.escort.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.escort.trip_destination_simulate : 0.431 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.othdiscr with 207 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 207 choosers\r\n", + "INFO - Running chunk 1 of 1 with 207 of 207 choosers\r\n", + "INFO - Running eval_interaction_utilities on 39330 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.othdiscr.trip_destination_sample : 0.117 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.othdiscr.compute_logsums with 39330 samples\r\n", + "INFO - trip_destination.trip_num_1.othdiscr.compute_logsums.od Running adaptive_chunked_choosers with 39330 choosers\r\n", + "INFO - Running chunk 1 of 1 with 39330 of 39330 choosers\r\n", + "INFO - trip_destination.trip_num_1.othdiscr.compute_logsums.dp Running adaptive_chunked_choosers with 39330 choosers\r\n", + "INFO - Running chunk 1 of 1 with 39330 of 39330 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.othdiscr.compute_logsums : 6.014 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 207 trips\r\n", + "INFO - trip_destination.trip_num_1.othdiscr.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 207 choosers and 39330 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 207 of 207 choosers\r\n", + "INFO - Running eval_interaction_utilities on 39330 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.othdiscr.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.othdiscr.trip_destination_simulate : 0.567 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.othmaint with 122 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 122 choosers\r\n", + "INFO - Running chunk 1 of 1 with 122 of 122 choosers\r\n", + "INFO - Running eval_interaction_utilities on 23180 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.othmaint.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.othmaint.trip_destination_sample : 0.095 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.othmaint.compute_logsums with 23180 samples\r\n", + "INFO - trip_destination.trip_num_1.othmaint.compute_logsums.od Running adaptive_chunked_choosers with 23180 choosers\r\n", + "INFO - Running chunk 1 of 1 with 23180 of 23180 choosers\r\n", + "INFO - trip_destination.trip_num_1.othmaint.compute_logsums.dp Running adaptive_chunked_choosers with 23180 choosers\r\n", + "INFO - Running chunk 1 of 1 with 23180 of 23180 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.othmaint.compute_logsums : 4.333 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 122 trips\r\n", + "INFO - trip_destination.trip_num_1.othmaint.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 122 choosers and 23180 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 122 of 122 choosers\r\n", + "INFO - Running eval_interaction_utilities on 23180 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.othmaint.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.othmaint.trip_destination_simulate : 0.417 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.school with 119 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.school.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.school.sample.interaction_sample Running adaptive_chunked_choosers with 119 choosers\r\n", + "INFO - Running chunk 1 of 1 with 119 of 119 choosers\r\n", + "INFO - Running eval_interaction_utilities on 22610 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.school.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.school.trip_destination_sample : 0.091 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.school.compute_logsums with 22610 samples\r\n", + "INFO - trip_destination.trip_num_1.school.compute_logsums.od Running adaptive_chunked_choosers with 22610 choosers\r\n", + "INFO - Running chunk 1 of 1 with 22610 of 22610 choosers\r\n", + "INFO - trip_destination.trip_num_1.school.compute_logsums.dp Running adaptive_chunked_choosers with 22610 choosers\r\n", + "INFO - Running chunk 1 of 1 with 22610 of 22610 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.school.compute_logsums : 4.473 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 119 trips\r\n", + "INFO - trip_destination.trip_num_1.school.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 119 choosers and 22610 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 119 of 119 choosers\r\n", + "INFO - Running eval_interaction_utilities on 22610 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.school.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.school.trip_destination_simulate : 0.423 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.shopping with 322 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.shopping.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 322 choosers\r\n", + "INFO - Running chunk 1 of 1 with 322 of 322 choosers\r\n", + "INFO - Running eval_interaction_utilities on 61180 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.shopping.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.shopping.trip_destination_sample : 0.16 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.shopping.compute_logsums with 61180 samples\r\n", + "INFO - trip_destination.trip_num_1.shopping.compute_logsums.od Running adaptive_chunked_choosers with 61180 choosers\r\n", + "INFO - Running chunk 1 of 1 with 61180 of 61180 choosers\r\n", + "INFO - trip_destination.trip_num_1.shopping.compute_logsums.dp Running adaptive_chunked_choosers with 61180 choosers\r\n", + "INFO - Running chunk 1 of 1 with 61180 of 61180 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.shopping.compute_logsums : 8.168 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 322 trips\r\n", + "INFO - trip_destination.trip_num_1.shopping.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 322 choosers and 61180 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 322 of 322 choosers\r\n", + "INFO - Running eval_interaction_utilities on 61180 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.shopping.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.shopping.trip_destination_simulate : 0.892 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.social with 50 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.social.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.social.sample.interaction_sample Running adaptive_chunked_choosers with 50 choosers\r\n", + "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", + "INFO - Running eval_interaction_utilities on 9500 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.social.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.social.trip_destination_sample : 0.074 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.social.compute_logsums with 9500 samples\r\n", + "INFO - trip_destination.trip_num_1.social.compute_logsums.od Running adaptive_chunked_choosers with 9500 choosers\r\n", + "INFO - Running chunk 1 of 1 with 9500 of 9500 choosers\r\n", + "INFO - trip_destination.trip_num_1.social.compute_logsums.dp Running adaptive_chunked_choosers with 9500 choosers\r\n", + "INFO - Running chunk 1 of 1 with 9500 of 9500 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.social.compute_logsums : 2.802 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 50 trips\r\n", + "INFO - trip_destination.trip_num_1.social.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 50 choosers and 9500 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", + "INFO - Running eval_interaction_utilities on 9500 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.social.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.social.trip_destination_simulate : 0.285 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.univ with 75 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.univ.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.univ.sample.interaction_sample Running adaptive_chunked_choosers with 75 choosers\r\n", + "INFO - Running chunk 1 of 1 with 75 of 75 choosers\r\n", + "INFO - Running eval_interaction_utilities on 14250 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.univ.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.univ.trip_destination_sample : 0.084 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.univ.compute_logsums with 14250 samples\r\n", + "INFO - trip_destination.trip_num_1.univ.compute_logsums.od Running adaptive_chunked_choosers with 14250 choosers\r\n", + "INFO - Running chunk 1 of 1 with 14250 of 14250 choosers\r\n", + "INFO - trip_destination.trip_num_1.univ.compute_logsums.dp Running adaptive_chunked_choosers with 14250 choosers\r\n", + "INFO - Running chunk 1 of 1 with 14250 of 14250 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.univ.compute_logsums : 3.17 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 75 trips\r\n", + "INFO - trip_destination.trip_num_1.univ.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 75 choosers and 14250 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 75 of 75 choosers\r\n", + "INFO - Running eval_interaction_utilities on 14250 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.univ.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.univ.trip_destination_simulate : 0.31 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_1.work with 859 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.work.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_1.work.sample.interaction_sample Running adaptive_chunked_choosers with 859 choosers\r\n", + "INFO - Running chunk 1 of 1 with 859 of 859 choosers\r\n", + "INFO - Running eval_interaction_utilities on 163210 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_1.work.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_1.work.trip_destination_sample : 0.278 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_1.work.compute_logsums with 163210 samples\r\n", + "INFO - trip_destination.trip_num_1.work.compute_logsums.od Running adaptive_chunked_choosers with 163210 choosers\r\n", + "INFO - Running chunk 1 of 1 with 163210 of 163210 choosers\r\n", + "INFO - trip_destination.trip_num_1.work.compute_logsums.dp Running adaptive_chunked_choosers with 163210 choosers\r\n", + "INFO - Running chunk 1 of 1 with 163210 of 163210 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_1.work.compute_logsums : 17.277 seconds (0.3 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 859 trips\r\n", + "INFO - trip_destination.trip_num_1.work.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 859 choosers and 163210 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 859 of 859 choosers\r\n", + "INFO - Running eval_interaction_utilities on 163210 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.work.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_1.work.trip_destination_simulate : 3.053 seconds (0.1 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2 with 542 trips\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.atwork with 18 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.atwork.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 18 choosers\r\n", + "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3420 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.atwork.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.atwork.trip_destination_sample : 0.07 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.atwork.compute_logsums with 3420 samples\r\n", + "INFO - trip_destination.trip_num_2.atwork.compute_logsums.od Running adaptive_chunked_choosers with 3420 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3420 of 3420 choosers\r\n", + "INFO - trip_destination.trip_num_2.atwork.compute_logsums.dp Running adaptive_chunked_choosers with 3420 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3420 of 3420 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.atwork.compute_logsums : 2.279 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 18 trips\r\n", + "INFO - trip_destination.trip_num_2.atwork.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 18 choosers and 3420 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3420 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.atwork.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.atwork.trip_destination_simulate : 0.383 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.eatout with 11 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.eatout.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 11 choosers\r\n", + "INFO - Running chunk 1 of 1 with 11 of 11 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2090 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.eatout.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.eatout.trip_destination_sample : 0.084 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.eatout.compute_logsums with 2090 samples\r\n", + "INFO - trip_destination.trip_num_2.eatout.compute_logsums.od Running adaptive_chunked_choosers with 2090 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2090 of 2090 choosers\r\n", + "INFO - trip_destination.trip_num_2.eatout.compute_logsums.dp Running adaptive_chunked_choosers with 2090 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2090 of 2090 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.eatout.compute_logsums : 2.19 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 11 trips\r\n", + "INFO - trip_destination.trip_num_2.eatout.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 11 choosers and 2090 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 11 of 11 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2090 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.eatout.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.eatout.trip_destination_simulate : 0.486 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.escort with 32 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.escort.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.escort.sample.interaction_sample Running adaptive_chunked_choosers with 32 choosers\r\n", + "INFO - Running chunk 1 of 1 with 32 of 32 choosers\r\n", + "INFO - Running eval_interaction_utilities on 6080 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.escort.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.escort.trip_destination_sample : 0.092 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.escort.compute_logsums with 6080 samples\r\n", + "INFO - trip_destination.trip_num_2.escort.compute_logsums.od Running adaptive_chunked_choosers with 6080 choosers\r\n", + "INFO - Running chunk 1 of 1 with 6080 of 6080 choosers\r\n", + "INFO - trip_destination.trip_num_2.escort.compute_logsums.dp Running adaptive_chunked_choosers with 6080 choosers\r\n", + "INFO - Running chunk 1 of 1 with 6080 of 6080 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.escort.compute_logsums : 2.883 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 32 trips\r\n", + "INFO - trip_destination.trip_num_2.escort.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 32 choosers and 6080 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 32 of 32 choosers\r\n", + "INFO - Running eval_interaction_utilities on 6080 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.escort.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.escort.trip_destination_simulate : 0.844 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.othdiscr with 56 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 56 choosers\r\n", + "INFO - Running chunk 1 of 1 with 56 of 56 choosers\r\n", + "INFO - Running eval_interaction_utilities on 10640 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.othdiscr.trip_destination_sample : 0.15 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.othdiscr.compute_logsums with 10640 samples\r\n", + "INFO - trip_destination.trip_num_2.othdiscr.compute_logsums.od Running adaptive_chunked_choosers with 10640 choosers\r\n", + "INFO - Running chunk 1 of 1 with 10640 of 10640 choosers\r\n", + "INFO - trip_destination.trip_num_2.othdiscr.compute_logsums.dp Running adaptive_chunked_choosers with 10640 choosers\r\n", + "INFO - Running chunk 1 of 1 with 10640 of 10640 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.othdiscr.compute_logsums : 4.791 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 56 trips\r\n", + "INFO - trip_destination.trip_num_2.othdiscr.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 56 choosers and 10640 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 56 of 56 choosers\r\n", + "INFO - Running eval_interaction_utilities on 10640 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.othdiscr.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.othdiscr.trip_destination_simulate : 0.442 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.othmaint with 53 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 53 choosers\r\n", + "INFO - Running chunk 1 of 1 with 53 of 53 choosers\r\n", + "INFO - Running eval_interaction_utilities on 10070 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.othmaint.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.othmaint.trip_destination_sample : 0.07 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.othmaint.compute_logsums with 10070 samples\r\n", + "INFO - trip_destination.trip_num_2.othmaint.compute_logsums.od Running adaptive_chunked_choosers with 10070 choosers\r\n", + "INFO - Running chunk 1 of 1 with 10070 of 10070 choosers\r\n", + "INFO - trip_destination.trip_num_2.othmaint.compute_logsums.dp Running adaptive_chunked_choosers with 10070 choosers\r\n", + "INFO - Running chunk 1 of 1 with 10070 of 10070 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.othmaint.compute_logsums : 2.728 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 53 trips\r\n", + "INFO - trip_destination.trip_num_2.othmaint.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 53 choosers and 10070 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 53 of 53 choosers\r\n", + "INFO - Running eval_interaction_utilities on 10070 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.othmaint.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.othmaint.trip_destination_simulate : 0.486 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.school with 29 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.school.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.school.sample.interaction_sample Running adaptive_chunked_choosers with 29 choosers\r\n", + "INFO - Running chunk 1 of 1 with 29 of 29 choosers\r\n", + "INFO - Running eval_interaction_utilities on 5510 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.school.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.school.trip_destination_sample : 0.055 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.school.compute_logsums with 5510 samples\r\n", + "INFO - trip_destination.trip_num_2.school.compute_logsums.od Running adaptive_chunked_choosers with 5510 choosers\r\n", + "INFO - Running chunk 1 of 1 with 5510 of 5510 choosers\r\n", + "INFO - trip_destination.trip_num_2.school.compute_logsums.dp Running adaptive_chunked_choosers with 5510 choosers\r\n", + "INFO - Running chunk 1 of 1 with 5510 of 5510 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.school.compute_logsums : 2.617 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 29 trips\r\n", + "INFO - trip_destination.trip_num_2.school.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 29 choosers and 5510 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 29 of 29 choosers\r\n", + "INFO - Running eval_interaction_utilities on 5510 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.school.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.school.trip_destination_simulate : 0.609 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.shopping with 80 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.shopping.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 80 choosers\r\n", + "INFO - Running chunk 1 of 1 with 80 of 80 choosers\r\n", + "INFO - Running eval_interaction_utilities on 15200 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.shopping.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.shopping.trip_destination_sample : 0.073 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.shopping.compute_logsums with 15200 samples\r\n", + "INFO - trip_destination.trip_num_2.shopping.compute_logsums.od Running adaptive_chunked_choosers with 15200 choosers\r\n", + "INFO - Running chunk 1 of 1 with 15200 of 15200 choosers\r\n", + "INFO - trip_destination.trip_num_2.shopping.compute_logsums.dp Running adaptive_chunked_choosers with 15200 choosers\r\n", + "INFO - Running chunk 1 of 1 with 15200 of 15200 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.shopping.compute_logsums : 3.393 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 80 trips\r\n", + "INFO - trip_destination.trip_num_2.shopping.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 80 choosers and 15200 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 80 of 80 choosers\r\n", + "INFO - Running eval_interaction_utilities on 15200 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.shopping.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.shopping.trip_destination_simulate : 0.622 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.social with 7 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.social.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.social.sample.interaction_sample Running adaptive_chunked_choosers with 7 choosers\r\n", + "INFO - Running chunk 1 of 1 with 7 of 7 choosers\r\n", + "INFO - Running eval_interaction_utilities on 1330 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.social.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.social.trip_destination_sample : 0.076 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.social.compute_logsums with 1330 samples\r\n", + "INFO - trip_destination.trip_num_2.social.compute_logsums.od Running adaptive_chunked_choosers with 1330 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1330 of 1330 choosers\r\n", + "INFO - trip_destination.trip_num_2.social.compute_logsums.dp Running adaptive_chunked_choosers with 1330 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1330 of 1330 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.social.compute_logsums : 2.247 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 7 trips\r\n", + "INFO - trip_destination.trip_num_2.social.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 7 choosers and 1330 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 7 of 7 choosers\r\n", + "INFO - Running eval_interaction_utilities on 1330 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.social.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.social.trip_destination_simulate : 0.414 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.univ with 20 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.univ.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.univ.sample.interaction_sample Running adaptive_chunked_choosers with 20 choosers\r\n", + "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3800 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.univ.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.univ.trip_destination_sample : 0.054 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.univ.compute_logsums with 3800 samples\r\n", + "INFO - trip_destination.trip_num_2.univ.compute_logsums.od Running adaptive_chunked_choosers with 3800 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3800 of 3800 choosers\r\n", + "INFO - trip_destination.trip_num_2.univ.compute_logsums.dp Running adaptive_chunked_choosers with 3800 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3800 of 3800 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.univ.compute_logsums : 2.624 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 20 trips\r\n", + "INFO - trip_destination.trip_num_2.univ.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 20 choosers and 3800 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", + "INFO - Running eval_interaction_utilities on 3800 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.univ.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.univ.trip_destination_simulate : 0.485 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_2.work with 236 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.work.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_2.work.sample.interaction_sample Running adaptive_chunked_choosers with 236 choosers\r\n", + "INFO - Running chunk 1 of 1 with 236 of 236 choosers\r\n", + "INFO - Running eval_interaction_utilities on 44840 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_2.work.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_2.work.trip_destination_sample : 0.121 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_2.work.compute_logsums with 44840 samples\r\n", + "INFO - trip_destination.trip_num_2.work.compute_logsums.od Running adaptive_chunked_choosers with 44840 choosers\r\n", + "INFO - Running chunk 1 of 1 with 44840 of 44840 choosers\r\n", + "INFO - trip_destination.trip_num_2.work.compute_logsums.dp Running adaptive_chunked_choosers with 44840 choosers\r\n", + "INFO - Running chunk 1 of 1 with 44840 of 44840 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_2.work.compute_logsums : 6.123 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 236 trips\r\n", + "INFO - trip_destination.trip_num_2.work.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 236 choosers and 44840 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 236 of 236 choosers\r\n", + "INFO - Running eval_interaction_utilities on 44840 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.work.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_2.work.trip_destination_simulate : 0.961 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3 with 193 trips\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.atwork with 3 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.atwork.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 3 choosers\r\n", + "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", + "INFO - Running eval_interaction_utilities on 570 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.atwork.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.atwork.trip_destination_sample : 0.062 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.atwork.compute_logsums with 570 samples\r\n", + "INFO - trip_destination.trip_num_3.atwork.compute_logsums.od Running adaptive_chunked_choosers with 570 choosers\r\n", + "INFO - Running chunk 1 of 1 with 570 of 570 choosers\r\n", + "INFO - trip_destination.trip_num_3.atwork.compute_logsums.dp Running adaptive_chunked_choosers with 570 choosers\r\n", + "INFO - Running chunk 1 of 1 with 570 of 570 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.atwork.compute_logsums : 1.823 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 3 trips\r\n", + "INFO - trip_destination.trip_num_3.atwork.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 3 choosers and 570 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", + "INFO - Running eval_interaction_utilities on 570 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.atwork.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.atwork.trip_destination_simulate : 0.433 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.eatout with 4 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.eatout.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 4 choosers\r\n", + "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", + "INFO - Running eval_interaction_utilities on 760 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.eatout.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.eatout.trip_destination_sample : 0.069 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.eatout.compute_logsums with 760 samples\r\n", + "INFO - trip_destination.trip_num_3.eatout.compute_logsums.od Running adaptive_chunked_choosers with 760 choosers\r\n", + "INFO - Running chunk 1 of 1 with 760 of 760 choosers\r\n", + "INFO - trip_destination.trip_num_3.eatout.compute_logsums.dp Running adaptive_chunked_choosers with 760 choosers\r\n", + "INFO - Running chunk 1 of 1 with 760 of 760 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.eatout.compute_logsums : 1.92 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 4 trips\r\n", + "INFO - trip_destination.trip_num_3.eatout.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 4 choosers and 760 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", + "INFO - Running eval_interaction_utilities on 760 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.eatout.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.eatout.trip_destination_simulate : 0.447 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.escort with 12 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.escort.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.escort.sample.interaction_sample Running adaptive_chunked_choosers with 12 choosers\r\n", + "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2280 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.escort.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.escort.trip_destination_sample : 0.048 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.escort.compute_logsums with 2280 samples\r\n", + "INFO - trip_destination.trip_num_3.escort.compute_logsums.od Running adaptive_chunked_choosers with 2280 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2280 of 2280 choosers\r\n", + "INFO - trip_destination.trip_num_3.escort.compute_logsums.dp Running adaptive_chunked_choosers with 2280 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2280 of 2280 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.escort.compute_logsums : 2.102 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 12 trips\r\n", + "INFO - trip_destination.trip_num_3.escort.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 12 choosers and 2280 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2280 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.escort.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.escort.trip_destination_simulate : 0.462 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.othdiscr with 22 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 22 choosers\r\n", + "INFO - Running chunk 1 of 1 with 22 of 22 choosers\r\n", + "INFO - Running eval_interaction_utilities on 4180 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.othdiscr.trip_destination_sample : 0.071 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.othdiscr.compute_logsums with 4180 samples\r\n", + "INFO - trip_destination.trip_num_3.othdiscr.compute_logsums.od Running adaptive_chunked_choosers with 4180 choosers\r\n", + "INFO - Running chunk 1 of 1 with 4180 of 4180 choosers\r\n", + "INFO - trip_destination.trip_num_3.othdiscr.compute_logsums.dp Running adaptive_chunked_choosers with 4180 choosers\r\n", + "INFO - Running chunk 1 of 1 with 4180 of 4180 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.othdiscr.compute_logsums : 2.22 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 22 trips\r\n", + "INFO - trip_destination.trip_num_3.othdiscr.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 22 choosers and 4180 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 22 of 22 choosers\r\n", + "INFO - Running eval_interaction_utilities on 4180 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.othdiscr.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.othdiscr.trip_destination_simulate : 0.506 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.othmaint with 14 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 14 choosers\r\n", + "INFO - Running chunk 1 of 1 with 14 of 14 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2660 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.othmaint.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.othmaint.trip_destination_sample : 0.066 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.othmaint.compute_logsums with 2660 samples\r\n", + "INFO - trip_destination.trip_num_3.othmaint.compute_logsums.od Running adaptive_chunked_choosers with 2660 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2660 of 2660 choosers\r\n", + "INFO - trip_destination.trip_num_3.othmaint.compute_logsums.dp Running adaptive_chunked_choosers with 2660 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2660 of 2660 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.othmaint.compute_logsums : 2.088 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 14 trips\r\n", + "INFO - trip_destination.trip_num_3.othmaint.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 14 choosers and 2660 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 14 of 14 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2660 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.othmaint.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.othmaint.trip_destination_simulate : 0.483 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.school with 13 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.school.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.school.sample.interaction_sample Running adaptive_chunked_choosers with 13 choosers\r\n", + "INFO - Running chunk 1 of 1 with 13 of 13 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2470 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.school.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.school.trip_destination_sample : 0.052 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.school.compute_logsums with 2470 samples\r\n", + "INFO - trip_destination.trip_num_3.school.compute_logsums.od Running adaptive_chunked_choosers with 2470 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2470 of 2470 choosers\r\n", + "INFO - trip_destination.trip_num_3.school.compute_logsums.dp Running adaptive_chunked_choosers with 2470 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2470 of 2470 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.school.compute_logsums : 2.017 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 13 trips\r\n", + "INFO - trip_destination.trip_num_3.school.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 13 choosers and 2470 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 13 of 13 choosers\r\n", + "INFO - Running eval_interaction_utilities on 2470 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.school.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.school.trip_destination_simulate : 0.458 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.shopping with 28 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.shopping.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 28 choosers\r\n", + "INFO - Running chunk 1 of 1 with 28 of 28 choosers\r\n", + "INFO - Running eval_interaction_utilities on 5320 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.shopping.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.shopping.trip_destination_sample : 0.078 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.shopping.compute_logsums with 5320 samples\r\n", + "INFO - trip_destination.trip_num_3.shopping.compute_logsums.od Running adaptive_chunked_choosers with 5320 choosers\r\n", + "INFO - Running chunk 1 of 1 with 5320 of 5320 choosers\r\n", + "INFO - trip_destination.trip_num_3.shopping.compute_logsums.dp Running adaptive_chunked_choosers with 5320 choosers\r\n", + "INFO - Running chunk 1 of 1 with 5320 of 5320 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.shopping.compute_logsums : 2.804 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 28 trips\r\n", + "INFO - trip_destination.trip_num_3.shopping.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 28 choosers and 5320 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 28 of 28 choosers\r\n", + "INFO - Running eval_interaction_utilities on 5320 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.shopping.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.shopping.trip_destination_simulate : 0.513 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.social with 2 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.social.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.social.sample.interaction_sample Running adaptive_chunked_choosers with 2 choosers\r\n", + "INFO - Running chunk 1 of 1 with 2 of 2 choosers\r\n", + "INFO - Running eval_interaction_utilities on 380 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.social.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.social.trip_destination_sample : 0.07 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.social.compute_logsums with 380 samples\r\n", + "INFO - trip_destination.trip_num_3.social.compute_logsums.od Running adaptive_chunked_choosers with 380 choosers\r\n", + "INFO - Running chunk 1 of 1 with 380 of 380 choosers\r\n", + "INFO - trip_destination.trip_num_3.social.compute_logsums.dp Running adaptive_chunked_choosers with 380 choosers\r\n", + "INFO - Running chunk 1 of 1 with 380 of 380 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.social.compute_logsums : 1.945 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 2 trips\r\n", + "INFO - trip_destination.trip_num_3.social.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 2 choosers and 380 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 2 of 2 choosers\r\n", + "INFO - Running eval_interaction_utilities on 380 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.social.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.social.trip_destination_simulate : 0.472 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.univ with 10 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.univ.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.univ.sample.interaction_sample Running adaptive_chunked_choosers with 10 choosers\r\n", + "INFO - Running chunk 1 of 1 with 10 of 10 choosers\r\n", + "INFO - Running eval_interaction_utilities on 1900 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.univ.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.univ.trip_destination_sample : 0.075 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.univ.compute_logsums with 1900 samples\r\n", + "INFO - trip_destination.trip_num_3.univ.compute_logsums.od Running adaptive_chunked_choosers with 1900 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1900 of 1900 choosers\r\n", + "INFO - trip_destination.trip_num_3.univ.compute_logsums.dp Running adaptive_chunked_choosers with 1900 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1900 of 1900 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.univ.compute_logsums : 1.925 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 10 trips\r\n", + "INFO - trip_destination.trip_num_3.univ.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 10 choosers and 1900 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 10 of 10 choosers\r\n", + "INFO - Running eval_interaction_utilities on 1900 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.univ.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.univ.trip_destination_simulate : 0.53 seconds (0.0 minutes)\r\n", + "INFO - choose_trip_destination trip_destination.trip_num_3.work with 85 trips\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.work.sample using unsampled alternatives short_circuit_choices\r\n", + "INFO - trip_destination.trip_num_3.work.sample.interaction_sample Running adaptive_chunked_choosers with 85 choosers\r\n", + "INFO - Running chunk 1 of 1 with 85 of 85 choosers\r\n", + "INFO - Running eval_interaction_utilities on 16150 rows\r\n", + "INFO - Estimation mode for trip_destination.trip_num_3.work.sample.interaction_sample using unsampled alternatives\r\n", + "INFO - Time to execute trip_destination.trip_num_3.work.trip_destination_sample : 0.083 seconds (0.0 minutes)\r\n", + "INFO - Running trip_destination.trip_num_3.work.compute_logsums with 16150 samples\r\n", + "INFO - trip_destination.trip_num_3.work.compute_logsums.od Running adaptive_chunked_choosers with 16150 choosers\r\n", + "INFO - Running chunk 1 of 1 with 16150 of 16150 choosers\r\n", + "INFO - trip_destination.trip_num_3.work.compute_logsums.dp Running adaptive_chunked_choosers with 16150 choosers\r\n", + "INFO - Running chunk 1 of 1 with 16150 of 16150 choosers\r\n", + "INFO - Time to execute trip_destination.trip_num_3.work.compute_logsums : 3.316 seconds (0.1 minutes)\r\n", + "DEBUG - trip_destination: write_table cache: choosers\r\n", + "INFO - Running trip_destination_simulate with 85 trips\r\n", + "INFO - trip_destination.trip_num_3.work.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 85 choosers and 16150 alternatives\r\n", + "INFO - Running chunk 1 of 1 with 85 of 85 choosers\r\n", + "INFO - Running eval_interaction_utilities on 16150 rows\r\n", + "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.work.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", + "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", + "DEBUG - trip_destination: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_destination: write_table cache: override_choices\r\n", + "INFO - Time to execute trip_destination.trip_num_3.work.trip_destination_simulate : 0.63 seconds (0.0 minutes)\r\n", + "DEBUG - trip_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", + "DEBUG - trip_destination: write_omnibus_choosers: output/estimation_data_bundle/trip_destination/trip_destination_choosers_combined.csv\r\n", + "DEBUG - trip_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", + "WARNING - trip_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", + "DEBUG - trip_destination: write_omnibus_choosers: output/estimation_data_bundle/trip_destination/trip_destination_alternatives_combined.csv\r\n", + "INFO - trip_destination: end estimation\r\n", + "Time to execute step 'trip_destination': 141.72 s\r\n", + "Total time to execute iteration 1 with iteration value None: 141.72 s\r\n", + "INFO - #run_model running step trip_scheduling\r\n", + "Running step 'trip_scheduling'\r\n", + "INFO - Initialize Estimator for'trip_scheduling'\r\n", + "DEBUG - trip_scheduling: estimate.write_spec: output/estimation_data_bundle/trip_scheduling/trip_scheduling_PROBS_SPEC.csv\r\n", + "DEBUG - trip_scheduling: write_table cache: choosers\r\n", + "INFO - trip_scheduling Running chunk 1 of 1 with 5778 of 5778 choosers\r\n", + "INFO - trip_scheduling.i1 scheduling 14352 trips within chunk 1\r\n", + "INFO - trip_scheduling.i1 236 failed\r\n", + "INFO - trip_scheduling.i2 scheduling 686 trips within chunk 1\r\n", + "INFO - trip_scheduling.i2 388 failed\r\n", + "INFO - trip_scheduling.i3 scheduling 622 trips within chunk 1\r\n", + "INFO - trip_scheduling.i3 376 failed\r\n", + "INFO - trip_scheduling.i4 scheduling 585 trips within chunk 1\r\n", + "INFO - trip_scheduling.i4 373 failed\r\n", + "INFO - trip_scheduling.i5 scheduling 563 trips within chunk 1\r\n", + "INFO - trip_scheduling.i5 370 failed\r\n", + "INFO - trip_scheduling.i6 scheduling 553 trips within chunk 1\r\n", + "INFO - trip_scheduling.i6 367 failed\r\n", + "INFO - trip_scheduling.i7 scheduling 546 trips within chunk 1\r\n", + "INFO - trip_scheduling.i7 367 failed\r\n", + "INFO - trip_scheduling.i8 scheduling 546 trips within chunk 1\r\n", + "INFO - trip_scheduling.i8 366 failed\r\n", + "INFO - trip_scheduling.i9 scheduling 542 trips within chunk 1\r\n", + "INFO - trip_scheduling.i9 365 failed\r\n", + "INFO - trip_scheduling.i10 scheduling 542 trips within chunk 1\r\n", + "INFO - trip_scheduling.i10 366 failed\r\n", + "INFO - trip_scheduling.i11 scheduling 542 trips within chunk 1\r\n", + "INFO - trip_scheduling.i11 363 failed\r\n", + "INFO - trip_scheduling.i12 scheduling 539 trips within chunk 1\r\n", + "INFO - trip_scheduling.i12 366 failed\r\n", + "INFO - trip_scheduling.i13 scheduling 539 trips within chunk 1\r\n", + "INFO - trip_scheduling.i13 364 failed\r\n", + "INFO - trip_scheduling.i14 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i14 361 failed\r\n", + "INFO - trip_scheduling.i15 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i15 363 failed\r\n", + "INFO - trip_scheduling.i16 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i16 364 failed\r\n", + "INFO - trip_scheduling.i17 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i17 365 failed\r\n", + "INFO - trip_scheduling.i18 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i18 363 failed\r\n", + "INFO - trip_scheduling.i19 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i19 363 failed\r\n", + "INFO - trip_scheduling.i20 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i20 362 failed\r\n", + "INFO - trip_scheduling.i21 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i21 365 failed\r\n", + "INFO - trip_scheduling.i22 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i22 363 failed\r\n", + "INFO - trip_scheduling.i23 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i23 365 failed\r\n", + "INFO - trip_scheduling.i24 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i24 365 failed\r\n", + "INFO - trip_scheduling.i25 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i25 363 failed\r\n", + "INFO - trip_scheduling.i26 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i26 364 failed\r\n", + "INFO - trip_scheduling.i27 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i27 365 failed\r\n", + "INFO - trip_scheduling.i28 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i28 365 failed\r\n", + "INFO - trip_scheduling.i29 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i29 363 failed\r\n", + "INFO - trip_scheduling.i30 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i30 363 failed\r\n", + "INFO - trip_scheduling.i31 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i31 365 failed\r\n", + "INFO - trip_scheduling.i32 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i32 364 failed\r\n", + "INFO - trip_scheduling.i33 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i33 364 failed\r\n", + "INFO - trip_scheduling.i34 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i34 364 failed\r\n", + "INFO - trip_scheduling.i35 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i35 363 failed\r\n", + "INFO - trip_scheduling.i36 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i36 363 failed\r\n", + "INFO - trip_scheduling.i37 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i37 364 failed\r\n", + "INFO - trip_scheduling.i38 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i38 364 failed\r\n", + "INFO - trip_scheduling.i39 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i39 364 failed\r\n", + "INFO - trip_scheduling.i40 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i40 365 failed\r\n", + "INFO - trip_scheduling.i41 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i41 365 failed\r\n", + "INFO - trip_scheduling.i42 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i42 365 failed\r\n", + "INFO - trip_scheduling.i43 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i43 366 failed\r\n", + "INFO - trip_scheduling.i44 scheduling 536 trips within chunk 1\r\n", + "INFO - trip_scheduling.i44 363 failed\r\n", + "INFO - trip_scheduling.i45 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i45 363 failed\r\n", + "INFO - trip_scheduling.i46 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i46 362 failed\r\n", + "INFO - trip_scheduling.i47 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i47 363 failed\r\n", + "INFO - trip_scheduling.i48 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i48 364 failed\r\n", + "INFO - trip_scheduling.i49 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i49 362 failed\r\n", + "INFO - trip_scheduling.i50 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i50 363 failed\r\n", + "INFO - trip_scheduling.i51 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i51 363 failed\r\n", + "INFO - trip_scheduling.i52 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i52 363 failed\r\n", + "INFO - trip_scheduling.i53 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i53 363 failed\r\n", + "INFO - trip_scheduling.i54 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i54 362 failed\r\n", + "INFO - trip_scheduling.i55 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i55 363 failed\r\n", + "INFO - trip_scheduling.i56 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i56 364 failed\r\n", + "INFO - trip_scheduling.i57 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i57 364 failed\r\n", + "INFO - trip_scheduling.i58 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i58 363 failed\r\n", + "INFO - trip_scheduling.i59 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i59 364 failed\r\n", + "INFO - trip_scheduling.i60 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i60 362 failed\r\n", + "INFO - trip_scheduling.i61 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i61 365 failed\r\n", + "INFO - trip_scheduling.i62 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i62 361 failed\r\n", + "INFO - trip_scheduling.i63 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i63 362 failed\r\n", + "INFO - trip_scheduling.i64 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i64 364 failed\r\n", + "INFO - trip_scheduling.i65 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i65 363 failed\r\n", + "INFO - trip_scheduling.i66 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i66 364 failed\r\n", + "INFO - trip_scheduling.i67 scheduling 533 trips within chunk 1\r\n", + "INFO - trip_scheduling.i67 363 failed\r\n", + "INFO - trip_scheduling.i68 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i68 364 failed\r\n", + "INFO - trip_scheduling.i69 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i69 362 failed\r\n", + "INFO - trip_scheduling.i70 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i70 360 failed\r\n", + "INFO - trip_scheduling.i71 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i71 361 failed\r\n", + "INFO - trip_scheduling.i72 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i72 363 failed\r\n", + "INFO - trip_scheduling.i73 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i73 363 failed\r\n", + "INFO - trip_scheduling.i74 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i74 363 failed\r\n", + "INFO - trip_scheduling.i75 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i75 362 failed\r\n", + "INFO - trip_scheduling.i76 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i76 362 failed\r\n", + "INFO - trip_scheduling.i77 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i77 363 failed\r\n", + "INFO - trip_scheduling.i78 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i78 361 failed\r\n", + "INFO - trip_scheduling.i79 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i79 363 failed\r\n", + "INFO - trip_scheduling.i80 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i80 361 failed\r\n", + "INFO - trip_scheduling.i81 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i81 363 failed\r\n", + "INFO - trip_scheduling.i82 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i82 362 failed\r\n", + "INFO - trip_scheduling.i83 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i83 363 failed\r\n", + "INFO - trip_scheduling.i84 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i84 363 failed\r\n", + "INFO - trip_scheduling.i85 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i85 362 failed\r\n", + "INFO - trip_scheduling.i86 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i86 358 failed\r\n", + "INFO - trip_scheduling.i87 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i87 361 failed\r\n", + "INFO - trip_scheduling.i88 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i88 361 failed\r\n", + "INFO - trip_scheduling.i89 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i89 362 failed\r\n", + "INFO - trip_scheduling.i90 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i90 362 failed\r\n", + "INFO - trip_scheduling.i91 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i91 362 failed\r\n", + "INFO - trip_scheduling.i92 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i92 362 failed\r\n", + "INFO - trip_scheduling.i93 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i93 362 failed\r\n", + "INFO - trip_scheduling.i94 scheduling 530 trips within chunk 1\r\n", + "INFO - trip_scheduling.i94 361 failed\r\n", + "INFO - trip_scheduling.i95 scheduling 526 trips within chunk 1\r\n", + "INFO - trip_scheduling.i95 360 failed\r\n", + "INFO - trip_scheduling.i96 scheduling 526 trips within chunk 1\r\n", + "INFO - trip_scheduling.i96 359 failed\r\n", + "INFO - trip_scheduling.i97 scheduling 526 trips within chunk 1\r\n", + "INFO - trip_scheduling.i97 359 failed\r\n", + "INFO - trip_scheduling.i98 scheduling 526 trips within chunk 1\r\n", + "INFO - trip_scheduling.i98 359 failed\r\n", + "INFO - trip_scheduling.i99 scheduling 526 trips within chunk 1\r\n", + "INFO - trip_scheduling.i99 359 failed\r\n", + "INFO - trip_scheduling.i100 scheduling 526 trips within chunk 1\r\n", + "INFO - dumping trip_scheduling.i100.outbound.num_2.failed_choosers\r\n", + "WARNING - trip_scheduling.i100.outbound.num_2 coercing 2 depart choices to most initial\r\n", + "INFO - dumping trip_scheduling.i100.outbound.num_3.failed_choosers\r\n", + "WARNING - trip_scheduling.i100.outbound.num_3 coercing 11 depart choices to most initial\r\n", + "INFO - dumping trip_scheduling.i100.outbound.num_4.failed_choosers\r\n", + "WARNING - trip_scheduling.i100.outbound.num_4 coercing 9 depart choices to most initial\r\n", + "INFO - dumping trip_scheduling.i100.inbound.num_1.failed_choosers\r\n", + "WARNING - trip_scheduling.i100.inbound.num_1 coercing 133 depart choices to most initial\r\n", + "INFO - dumping trip_scheduling.i100.inbound.num_2.failed_choosers\r\n", + "WARNING - trip_scheduling.i100.inbound.num_2 coercing 133 depart choices to most initial\r\n", + "INFO - dumping trip_scheduling.i100.inbound.num_3.failed_choosers\r\n", + "WARNING - trip_scheduling.i100.inbound.num_3 coercing 73 depart choices to most initial\r\n", + "INFO - trip_scheduling.i100 0 failed\r\n", + "DEBUG - trip_scheduling: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_scheduling: write_table cache: override_choices\r\n", + "DEBUG - trip_scheduling: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'choosers', 'probs']\r\n", + "WARNING - trip_scheduling: write_omnibus_table: values_combined table 'probs' not found\r\n", + "DEBUG - trip_scheduling: write_omnibus_choosers: output/estimation_data_bundle/trip_scheduling/trip_scheduling_values_combined.csv\r\n", + "INFO - trip_scheduling: end estimation\r\n", + "Time to execute step 'trip_scheduling': 19.54 s\r\n", + "Total time to execute iteration 1 with iteration value None: 19.54 s\r\n", + "INFO - #run_model running step trip_mode_choice\r\n", + "Running step 'trip_mode_choice'\r\n", + "INFO - Running trip_mode_choice with 14352 trips\r\n", + "INFO - primary_purpose top 10 value counts:\r\n", + "work 5428\r\n", + "shopping 1876\r\n", + "othdiscr 1413\r\n", + "school 1081\r\n", + "atwork 1073\r\n", + "othmaint 975\r\n", + "escort 967\r\n", + "eatout 713\r\n", + "social 423\r\n", + "univ 403\r\n", + "Name: primary_purpose, dtype: int64\r\n", + "INFO - Initialize Estimator for'trip_mode_choice'\r\n", + "DEBUG - trip_mode_choice: write_table write: trip_mode_choice_coefficients.csv\r\n", + "DEBUG - trip_mode_choice: write_table write: coefficients_template\r\n", + "DEBUG - trip_mode_choice: estimate.write_spec: output/estimation_data_bundle/trip_mode_choice/trip_mode_choice_SPEC.csv\r\n", + "INFO - trip_mode_choice tour_type 'atwork' (1073 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.atwork.simple_simulate Running adaptive_chunked_choosers with 1073 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1073 of 1073 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'eatout' (713 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.eatout.simple_simulate Running adaptive_chunked_choosers with 713 choosers\r\n", + "INFO - Running chunk 1 of 1 with 713 of 713 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'escort' (967 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.escort.simple_simulate Running adaptive_chunked_choosers with 967 choosers\r\n", + "INFO - Running chunk 1 of 1 with 967 of 967 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'othdiscr' (1413 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.othdiscr.simple_simulate Running adaptive_chunked_choosers with 1413 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1413 of 1413 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'othmaint' (975 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.othmaint.simple_simulate Running adaptive_chunked_choosers with 975 choosers\r\n", + "INFO - Running chunk 1 of 1 with 975 of 975 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'school' (1081 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.school.simple_simulate Running adaptive_chunked_choosers with 1081 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1081 of 1081 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'shopping' (1876 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.shopping.simple_simulate Running adaptive_chunked_choosers with 1876 choosers\r\n", + "INFO - Running chunk 1 of 1 with 1876 of 1876 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'social' (423 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.social.simple_simulate Running adaptive_chunked_choosers with 423 choosers\r\n", + "INFO - Running chunk 1 of 1 with 423 of 423 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'univ' (403 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.univ.simple_simulate Running adaptive_chunked_choosers with 403 choosers\r\n", + "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "INFO - trip_mode_choice tour_type 'work' (5428 trips)\r\n", + "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", + "INFO - trip_mode_choice.work.simple_simulate Running adaptive_chunked_choosers with 5428 choosers\r\n", + "INFO - Running chunk 1 of 1 with 5428 of 5428 choosers\r\n", + "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", + "DEBUG - trip_mode_choice: write_table cache: choices\r\n", + "DEBUG - get_survey_values: reindexing using trips.index\r\n", + "DEBUG - trip_mode_choice: write_table cache: override_choices\r\n", + "DEBUG - trip_mode_choice: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", + "DEBUG - trip_mode_choice: write_omnibus_choosers: output/estimation_data_bundle/trip_mode_choice/trip_mode_choice_values_combined.csv\r\n", + "INFO - trip_mode_choice: end estimation\r\n", + "INFO - trip_modes top 10 value counts:\r\n", + "DRIVEALONEFREE 3864\r\n", + "SHARED2FREE 2283\r\n", + "SHARED3FREE 1921\r\n", + "WALK_LOC 1524\r\n", + "WALK_LRF 1427\r\n", + "WALK 1259\r\n", + "TNC_SINGLE 871\r\n", + "WALK_HVY 473\r\n", + "BIKE 469\r\n", + "TNC_SHARED 149\r\n", + "Name: tour_mode, dtype: int64\r\n", + "INFO - trip_mode_choice choices top 10 value counts:\r\n", + "DRIVEALONEFREE 4317\r\n", + "SHARED2FREE 1872\r\n", + "WALK_LOC 1828\r\n", + "WALK 1621\r\n", + "SHARED3FREE 1345\r\n", + "WALK_LRF 1189\r\n", + "TNC_SINGLE 1038\r\n", + "BIKE 434\r\n", + "TNC_SHARED 361\r\n", + "WALK_HVY 242\r\n", + "Name: trip_mode, dtype: int64\r\n", + "Time to execute step 'trip_mode_choice': 18.31 s\r\n", + "Total time to execute iteration 1 with iteration value None: 18.31 s\r\n", + "INFO - #run_model running step write_tables\r\n", + "Running step 'write_tables'\r\n", + "Time to execute step 'write_tables': 0.41 s\r\n", + "Total time to execute iteration 1 with iteration value None: 0.41 s\r\n", + "INFO - Time to execute run_model (29 models) : 713.015 seconds (11.9 minutes)\r\n", + "INFO - MainProcess high water mark rss: 1_529_487_360 (1.5 GB) timestamp: 09/05/2022 14:57:08 label:pipeline.run_model trip_destination finished\r\n", + "INFO - MainProcess high water mark uss: 0 (0 B) timestamp: 09/05/2022 14:45:55 label:pipeline.run before preload_injectables\r\n", + "INFO - Time to execute all models : 713.145 seconds (11.9 minutes)\r\n" + ] + } + ], + "source": [ + "# run estimation mode TODO: smaller data set?\n", + "!activitysim run -c configs_estimation/configs -c configs -o output -d data_sf" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 22, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "req_data does not request avail_ca or avail_co but it is set and being provided\n", + "problem: chosen-but-not-available (2 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (2 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (1 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (2 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (1 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (2 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (1 issues)\n", + "problem: low-variance-data-co (1 issues)\n", + "problem: chosen-but-not-available (2 issues)\n", + "problem: low-variance-data-co (1 issues)\n" + ] + }, + { + "data": { + "text/plain": "-10094.898223413013" + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "modelname = \"trip_mode_choice\"\n", + "\n", + "from activitysim.estimation.larch import component_model\n", + "model, data = component_model(modelname, return_data=True)\n", + "# data.coefficients\n", + "# data.spec\n", + "# data.chooser_data\n", + "model.load_data()\n", + "model.doctor(repair_ch_av='-')\n", + "model.loglike()\n", + "# DO NOT ESTIMATE just apply probs\n", + "# model.maximize_loglike(method='SLSQP', options={\"maxiter\": 1000})\n", + "probs = [m.probability() for m in model._k_models]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 124, + "outputs": [], + "source": [ + "probs_0 = pd.DataFrame(probs[0])\n", + "m_0 = model._k_models[0]\n", + "\n", + "probs_0.index = m_0.dataframes.data_ch.index" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 110, + "outputs": [ + { + "data": { + "text/plain": "(2124,)" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "(2124,)" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(m_0.dataframes.data_ch.index.unique().shape)\n", + "display(np.intersect1d(data.chooser_data.index.values, m_0.dataframes.data_ch.index.values).shape)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 111, + "outputs": [], + "source": [ + "#data.chooser_data[['override_choice', 'override_choice_code']].drop_duplicates().reset_index(drop=True).set_index\\\n", + "# ('override_choice_code').to_dict()['override_choice']\n", + "mode_map = pd.DataFrame.from_dict(data.settings[\"CONSTANTS\"][\"I_MODE_MAP\"], orient='index', columns=[\"mode_code\"])\\\n", + " .reset_index().rename(columns={\"index\": \"mode_name\"})" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 127, + "outputs": [ + { + "data": { + "text/plain": " trip_id model_choice override_choice \\\ntour_id \n3021985 24175881 DRIVEALONEFREE DRIVEALONEFREE \n3021985 24175885 DRIVEALONEFREE DRIVEALONEFREE \n\n util_DRIVEALONEFREE_Unavailable \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_zero_auto_households \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_persons_less_than_16 \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_joint_tours \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_work \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_In_vehicle_time \\\ntour_id \n3021985 11.50 \n3021985 11.81 \n\n util_DRIVEALONEFREE_Terminal_time ... \\\ntour_id ... \n3021985 4.14944 ... \n3021985 4.14944 ... \n\n drive_heavyrail_available_outbound \\\ntour_id \n3021985 False \n3021985 False \n\n drive_heavyrail_available_inbound drive_commuter_available_outbound \\\ntour_id \n3021985 False False \n3021985 False False \n\n drive_commuter_available_inbound walk_ferry_available \\\ntour_id \n3021985 False False \n3021985 False False \n\n drive_ferry_available distance distance_walk_od distance_bike_od \\\ntour_id \n3021985 False 5.96 5.99 5.99 \n3021985 False 5.99 5.99 5.99 \n\n override_choice_code \ntour_id \n3021985 1 \n3021985 1 \n\n[2 rows x 478 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_idmodel_choiceoverride_choiceutil_DRIVEALONEFREE_Unavailableutil_DRIVEALONEFREE_Unavailable_for_zero_auto_householdsutil_DRIVEALONEFREE_Unavailable_for_persons_less_than_16util_DRIVEALONEFREE_Unavailable_for_joint_toursutil_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_workutil_DRIVEALONEFREE_In_vehicle_timeutil_DRIVEALONEFREE_Terminal_time...drive_heavyrail_available_outbounddrive_heavyrail_available_inbounddrive_commuter_available_outbounddrive_commuter_available_inboundwalk_ferry_availabledrive_ferry_availabledistancedistance_walk_oddistance_bike_odoverride_choice_code
tour_id
302198524175881DRIVEALONEFREEDRIVEALONEFREE0.00.00.00.00.011.504.14944...FalseFalseFalseFalseFalseFalse5.965.995.991
302198524175885DRIVEALONEFREEDRIVEALONEFREE0.00.00.00.00.011.814.14944...FalseFalseFalseFalseFalseFalse5.995.995.991
\n

2 rows × 478 columns

\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": " 1 2 3 4 5 6 7 8 9 10 ... 12 13 \\\n_caseid_ ... \n3021985 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 \n3021985 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 \n\n 14 15 16 17 18 19 20 21 \n_caseid_ \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n\n[2 rows x 21 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
12345678910...12131415161718192021
_caseid_
30219851.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
30219851.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n

2 rows × 21 columns

\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": " 0 1 2 3 4 5 6 7 8 9 ... \\\n_caseid_ ... \n3021985 0.999871 0.0 0.0 0.0 0.0 0.0 0.000006 0.0 0.0 0.0 ... \n3021985 0.999886 0.0 0.0 0.0 0.0 0.0 0.000006 0.0 0.0 0.0 ... \n\n 11 12 13 14 15 16 17 18 19 20 \n_caseid_ \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.368530e-06 0.000021 0.000100 \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.692194e-07 0.000011 0.000096 \n\n[2 rows x 21 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
0123456789...11121314151617181920
_caseid_
30219850.9998710.00.00.00.00.00.0000060.00.00.0...0.00.00.00.00.00.00.01.368530e-060.0000210.000100
30219850.9998860.00.00.00.00.00.0000060.00.00.0...0.00.00.00.00.00.00.03.692194e-070.0000110.000096
\n

2 rows × 21 columns

\n
" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "tour_id_to_check = 3021985\n", + "\n", + "display(data.chooser_data.loc[data.chooser_data.index == tour_id_to_check])\n", + "display(m_0.dataframes.data_ch.loc[m_0.dataframes.data_ch.index == tour_id_to_check])\n", + "display(probs_0.loc[probs_0.index == tour_id_to_check])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## now use my code to calculate probabilities" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 130f805135..84605dcd55 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -5,7 +5,11 @@ "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", "metadata": {}, "source": [ - "# validate results" + "# validate results\n", + "\n", + "## TODO\n", + "what happened to tracing when I fixed probability calcs - it seems like all trip ids are attached when I add one by\n", + "hand below - why?" ] }, { @@ -424,7 +428,7 @@ "\n", "choose_individual_max_utility = True\n", "ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", - "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" + "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" ], "metadata": { "collapsed": false, @@ -449,8 +453,7 @@ "cell_type": "markdown", "source": [ "## try to improve runtime of apply\n", - "\n", - "try caching {nest_name: nest alternatives}\n" + "\n" ], "metadata": { "collapsed": false, @@ -564,6 +567,7 @@ "execution_count": 242, "outputs": [], "source": [ + "%%prun\n", "all_alternatives = set(nest.name for nest in logit.each_nest(nest_spec, type='leaf'))\n", "logit_nest_groups = group_nests_by_name(nest_spec)\n", "nest_alternatives_by_name = {n.name: n.alternatives for n in logit.each_nest(nest_spec)}\n", From e02b46a8c93451edab1c3d5c44bdb6b5258b38e8 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 10 May 2022 00:35:23 +1000 Subject: [PATCH 023/135] compare to larch probs --- notebooks/check_sf_probs.ipynb | 281 ++ notebooks/fru_utils.py | 72 +- .../larch_probabilities_via_estimation.ipynb | 2848 +---------------- notebooks/validate_frozen_impl.ipynb | 173 +- 4 files changed, 530 insertions(+), 2844 deletions(-) create mode 100644 notebooks/check_sf_probs.ipynb diff --git a/notebooks/check_sf_probs.ipynb b/notebooks/check_sf_probs.ipynb new file mode 100644 index 0000000000..de5d82d521 --- /dev/null +++ b/notebooks/check_sf_probs.ipynb @@ -0,0 +1,281 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", + "metadata": {}, + "source": [ + "# Compare results against larch\n", + "\n", + "We use estimation data as prepared by larch and draw the Asim way and our way, then compare to larch probabilities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.385153Z", + "iopub.status.busy": "2022-05-01T10:17:57.384881Z", + "iopub.status.idle": "2022-05-01T10:17:57.534433Z", + "shell.execute_reply": "2022-05-01T10:17:57.533096Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3dba451-1e10-403e-8614-35d57e6577f4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.536623Z", + "iopub.status.busy": "2022-05-01T10:17:57.536012Z", + "iopub.status.idle": "2022-05-01T10:17:57.542755Z", + "shell.execute_reply": "2022-05-01T10:17:57.541685Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:00.350944Z", + "iopub.status.busy": "2022-05-01T10:18:00.350730Z", + "iopub.status.idle": "2022-05-01T10:18:12.760977Z", + "shell.execute_reply": "2022-05-01T10:18:12.760013Z", + "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from activitysim.cli import run\n", + "from activitysim.core import inject\n", + "\n", + "from fru_utils import mode_choice_for_trip\n", + "\n", + "pd.set_option(\"max_columns\", 500)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:17.839947Z", + "iopub.status.busy": "2022-05-01T10:18:17.839070Z", + "iopub.status.idle": "2022-05-01T10:18:18.019676Z", + "shell.execute_reply": "2022-05-01T10:18:18.018689Z", + "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_example_sf\")\n", + "os.chdir(example_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "outputs": [], + "source": [ + "#!activitysim run -c configs -o output -d data_sf" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:22.111723Z", + "iopub.status.busy": "2022-05-01T10:18:22.111490Z", + "iopub.status.idle": "2022-05-01T10:18:22.297437Z", + "shell.execute_reply": "2022-05-01T10:18:22.296501Z", + "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "run.add_run_args(parser)\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data_sf'])\n", + "if not inject.is_injectable('preload_injectables'):\n", + " from activitysim import abm # register abm steps and other abm-specific injectables\n", + "run.handle_standard_args(args) # possibly update injectables" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "outputs": [], + "source": [ + "larch_chooser_data = pd.read_csv(os.path.join(root_dir, \"test_estimation_sf\", \"choosers_larch.csv\"))\n", + "larch_probs = pd.read_csv(os.path.join(root_dir, \"test_estimation_sf\", \"probabilities_larch.csv\"), index_col=0)\n", + "# __caseids__ are tour ids, with trips in same order\n", + "assert (larch_probs.index.values == larch_chooser_data.tour_id).all()\n", + "larch_probs.index = larch_chooser_data.trip_id" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 50, + "outputs": [], + "source": [ + "larch_tours = pd.read_csv(os.path.join(root_dir, \"test_estimation_sf\", \"output\", \"estimation_data_bundle\",\n", + " \"trip_mode_choice\", \"trip_mode_choice_values_combined.csv\"))\n", + "trips_merged_cols = ['person_id', 'household_id', 'primary_purpose', 'trip_num', 'outbound',\n", + " 'trip_count', 'destination', 'origin', 'tour_id', 'purpose',\n", + " 'destination_logsum', 'depart', 'hhsize', 'age', 'auto_ownership',\n", + " 'number_of_participants', 'tour_category', 'parent_tour_id',\n", + " 'tour_mode', 'duration', 'value_of_time', 'tour_type',\n", + " 'free_parking_at_work', 'trip_period', \"trip_id\"]\n", + "\n", + "larch_trips_merged = larch_tours[trips_merged_cols].set_index(\"trip_id\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 87, + "outputs": [], + "source": [ + "def compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs):\n", + " t_, c_, ns_ = mode_choice_for_trip(choose_individual_max_utility=True, trip_id_to_check=trip_id_to_check,\n", + " num_samples=num_samples, trips_merged=larch_trips_merged)\n", + " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", + "\n", + " t_asim, c_asim, _ = mode_choice_for_trip(choose_individual_max_utility=False, trip_id_to_check=trip_id_to_check,\n", + " num_samples=num_samples, trips_merged=larch_trips_merged)\n", + " asim_mode_shares = c_asim.trip_mode.value_counts() / c_asim.shape[0]\n", + "\n", + " larch_mode_shares = larch_probs.loc[larch_probs.index == trip_id_to_check].T\n", + " larch_mode_shares.columns = [\"larch\"]\n", + "\n", + " comp = sim_mode_shares.to_frame(\"me\")\\\n", + " .merge(asim_mode_shares.to_frame(\"asim\"), left_index=True, right_index=True,how=\"outer\")\\\n", + " .merge(larch_mode_shares,left_index=True, right_index=True, how=\"outer\").fillna(0)\n", + "\n", + " return comp" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'work' (10000 trips)\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "trip_id_to_check = 23695361\n", + "num_samples = 10000\n", + "\n", + "comp = compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs)\n", + "comp.loc[(comp != 0).any(axis=1), :].applymap(lambda x: f\"{x:.3%}\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index 0f13679b52..d466184af0 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -17,7 +17,9 @@ -def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples): + + +def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples, trips_merged=None): """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" resume_after = "trip_scheduling" model_name = "trip_mode_choice" @@ -32,8 +34,6 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa #checkpoint = pipeline.intermediate_checkpoint(model_name) inject.set_step_args(args) - trips = inject.get_table('trips') - tours_merged = inject.get_table('tours_merged') network_los = inject.get_injectable('network_los') trace_label = 'trip_mode_choice' @@ -43,24 +43,21 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') mode_column_name = 'trip_mode' - trips_df = trips.to_frame() - print("Running with %d trips", trips_df.shape[0]) - - tours_merged = tours_merged.to_frame() - tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] - - # - trips_merged - merge trips and tours_merged - trips_merged = pd.merge( - trips_df, - tours_merged, - left_on='tour_id', - right_index=True, - how="left") - assert trips_merged.index.equals(trips.index) - - # setup skim keys - assert ('trip_period' not in trips_merged) - trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) + if trips_merged is None: + trips = inject.get_table('trips') + tours_merged = inject.get_table('tours_merged') + trips_df = trips.to_frame() + tours_merged = tours_merged.to_frame() + tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] + trips_merged = pd.merge( + trips_df, + tours_merged, + left_on='tour_id', + right_index=True, + how="left") + assert trips_merged.index.equals(trips.index) + assert ('trip_period' not in trips_merged) + trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) orig_col = 'origin' dest_col = 'destination' @@ -91,9 +88,6 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa estimator = estimation.manager.begin_estimation('trip_mode_choice') - #choices_list = [] - - # grab one, duplicate num_samples times trips_segment = trips_merged.loc[trips_merged.index == trip_id_to_check].copy() primary_purpose = trips_segment['primary_purpose'].values[0] @@ -103,23 +97,12 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa existing_indexes = pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values num_new_indexes = trips_segment.shape[0] new_indexes = np.arange(existing_indexes.max()+1, existing_indexes.max() + num_new_indexes + 1) - - trips_segment.index = new_indexes #+= np.arange(num_samples) + trips_segment.index = new_indexes # name index so tracing knows how to slice trips_segment.index.name = 'trip_id' - - # #logger.warning("Change seeding back when done with testing") - # pipeline._PIPELINE.rng.row_states = pd.DataFrame(columns=['row_seed', 'offset'], index=trips_segment.index) - # pipeline._PIPELINE.rng.row_states["row_seed"] = trips_segment.index.values - # pipeline._PIPELINE.rng.row_states["offset"] = 0 pipeline._PIPELINE.rng().add_channel("trips", trips_segment) - - #for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'): - #if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes): - # continue - print("trip_mode_choice tour_type '%s' (%s trips)" % (primary_purpose, len(trips_segment.index), )) @@ -163,18 +146,6 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa if not choose_individual_max_utility: alts = spec.columns choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) - ################ - #choices_list.append(choices) - #choices_df_asim = pd.concat(choices_list) - - # update trips table with choices (and potionally logssums) - #trips_df = trips_merged.copy() # trips.to_frame() - - #if (do_these_purposes is not None): - # trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)] - - #assign_in_place(trips_df, choices) - #assert not trips_df[mode_column_name].isnull().any() finalise = True if finalise: @@ -188,11 +159,11 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa pipeline.close_pipeline() print("Done") - return trips_merged, choices + return trips_merged, choices, nest_spec def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check): - t_, c_ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, + t_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, trip_id_to_check=trip_id_to_check, num_samples=num_samples) sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0] @@ -208,7 +179,6 @@ def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, tri return ms_comp - def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True): """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" diff --git a/notebooks/larch_probabilities_via_estimation.ipynb b/notebooks/larch_probabilities_via_estimation.ipynb index 0bb5f0a04d..db362e994c 100644 --- a/notebooks/larch_probabilities_via_estimation.ipynb +++ b/notebooks/larch_probabilities_via_estimation.ipynb @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 1, "outputs": [], "source": [ "import os\n", @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 2, "outputs": [], "source": [ "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 3, "outputs": [], "source": [ "# create estimation test example\n", @@ -78,2745 +78,11 @@ }, { "cell_type": "code", - "execution_count": 13, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Configured logging using basicConfig\r\n", - "INFO:activitysim:Configured logging using basicConfig\r\n", - "INFO - Read logging configuration from: configs_estimation/configs/logging.yaml\r\n", - "INFO - SETTING configs_dir: ['configs_estimation/configs', 'configs']\r\n", - "INFO - SETTING settings_file_name: settings.yaml\r\n", - "INFO - SETTING data_dir: ['data_sf']\r\n", - "INFO - SETTING output_dir: output\r\n", - "INFO - SETTING households_sample_size: 0\r\n", - "INFO - SETTING chunk_size: 0\r\n", - "INFO - SETTING chunk_method: hybrid_uss\r\n", - "INFO - SETTING chunk_training_mode: disabled\r\n", - "INFO - SETTING multiprocess: None\r\n", - "INFO - SETTING num_processes: None\r\n", - "INFO - SETTING resume_after: None\r\n", - "INFO - SETTING trace_hh_id: None\r\n", - "INFO - ENV MKL_NUM_THREADS: None\r\n", - "INFO - ENV OMP_NUM_THREADS: None\r\n", - "INFO - ENV OPENBLAS_NUM_THREADS: None\r\n", - "INFO - NUMPY blas_opt_info libraries: ['openblas', 'openblas']\r\n", - "INFO - NUMPY lapack_opt_info libraries: ['openblas', 'openblas']\r\n", - "INFO - run single process simulation\r\n", - "INFO - Time to execute open_pipeline : 0.029 seconds (0.0 minutes)\r\n", - "INFO - preload_injectables\r\n", - "INFO - Time to execute preload_injectables : 0.036 seconds (0.0 minutes)\r\n", - "INFO - #run_model running step initialize_landuse\r\n", - "Running step 'initialize_landuse'\r\n", - "INFO - Reading CSV file data_sf/land_use.csv\r\n", - "INFO - loaded land_use (190, 24)\r\n", - "INFO - initialize_landuse.annotate_tables - annotating land_use SPEC annotate_landuse\r\n", - "INFO - Network_LOS using skim_dict_factory: NumpyArraySkimFactory\r\n", - "INFO - allocate_skim_buffer shared False taz shape (826, 190, 190) total size: 119_274_400 (119.3 MB)\r\n", - "INFO - _read_skims_from_omx data_sf/skims.omx\r\n", - "INFO - _read_skims_from_omx loaded 826 skims from data_sf/skims.omx\r\n", - "INFO - writing skim cache taz (826, 190, 190) to output/cache/cached_taz.mmap\r\n", - "INFO - load_skims_to_buffer taz shape (826, 190, 190)\r\n", - "INFO - get_skim_data taz SkimData shape (826, 190, 190)\r\n", - "INFO - SkimDict init taz\r\n", - "INFO - SkimDict.build_3d_skim_block_offset_table registered 167 3d keys\r\n", - "Time to execute step 'initialize_landuse': 3.69 s\r\n", - "Total time to execute iteration 1 with iteration value None: 3.69 s\r\n", - "INFO - #run_model running step initialize_households\r\n", - "Running step 'initialize_households'\r\n", - "INFO - Reading CSV file data_sf/survey_data/override_households.csv\r\n", - "INFO - full household list contains 2000 households\r\n", - "INFO - loaded households (2000, 7)\r\n", - "INFO - Reading CSV file data_sf/survey_data/override_persons.csv\r\n", - "INFO - loaded persons (4405, 7)\r\n", - "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons\r\n", - "INFO - initialize_households.annotate_tables - annotating households SPEC annotate_households\r\n", - "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons_after_hh\r\n", - "Time to execute step 'initialize_households': 0.39 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.39 s\r\n", - "INFO - #run_model running step compute_accessibility\r\n", - "Running step 'compute_accessibility'\r\n", - "INFO - Running compute_accessibility with 190 orig zones 190 dest zones\r\n", - "INFO - compute_accessibility Running adaptive_chunked_choosers with 190 choosers\r\n", - "INFO - Running chunk 1 of 1 with 190 of 190 choosers\r\n", - "INFO - Running compute_accessibility with 190 orig zones 190 dest zones\r\n", - "INFO - compute_accessibility computed accessibilities (190, 10)\r\n", - "Time to execute step 'compute_accessibility': 0.12 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.12 s\r\n", - "INFO - #run_model running step school_location\r\n", - "Running step 'school_location'\r\n", - "INFO - Initialize Estimator for'school_location'\r\n", - "DEBUG - school_location: estimate.write_spec: output/estimation_data_bundle/school_location/school_location_SPEC.csv\r\n", - "DEBUG - school_location: write_table write: school_location_coefficients.csv\r\n", - "DEBUG - school_location: write_table write: size_terms\r\n", - "DEBUG - school_location: write_table write: landuse\r\n", - "INFO - Running school_location.i1.sample.university with 329 persons\r\n", - "INFO - Estimation mode for school_location.i1.sample.university using unsampled alternatives short_circuit_choices\r\n", - "INFO - school_location.i1.sample.university.interaction_sample Running adaptive_chunked_choosers with 329 choosers\r\n", - "INFO - Running chunk 1 of 1 with 329 of 329 choosers\r\n", - "INFO - Running eval_interaction_utilities on 5593 rows\r\n", - "INFO - Estimation mode for school_location.i1.sample.university.interaction_sample using unsampled alternatives\r\n", - "INFO - Running school_location.i1.logsums.university with 5593 rows\r\n", - "INFO - school_location.i1.logsums.university.compute_logsums Running adaptive_chunked_choosers with 5593 choosers\r\n", - "INFO - Running chunk 1 of 1 with 5593 of 5593 choosers\r\n", - "INFO - Running school_location.i1.simulate.university with 329 persons\r\n", - "DEBUG - school_location: write_table cache: choosers\r\n", - "DEBUG - school_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - school_location.i1.simulate.university.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 329 choosers and 5593 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 329 of 329 choosers\r\n", - "INFO - Running eval_interaction_utilities on 5593 rows\r\n", - "INFO - school_location: eval_interaction_utilities write_interaction_expression_values school_location.i1.simulate.university.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - school_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - school_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - school_location: write_table cache: override_choices\r\n", - "INFO - Running school_location.i1.sample.highschool with 111 persons\r\n", - "INFO - Estimation mode for school_location.i1.sample.highschool using unsampled alternatives short_circuit_choices\r\n", - "INFO - school_location.i1.sample.highschool.interaction_sample Running adaptive_chunked_choosers with 111 choosers\r\n", - "INFO - Running chunk 1 of 1 with 111 of 111 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2775 rows\r\n", - "INFO - Estimation mode for school_location.i1.sample.highschool.interaction_sample using unsampled alternatives\r\n", - "INFO - Running school_location.i1.logsums.highschool with 2775 rows\r\n", - "INFO - school_location.i1.logsums.highschool.compute_logsums Running adaptive_chunked_choosers with 2775 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2775 of 2775 choosers\r\n", - "INFO - Running school_location.i1.simulate.highschool with 111 persons\r\n", - "DEBUG - school_location: write_table cache: choosers\r\n", - "DEBUG - school_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - school_location.i1.simulate.highschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 111 choosers and 2775 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 111 of 111 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2775 rows\r\n", - "INFO - school_location: eval_interaction_utilities write_interaction_expression_values school_location.i1.simulate.highschool.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - school_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - school_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - school_location: write_table cache: override_choices\r\n", - "INFO - Running school_location.i1.sample.gradeschool with 544 persons\r\n", - "INFO - Estimation mode for school_location.i1.sample.gradeschool using unsampled alternatives short_circuit_choices\r\n", - "INFO - school_location.i1.sample.gradeschool.interaction_sample Running adaptive_chunked_choosers with 544 choosers\r\n", - "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", - "INFO - Running eval_interaction_utilities on 103360 rows\r\n", - "INFO - Estimation mode for school_location.i1.sample.gradeschool.interaction_sample using unsampled alternatives\r\n", - "INFO - Running school_location.i1.logsums.gradeschool with 103360 rows\r\n", - "INFO - school_location.i1.logsums.gradeschool.compute_logsums Running adaptive_chunked_choosers with 103360 choosers\r\n", - "INFO - Running chunk 1 of 1 with 103360 of 103360 choosers\r\n", - "INFO - Running school_location.i1.simulate.gradeschool with 544 persons\r\n", - "DEBUG - school_location: write_table cache: choosers\r\n", - "DEBUG - school_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - school_location.i1.simulate.gradeschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 544 choosers and 103360 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", - "INFO - Running eval_interaction_utilities on 103360 rows\r\n", - "INFO - school_location: eval_interaction_utilities write_interaction_expression_values school_location.i1.simulate.gradeschool.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - school_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - school_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - school_location: write_table cache: override_choices\r\n", - "INFO - write_trace_files iteration 1\r\n", - "INFO - school_location_logsum top 10 value counts:\r\n", - "-1.824183 2\r\n", - "-0.909839 2\r\n", - "-1.366984 2\r\n", - "-1.278060 1\r\n", - "-1.180527 1\r\n", - "-1.674889 1\r\n", - "-1.291101 1\r\n", - "-1.813244 1\r\n", - "-1.069701 1\r\n", - " 0.053658 1\r\n", - "Name: logsum, dtype: int64\r\n", - "DEBUG - school_location: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - school_location: write_omnibus_choosers: output/estimation_data_bundle/school_location/school_location_choosers_combined.csv\r\n", - "DEBUG - school_location: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - school_location: write_omnibus_choosers: output/estimation_data_bundle/school_location/school_location_alternatives_combined.csv\r\n", - "INFO - school_location: end estimation\r\n", - "Time to execute step 'school_location': 15.20 s\r\n", - "Total time to execute iteration 1 with iteration value None: 15.21 s\r\n", - "INFO - #run_model running step workplace_location\r\n", - "Running step 'workplace_location'\r\n", - "INFO - Initialize Estimator for'workplace_location'\r\n", - "DEBUG - workplace_location: estimate.write_spec: output/estimation_data_bundle/workplace_location/workplace_location_SPEC.csv\r\n", - "DEBUG - workplace_location: write_table write: workplace_location_coefficients.csv\r\n", - "DEBUG - workplace_location: write_table write: size_terms\r\n", - "DEBUG - workplace_location: write_table write: landuse\r\n", - "INFO - Running workplace_location.i1.sample.work_low with 518 persons\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_low using unsampled alternatives short_circuit_choices\r\n", - "INFO - workplace_location.i1.sample.work_low.interaction_sample Running adaptive_chunked_choosers with 518 choosers\r\n", - "INFO - Running chunk 1 of 1 with 518 of 518 choosers\r\n", - "INFO - Running eval_interaction_utilities on 98420 rows\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_low.interaction_sample using unsampled alternatives\r\n", - "INFO - Running workplace_location.i1.logsums.work_low with 98420 rows\r\n", - "INFO - workplace_location.i1.logsums.work_low.compute_logsums Running adaptive_chunked_choosers with 98420 choosers\r\n", - "INFO - Running chunk 1 of 1 with 98420 of 98420 choosers\r\n", - "INFO - Running workplace_location.i1.simulate.work_low with 518 persons\r\n", - "DEBUG - workplace_location: write_table cache: choosers\r\n", - "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - workplace_location.i1.simulate.work_low.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 518 choosers and 98420 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 518 of 518 choosers\r\n", - "INFO - Running eval_interaction_utilities on 98420 rows\r\n", - "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_low.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - workplace_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - workplace_location: write_table cache: override_choices\r\n", - "INFO - Running workplace_location.i1.sample.work_med with 545 persons\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_med using unsampled alternatives short_circuit_choices\r\n", - "INFO - workplace_location.i1.sample.work_med.interaction_sample Running adaptive_chunked_choosers with 545 choosers\r\n", - "INFO - Running chunk 1 of 1 with 545 of 545 choosers\r\n", - "INFO - Running eval_interaction_utilities on 103550 rows\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_med.interaction_sample using unsampled alternatives\r\n", - "INFO - Running workplace_location.i1.logsums.work_med with 103550 rows\r\n", - "INFO - workplace_location.i1.logsums.work_med.compute_logsums Running adaptive_chunked_choosers with 103550 choosers\r\n", - "INFO - Running chunk 1 of 1 with 103550 of 103550 choosers\r\n", - "INFO - Running workplace_location.i1.simulate.work_med with 545 persons\r\n", - "DEBUG - workplace_location: write_table cache: choosers\r\n", - "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - workplace_location.i1.simulate.work_med.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 545 choosers and 103550 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 545 of 545 choosers\r\n", - "INFO - Running eval_interaction_utilities on 103550 rows\r\n", - "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_med.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - workplace_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - workplace_location: write_table cache: override_choices\r\n", - "INFO - Running workplace_location.i1.sample.work_high with 614 persons\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_high using unsampled alternatives short_circuit_choices\r\n", - "INFO - workplace_location.i1.sample.work_high.interaction_sample Running adaptive_chunked_choosers with 614 choosers\r\n", - "INFO - Running chunk 1 of 1 with 614 of 614 choosers\r\n", - "INFO - Running eval_interaction_utilities on 116660 rows\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_high.interaction_sample using unsampled alternatives\r\n", - "INFO - Running workplace_location.i1.logsums.work_high with 116660 rows\r\n", - "INFO - workplace_location.i1.logsums.work_high.compute_logsums Running adaptive_chunked_choosers with 116660 choosers\r\n", - "INFO - Running chunk 1 of 1 with 116660 of 116660 choosers\r\n", - "INFO - Running workplace_location.i1.simulate.work_high with 614 persons\r\n", - "DEBUG - workplace_location: write_table cache: choosers\r\n", - "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - workplace_location.i1.simulate.work_high.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 614 choosers and 116660 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 614 of 614 choosers\r\n", - "INFO - Running eval_interaction_utilities on 116660 rows\r\n", - "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_high.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - workplace_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - workplace_location: write_table cache: override_choices\r\n", - "INFO - Running workplace_location.i1.sample.work_veryhigh with 905 persons\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_veryhigh using unsampled alternatives short_circuit_choices\r\n", - "INFO - workplace_location.i1.sample.work_veryhigh.interaction_sample Running adaptive_chunked_choosers with 905 choosers\r\n", - "INFO - Running chunk 1 of 1 with 905 of 905 choosers\r\n", - "INFO - Running eval_interaction_utilities on 171950 rows\r\n", - "INFO - Estimation mode for workplace_location.i1.sample.work_veryhigh.interaction_sample using unsampled alternatives\r\n", - "INFO - Running workplace_location.i1.logsums.work_veryhigh with 171950 rows\r\n", - "INFO - workplace_location.i1.logsums.work_veryhigh.compute_logsums Running adaptive_chunked_choosers with 171950 choosers\r\n", - "INFO - Running chunk 1 of 1 with 171950 of 171950 choosers\r\n", - "INFO - Running workplace_location.i1.simulate.work_veryhigh with 905 persons\r\n", - "DEBUG - workplace_location: write_table cache: choosers\r\n", - "DEBUG - workplace_location: write_table cache: interaction_sample_alternatives\r\n", - "INFO - workplace_location.i1.simulate.work_veryhigh.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 905 choosers and 171950 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 905 of 905 choosers\r\n", - "INFO - Running eval_interaction_utilities on 171950 rows\r\n", - "INFO - workplace_location: eval_interaction_utilities write_interaction_expression_values workplace_location.i1.simulate.work_veryhigh.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - workplace_location: write_table cache: interaction_expression_values\r\n", - "DEBUG - workplace_location: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - workplace_location: write_table cache: override_choices\r\n", - "INFO - write_trace_files iteration 1\r\n", - "INFO - workplace_location_logsum top 10 value counts:\r\n", - "-0.007794 2\r\n", - " 1.586426 2\r\n", - "-1.938350 2\r\n", - " 0.491470 2\r\n", - " 0.267120 2\r\n", - " 0.047726 2\r\n", - " 0.657742 1\r\n", - "-0.051882 1\r\n", - " 6.017921 1\r\n", - "-0.325755 1\r\n", - "Name: logsum, dtype: int64\r\n", - "DEBUG - workplace_location: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - workplace_location: write_omnibus_choosers: output/estimation_data_bundle/workplace_location/workplace_location_choosers_combined.csv\r\n", - "DEBUG - workplace_location: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - workplace_location: write_omnibus_choosers: output/estimation_data_bundle/workplace_location/workplace_location_alternatives_combined.csv\r\n", - "INFO - workplace_location: end estimation\r\n", - "Time to execute step 'workplace_location': 51.96 s\r\n", - "Total time to execute iteration 1 with iteration value None: 51.96 s\r\n", - "INFO - #run_model running step auto_ownership_simulate\r\n", - "Running step 'auto_ownership_simulate'\r\n", - "INFO - Initialize Estimator for'auto_ownership'\r\n", - "INFO - Running auto_ownership_simulate with 2000 households\r\n", - "DEBUG - auto_ownership: estimate.write_spec: output/estimation_data_bundle/auto_ownership/auto_ownership_SPEC.csv\r\n", - "DEBUG - auto_ownership: write_table write: auto_ownership_coefficients.csv\r\n", - "DEBUG - auto_ownership: write_table cache: choosers\r\n", - "INFO - auto_ownership_simulate.simple_simulate Running adaptive_chunked_choosers with 2000 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2000 of 2000 choosers\r\n", - "DEBUG - auto_ownership: write_table cache: expression_values\r\n", - "DEBUG - auto_ownership: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using households.index\r\n", - "DEBUG - auto_ownership: write_table cache: override_choices\r\n", - "DEBUG - auto_ownership: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - auto_ownership: write_omnibus_choosers: output/estimation_data_bundle/auto_ownership/auto_ownership_values_combined.csv\r\n", - "INFO - auto_ownership: end estimation\r\n", - "INFO - auto_ownership top 10 value counts:\r\n", - "1 883\r\n", - "0 615\r\n", - "2 390\r\n", - "3 74\r\n", - "4 38\r\n", - "Name: auto_ownership, dtype: int64\r\n", - "Time to execute step 'auto_ownership_simulate': 0.41 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.41 s\r\n", - "INFO - #run_model running step free_parking\r\n", - "Running step 'free_parking'\r\n", - "INFO - Running free_parking with 2582 persons\r\n", - "INFO - Initialize Estimator for'free_parking'\r\n", - "DEBUG - free_parking: estimate.write_spec: output/estimation_data_bundle/free_parking/free_parking_SPEC.csv\r\n", - "DEBUG - free_parking: write_table write: free_parking_coefficients.csv\r\n", - "DEBUG - free_parking: write_table cache: choosers\r\n", - "INFO - free_parking.simple_simulate Running adaptive_chunked_choosers with 2582 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2582 of 2582 choosers\r\n", - "DEBUG - free_parking: write_table cache: expression_values\r\n", - "DEBUG - free_parking: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - free_parking: write_table cache: override_choices\r\n", - "DEBUG - free_parking: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - free_parking: write_omnibus_choosers: output/estimation_data_bundle/free_parking/free_parking_values_combined.csv\r\n", - "INFO - free_parking: end estimation\r\n", - "INFO - free_parking top 10 value counts:\r\n", - "False 4272\r\n", - "True 133\r\n", - "Name: free_parking_at_work, dtype: int64\r\n", - "Time to execute step 'free_parking': 0.31 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.31 s\r\n", - "INFO - #run_model running step cdap_simulate\r\n", - "Running step 'cdap_simulate'\r\n", - "INFO - Initialize Estimator for'cdap'\r\n", - "INFO - Pre-building cdap specs\r\n", - "INFO - Time to execute build_cdap_spec hh_size 2 : 0.214 seconds (0.0 minutes)\r\n", - "INFO - Time to execute build_cdap_spec hh_size 3 : 0.5 seconds (0.0 minutes)\r\n", - "INFO - Time to execute build_cdap_spec hh_size 4 : 1.268 seconds (0.0 minutes)\r\n", - "INFO - Time to execute build_cdap_spec hh_size 5 : 3.119 seconds (0.1 minutes)\r\n", - "DEBUG - cdap: estimate.write_spec: output/estimation_data_bundle/cdap/cdap_INDIV_AND_HHSIZE1_SPEC.csv\r\n", - "DEBUG - cdap: estimate.write_spec: output/estimation_data_bundle/cdap/cdap_FIXED_RELATIVE_PROPORTIONS_SPEC.csv\r\n", - "DEBUG - cdap: write_table write: cdap_coefficients.csv\r\n", - "DEBUG - cdap: write_table write: interaction_coefficients\r\n", - "DEBUG - cdap: write_table cache: choosers\r\n", - "DEBUG - cdap: write_table write: spec_2\r\n", - "DEBUG - cdap: write_table write: spec_3\r\n", - "DEBUG - cdap: write_table write: spec_4\r\n", - "DEBUG - cdap: write_table write: spec_5\r\n", - "INFO - Running cdap_simulate with 4405 persons\r\n", - "INFO - cdap.cdap Running chunk 1 of 1 with 2000 of 2000 choosers\r\n", - "DEBUG - cdap: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - cdap: write_table cache: override_choices\r\n", - "DEBUG - cdap: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - cdap: write_omnibus_choosers: output/estimation_data_bundle/cdap/cdap_values_combined.csv\r\n", - "INFO - cdap: end estimation\r\n", - "INFO - cdap_activity top 10 value counts:\r\n", - "M 2620\r\n", - "N 1213\r\n", - "H 572\r\n", - "Name: cdap_activity, dtype: int64\r\n", - "INFO - cdap crosstabs:\r\n", - "cdap_activity H M N All\r\n", - "ptype \r\n", - "1 156 1606 158 1920\r\n", - "2 47 331 118 496\r\n", - "3 33 230 66 329\r\n", - "4 114 0 420 534\r\n", - "5 114 0 357 471\r\n", - "6 22 42 8 72\r\n", - "7 39 274 39 352\r\n", - "8 47 137 47 231\r\n", - "All 572 2620 1213 4405\r\n", - "Time to execute step 'cdap_simulate': 9.84 s\r\n", - "Total time to execute iteration 1 with iteration value None: 9.84 s\r\n", - "INFO - #run_model running step mandatory_tour_frequency\r\n", - "Running step 'mandatory_tour_frequency'\r\n", - "INFO - Running mandatory_tour_frequency with 2620 persons\r\n", - "INFO - Initialize Estimator for'mandatory_tour_frequency'\r\n", - "DEBUG - mandatory_tour_frequency: estimate.write_spec: output/estimation_data_bundle/mandatory_tour_frequency/mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - mandatory_tour_frequency: write_table write: mandatory_tour_frequency_coefficients.csv\r\n", - "DEBUG - mandatory_tour_frequency: write_table cache: choosers\r\n", - "INFO - mandatory_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 2620 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2620 of 2620 choosers\r\n", - "DEBUG - mandatory_tour_frequency: write_table cache: expression_values\r\n", - "DEBUG - mandatory_tour_frequency: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - mandatory_tour_frequency: write_table cache: override_choices\r\n", - "DEBUG - mandatory_tour_frequency: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - mandatory_tour_frequency: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_frequency/mandatory_tour_frequency_values_combined.csv\r\n", - "INFO - mandatory_tour_frequency: end estimation\r\n", - "INFO - mandatory_tour_frequency top 10 value counts:\r\n", - "work1 1941\r\n", - " 1785\r\n", - "school1 566\r\n", - "work2 81\r\n", - "work_and_school 21\r\n", - "school2 11\r\n", - "Name: mandatory_tour_frequency, dtype: int64\r\n", - "Time to execute step 'mandatory_tour_frequency': 1.21 s\r\n", - "Total time to execute iteration 1 with iteration value None: 1.21 s\r\n", - "INFO - #run_model running step mandatory_tour_scheduling\r\n", - "Running step 'mandatory_tour_scheduling'\r\n", - "DEBUG - @inject timetable\r\n", - "INFO - Initialize Estimator for'mandatory_tour_scheduling_work'\r\n", - "DEBUG - mandatory_tour_scheduling_work: estimate.write_spec: output/estimation_data_bundle/mandatory_tour_scheduling_work/mandatory_tour_scheduling_work_SPEC.csv\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table write: tour_scheduling_work_coefficients.csv\r\n", - "INFO - Initialize Estimator for'mandatory_tour_scheduling_school'\r\n", - "DEBUG - mandatory_tour_scheduling_school: estimate.write_spec: output/estimation_data_bundle/mandatory_tour_scheduling_school/mandatory_tour_scheduling_school_SPEC.csv\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table write: tour_scheduling_school_coefficients.csv\r\n", - "INFO - mandatory_tour_scheduling_work: timetable.begin_transaction person_windows\r\n", - "INFO - mandatory_tour_scheduling_school: timetable.begin_transaction person_windows\r\n", - "INFO - Running mandatory_tour_scheduling with 2733 tours\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 2043 tour choices\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work Running adaptive_chunked_choosers with 2043 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2043 of 2043 choosers\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 2043 tour choices\r\n", - "INFO - tdd_alt_segments specified for representative logsums\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 388170 to 30645 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums compute_logsums for 30645 choosers 30645 alts\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums Running adaptive_chunked_choosers with 30645 choosers\r\n", - "INFO - Running chunk 1 of 1 with 30645 of 30645 choosers\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: choosers\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_sample_alternatives\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 2043 choosers and 388170 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 2043 of 2043 choosers\r\n", - "INFO - Running eval_interaction_utilities on 388170 rows\r\n", - "INFO - mandatory_tour_scheduling_work: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_expression_values\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 451 tour choices\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school Running adaptive_chunked_choosers with 451 choosers\r\n", - "INFO - Running chunk 1 of 1 with 451 of 451 choosers\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 451 tour choices\r\n", - "INFO - tdd_alt_segments specified for representative logsums\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 85690 to 6765 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums compute_logsums for 6765 choosers 6765 alts\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums Running adaptive_chunked_choosers with 6765 choosers\r\n", - "INFO - Running chunk 1 of 1 with 6765 of 6765 choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 451 choosers and 85690 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 451 of 451 choosers\r\n", - "INFO - Running eval_interaction_utilities on 85690 rows\r\n", - "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 126 tour choices\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ Running adaptive_chunked_choosers with 126 choosers\r\n", - "INFO - Running chunk 1 of 1 with 126 of 126 choosers\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 126 tour choices\r\n", - "INFO - tdd_alt_segments specified for representative logsums\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 23940 to 1890 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums compute_logsums for 1890 choosers 1890 alts\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums Running adaptive_chunked_choosers with 1890 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1890 of 1890 choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 126 choosers and 23940 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 126 of 126 choosers\r\n", - "INFO - Running eval_interaction_utilities on 23940 rows\r\n", - "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work schedule_tours running 81 tour choices\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work Running adaptive_chunked_choosers with 81 choosers\r\n", - "INFO - Running chunk 1 of 1 with 81 of 81 choosers\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work schedule_tours running 81 tour choices\r\n", - "INFO - tdd_alt_segments specified for representative logsums\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 89.13% from 6074 to 660 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.compute_logsums.logsums compute_logsums for 660 choosers 660 alts\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.compute_logsums.logsums Running adaptive_chunked_choosers with 660 choosers\r\n", - "INFO - Running chunk 1 of 1 with 660 of 660 choosers\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: choosers\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_sample_alternatives\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 81 choosers and 6074 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 81 of 81 choosers\r\n", - "INFO - Running eval_interaction_utilities on 6074 rows\r\n", - "INFO - mandatory_tour_scheduling_work: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.work.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: interaction_expression_values\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school schedule_tours running 9 tour choices\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school Running adaptive_chunked_choosers with 9 choosers\r\n", - "INFO - Running chunk 1 of 1 with 9 of 9 choosers\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school schedule_tours running 9 tour choices\r\n", - "INFO - tdd_alt_segments specified for representative logsums\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 89.84% from 817 to 83 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.compute_logsums.logsums compute_logsums for 83 choosers 83 alts\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.compute_logsums.logsums Running adaptive_chunked_choosers with 83 choosers\r\n", - "INFO - Running chunk 1 of 1 with 83 of 83 choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 9 choosers and 817 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 9 of 9 choosers\r\n", - "INFO - Running eval_interaction_utilities on 817 rows\r\n", - "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.school.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ schedule_tours running 23 tour choices\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ Running adaptive_chunked_choosers with 23 choosers\r\n", - "INFO - Running chunk 1 of 1 with 23 of 23 choosers\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ schedule_tours running 23 tour choices\r\n", - "INFO - tdd_alt_segments specified for representative logsums\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 90.26% from 2176 to 212 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.compute_logsums.logsums compute_logsums for 212 choosers 212 alts\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.compute_logsums.logsums Running adaptive_chunked_choosers with 212 choosers\r\n", - "INFO - Running chunk 1 of 1 with 212 of 212 choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: choosers\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_sample_alternatives\r\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 23 choosers and 2176 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 23 of 23 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2176 rows\r\n", - "INFO - mandatory_tour_scheduling_school: eval_interaction_utilities write_interaction_expression_values mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.univ.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: interaction_expression_values\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_table cache: override_choices\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_work/mandatory_tour_scheduling_work_choosers_combined.csv\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - mandatory_tour_scheduling_work: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_work/mandatory_tour_scheduling_work_alternatives_combined.csv\r\n", - "INFO - mandatory_tour_scheduling_work: end estimation\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_table cache: override_choices\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_school/mandatory_tour_scheduling_school_choosers_combined.csv\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - mandatory_tour_scheduling_school: write_omnibus_choosers: output/estimation_data_bundle/mandatory_tour_scheduling_school/mandatory_tour_scheduling_school_alternatives_combined.csv\r\n", - "INFO - mandatory_tour_scheduling_school: end estimation\r\n", - "INFO - mandatory_tour_scheduling_work: timetable.rollback person_windows\r\n", - "INFO - mandatory_tour_scheduling_school: timetable.rollback person_windows\r\n", - "Time to execute step 'mandatory_tour_scheduling': 84.87 s\r\n", - "Total time to execute iteration 1 with iteration value None: 84.87 s\r\n", - "INFO - #run_model running step joint_tour_frequency\r\n", - "Running step 'joint_tour_frequency'\r\n", - "INFO - Initialize Estimator for'joint_tour_frequency'\r\n", - "INFO - Running joint_tour_frequency with 1028 multi-person households\r\n", - "DEBUG - @inject timetable\r\n", - "DEBUG - joint_tour_frequency: estimate.write_spec: output/estimation_data_bundle/joint_tour_frequency/joint_tour_frequency_SPEC.csv\r\n", - "DEBUG - joint_tour_frequency: write_table write: joint_tour_frequency_coefficients.csv\r\n", - "DEBUG - joint_tour_frequency: write_table cache: choosers\r\n", - "INFO - joint_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 1028 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1028 of 1028 choosers\r\n", - "DEBUG - joint_tour_frequency: write_table cache: expression_values\r\n", - "DEBUG - joint_tour_frequency: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using households.index\r\n", - "DEBUG - joint_tour_frequency: write_table cache: override_choices\r\n", - "DEBUG - joint_tour_frequency: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - joint_tour_frequency: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_frequency/joint_tour_frequency_values_combined.csv\r\n", - "INFO - joint_tour_frequency: end estimation\r\n", - "INFO - joint_tour_frequency top 10 value counts:\r\n", - "0_tours 1910\r\n", - "1_Main 26\r\n", - "1_Disc 20\r\n", - "1_Shop 16\r\n", - "1_Eat 15\r\n", - "1_Visit 12\r\n", - "2_SS 1\r\n", - "Name: joint_tour_frequency, dtype: int64\r\n", - "len(survey_tours) 91\r\n", - "len(joint_tours) 91\r\n", - "Time to execute step 'joint_tour_frequency': 0.68 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.68 s\r\n", - "INFO - #run_model running step joint_tour_composition\r\n", - "Running step 'joint_tour_composition'\r\n", - "INFO - Initialize Estimator for'joint_tour_composition'\r\n", - "INFO - Running joint_tour_composition with 91 joint tours\r\n", - "DEBUG - @inject timetable\r\n", - "DEBUG - joint_tour_composition: estimate.write_spec: output/estimation_data_bundle/joint_tour_composition/joint_tour_composition_SPEC.csv\r\n", - "DEBUG - joint_tour_composition: write_table write: joint_tour_composition_coefficients.csv\r\n", - "DEBUG - joint_tour_composition: write_table cache: choosers\r\n", - "INFO - joint_tour_composition.simple_simulate Running adaptive_chunked_choosers with 91 choosers\r\n", - "INFO - Running chunk 1 of 1 with 91 of 91 choosers\r\n", - "DEBUG - joint_tour_composition: write_table cache: expression_values\r\n", - "DEBUG - joint_tour_composition: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - joint_tour_composition: write_table cache: override_choices\r\n", - "DEBUG - joint_tour_composition: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - joint_tour_composition: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_composition/joint_tour_composition_values_combined.csv\r\n", - "INFO - joint_tour_composition: end estimation\r\n", - "INFO - joint_tour_composition top 10 value counts:\r\n", - "adults 54\r\n", - "mixed 32\r\n", - "children 5\r\n", - "Name: composition, dtype: int64\r\n", - "Time to execute step 'joint_tour_composition': 0.31 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.31 s\r\n", - "INFO - #run_model running step joint_tour_participation\r\n", - "Running step 'joint_tour_participation'\r\n", - "INFO - Running joint_tours_participation with 304 potential participants (candidates)\r\n", - "DEBUG - @inject timetable\r\n", - "INFO - Initialize Estimator for'joint_tour_participation'\r\n", - "DEBUG - joint_tour_participation: estimate.write_spec: output/estimation_data_bundle/joint_tour_participation/joint_tour_participation_SPEC.csv\r\n", - "DEBUG - joint_tour_participation: write_table write: joint_tour_participation_coefficients.csv\r\n", - "DEBUG - joint_tour_participation: write_table cache: choosers\r\n", - "INFO - joint_tour_participation Running chunk 1 of 1 with 90 of 90 choosers\r\n", - "DEBUG - joint_tour_participation: write_table cache: expression_values\r\n", - "INFO - joint_tour_participation.eval_mnl.participants_chooser 91 joint tours to satisfy.\r\n", - "INFO - joint_tour_participation.eval_mnl.participants_chooser 8 iterations to satisfy all joint tours.\r\n", - "DEBUG - joint_tour_participation: write_table cache: choices\r\n", - "DEBUG - joint_tour_participation: write_table cache: override_choices\r\n", - "DEBUG - joint_tour_participation: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - joint_tour_participation: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_participation/joint_tour_participation_values_combined.csv\r\n", - "INFO - joint_tour_participation: end estimation\r\n", - "Time to execute step 'joint_tour_participation': 0.99 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.99 s\r\n", - "INFO - #run_model running step joint_tour_destination\r\n", - "Running step 'joint_tour_destination'\r\n", - "INFO - Initialize Estimator for'joint_tour_destination'\r\n", - "DEBUG - joint_tour_destination: write_table write: non_mandatory_tour_destination_coefficients.csv\r\n", - "DEBUG - joint_tour_destination: estimate.write_spec: output/estimation_data_bundle/joint_tour_destination/joint_tour_destination_SPEC.csv\r\n", - "DEBUG - joint_tour_destination: write_table write: size_terms\r\n", - "DEBUG - joint_tour_destination: write_table write: landuse\r\n", - "INFO - running joint_tour_destination.shopping.sample with 18 tours\r\n", - "INFO - Estimation mode for joint_tour_destination.shopping.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - joint_tour_destination.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 18 choosers\r\n", - "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3420 rows\r\n", - "INFO - Estimation mode for joint_tour_destination.shopping.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running joint_tour_destination.shopping.logsums with 3420 rows\r\n", - "INFO - joint_tour_destination.shopping.logsums.compute_logsums Running adaptive_chunked_choosers with 3420 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3420 of 3420 choosers\r\n", - "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 18 persons\r\n", - "INFO - joint_tour_destination.shopping.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 18 choosers and 3420 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3420 rows\r\n", - "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.shopping.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running joint_tour_destination.othmaint.sample with 26 tours\r\n", - "INFO - Estimation mode for joint_tour_destination.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - joint_tour_destination.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 26 choosers\r\n", - "INFO - Running chunk 1 of 1 with 26 of 26 choosers\r\n", - "INFO - Running eval_interaction_utilities on 4940 rows\r\n", - "INFO - Estimation mode for joint_tour_destination.othmaint.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running joint_tour_destination.othmaint.logsums with 4940 rows\r\n", - "INFO - joint_tour_destination.othmaint.logsums.compute_logsums Running adaptive_chunked_choosers with 4940 choosers\r\n", - "INFO - Running chunk 1 of 1 with 4940 of 4940 choosers\r\n", - "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 26 persons\r\n", - "INFO - joint_tour_destination.othmaint.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 26 choosers and 4940 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 26 of 26 choosers\r\n", - "INFO - Running eval_interaction_utilities on 4940 rows\r\n", - "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.othmaint.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running joint_tour_destination.othdiscr.sample with 20 tours\r\n", - "INFO - Estimation mode for joint_tour_destination.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - joint_tour_destination.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 20 choosers\r\n", - "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3800 rows\r\n", - "INFO - Estimation mode for joint_tour_destination.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running joint_tour_destination.othdiscr.logsums with 3800 rows\r\n", - "INFO - joint_tour_destination.othdiscr.logsums.compute_logsums Running adaptive_chunked_choosers with 3800 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3800 of 3800 choosers\r\n", - "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 20 persons\r\n", - "INFO - joint_tour_destination.othdiscr.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 20 choosers and 3800 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3800 rows\r\n", - "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.othdiscr.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running joint_tour_destination.eatout.sample with 15 tours\r\n", - "INFO - Estimation mode for joint_tour_destination.eatout.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - joint_tour_destination.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 15 choosers\r\n", - "INFO - Running chunk 1 of 1 with 15 of 15 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2850 rows\r\n", - "INFO - Estimation mode for joint_tour_destination.eatout.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running joint_tour_destination.eatout.logsums with 2850 rows\r\n", - "INFO - joint_tour_destination.eatout.logsums.compute_logsums Running adaptive_chunked_choosers with 2850 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2850 of 2850 choosers\r\n", - "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 15 persons\r\n", - "INFO - joint_tour_destination.eatout.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 15 choosers and 2850 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 15 of 15 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2850 rows\r\n", - "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.eatout.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running joint_tour_destination.social.sample with 12 tours\r\n", - "INFO - Estimation mode for joint_tour_destination.social.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - joint_tour_destination.social.sample.interaction_sample Running adaptive_chunked_choosers with 12 choosers\r\n", - "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2280 rows\r\n", - "INFO - Estimation mode for joint_tour_destination.social.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running joint_tour_destination.social.logsums with 2280 rows\r\n", - "INFO - joint_tour_destination.social.logsums.compute_logsums Running adaptive_chunked_choosers with 2280 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2280 of 2280 choosers\r\n", - "DEBUG - joint_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 12 persons\r\n", - "INFO - joint_tour_destination.social.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 12 choosers and 2280 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2280 rows\r\n", - "INFO - joint_tour_destination: eval_interaction_utilities write_interaction_expression_values joint_tour_destination.social.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - joint_tour_destination skipping segment escort: no choosers\r\n", - "DEBUG - joint_tour_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - joint_tour_destination: write_table cache: override_choices\r\n", - "DEBUG - joint_tour_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - joint_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_destination/joint_tour_destination_choosers_combined.csv\r\n", - "DEBUG - joint_tour_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "WARNING - joint_tour_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", - "DEBUG - joint_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_destination/joint_tour_destination_alternatives_combined.csv\r\n", - "INFO - joint_tour_destination: end estimation\r\n", - "INFO - destination summary:\r\n", - "count 91.000000\r\n", - "mean 101.439560\r\n", - "std 56.893899\r\n", - "min 4.000000\r\n", - "25% 45.000000\r\n", - "50% 114.000000\r\n", - "75% 149.000000\r\n", - "max 190.000000\r\n", - "Name: destination, dtype: float64\r\n", - "Time to execute step 'joint_tour_destination': 7.54 s\r\n", - "Total time to execute iteration 1 with iteration value None: 7.54 s\r\n", - "INFO - #run_model running step joint_tour_scheduling\r\n", - "Running step 'joint_tour_scheduling'\r\n", - "INFO - Running joint_tour_scheduling with 91 joint tours\r\n", - "DEBUG - @inject timetable\r\n", - "INFO - Initialize Estimator for'joint_tour_scheduling'\r\n", - "DEBUG - joint_tour_scheduling: estimate.write_spec: output/estimation_data_bundle/joint_tour_scheduling/joint_tour_scheduling_SPEC.csv\r\n", - "DEBUG - joint_tour_scheduling: write_table write: tour_scheduling_joint_coefficients.csv\r\n", - "INFO - joint_tour_scheduling: timetable.begin_transaction person_windows\r\n", - "INFO - schedule_tours %s tours not monotonic_increasing - sorting df\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1 schedule_tours running 90 tour choices\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1 Running adaptive_chunked_choosers with 90 choosers\r\n", - "INFO - Running chunk 1 of 1 with 90 of 90 choosers\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1 schedule_tours running 90 tour choices\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 90 choosers and 6458 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 90 of 90 choosers\r\n", - "INFO - Running eval_interaction_utilities on 6458 rows\r\n", - "INFO - joint_tour_scheduling: eval_interaction_utilities write_interaction_expression_values joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_1.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2 schedule_tours running 1 tour choices\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2 Running adaptive_chunked_choosers with 1 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1 of 1 choosers\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2 schedule_tours running 1 tour choices\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 1 choosers and 73 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 1 of 1 choosers\r\n", - "INFO - Running eval_interaction_utilities on 73 rows\r\n", - "INFO - joint_tour_scheduling: eval_interaction_utilities write_interaction_expression_values joint_tour_scheduling.vectorize_joint_tour_scheduling.tour_2.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - joint_tour_scheduling: write_table cache: override_choices\r\n", - "DEBUG - joint_tour_scheduling: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - joint_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_scheduling/joint_tour_scheduling_choosers_combined.csv\r\n", - "DEBUG - joint_tour_scheduling: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - joint_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/joint_tour_scheduling/joint_tour_scheduling_alternatives_combined.csv\r\n", - "INFO - joint_tour_scheduling: end estimation\r\n", - "INFO - joint_tour_scheduling: timetable.rollback person_windows\r\n", - "INFO - joint_tour_scheduling: assign timetable for 224 participants in 90 tours with tour_num 1\r\n", - "INFO - joint_tour_scheduling: assign timetable for 4 participants in 1 tours with tour_num 2\r\n", - "Time to execute step 'joint_tour_scheduling': 1.89 s\r\n", - "Total time to execute iteration 1 with iteration value None: 1.89 s\r\n", - "INFO - #run_model running step non_mandatory_tour_frequency\r\n", - "Running step 'non_mandatory_tour_frequency'\r\n", - "DEBUG - @inject timetable\r\n", - "INFO - Running non_mandatory_tour_frequency with 3833 persons\r\n", - "INFO - Running segment 'PTYPE_FULL' of size 1764\r\n", - "INFO - Initialize Estimator for'PTYPE_FULL'\r\n", - "DEBUG - PTYPE_FULL: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_FULL: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_FULL.csv\r\n", - "DEBUG - PTYPE_FULL: write_table cache: choosers\r\n", - "DEBUG - PTYPE_FULL: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_FULL.interaction_simulate Running adaptive_chunked_choosers with 1764 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1764 of 1764 choosers\r\n", - "WARNING - /mnt/c/Users/jan.zill/code/activitysim/activitysim/core/logit.py:358: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", - " alts_sample[c_chooser] = np.repeat(choosers[c].values, sample_size)\r\n", - "\r\n", - "INFO - Running eval_interaction_utilities on 169344 rows\r\n", - "WARNING - /mnt/c/Users/jan.zill/code/activitysim/activitysim/core/interaction_simulate.py:482: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", - " choices = _interaction_simulate(chooser_chunk, alternatives, spec,\r\n", - "\r\n", - "INFO - PTYPE_FULL: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_FULL.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_FULL: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_FULL: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_FULL: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_FULL: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_FULL: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_FULL/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_FULL: end estimation\r\n", - "INFO - Running segment 'PTYPE_PART' of size 449\r\n", - "INFO - Initialize Estimator for'PTYPE_PART'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_PART: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_PART: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_PART.csv\r\n", - "DEBUG - PTYPE_PART: write_table cache: choosers\r\n", - "DEBUG - PTYPE_PART: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_PART.interaction_simulate Running adaptive_chunked_choosers with 449 choosers\r\n", - "INFO - Running chunk 1 of 1 with 449 of 449 choosers\r\n", - "INFO - Running eval_interaction_utilities on 43104 rows\r\n", - "INFO - PTYPE_PART: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_PART.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_PART: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_PART: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_PART: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_PART: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_PART: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_PART/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_PART: end estimation\r\n", - "INFO - Running segment 'PTYPE_UNIVERSITY' of size 296\r\n", - "INFO - Initialize Estimator for'PTYPE_UNIVERSITY'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_UNIVERSITY: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_UNIVERSITY.csv\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_table cache: choosers\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_UNIVERSITY.interaction_simulate Running adaptive_chunked_choosers with 296 choosers\r\n", - "INFO - Running chunk 1 of 1 with 296 of 296 choosers\r\n", - "INFO - Running eval_interaction_utilities on 28416 rows\r\n", - "INFO - PTYPE_UNIVERSITY: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_UNIVERSITY.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_UNIVERSITY: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_UNIVERSITY/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_UNIVERSITY: end estimation\r\n", - "INFO - Running segment 'PTYPE_NONWORK' of size 420\r\n", - "INFO - Initialize Estimator for'PTYPE_NONWORK'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_NONWORK: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_NONWORK: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_NONWORK.csv\r\n", - "DEBUG - PTYPE_NONWORK: write_table cache: choosers\r\n", - "DEBUG - PTYPE_NONWORK: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_NONWORK.interaction_simulate Running adaptive_chunked_choosers with 420 choosers\r\n", - "INFO - Running chunk 1 of 1 with 420 of 420 choosers\r\n", - "INFO - Running eval_interaction_utilities on 40320 rows\r\n", - "INFO - PTYPE_NONWORK: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_NONWORK.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_NONWORK: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_NONWORK: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_NONWORK: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_NONWORK: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_NONWORK: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_NONWORK/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_NONWORK: end estimation\r\n", - "INFO - Running segment 'PTYPE_RETIRED' of size 357\r\n", - "INFO - Initialize Estimator for'PTYPE_RETIRED'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_RETIRED: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_RETIRED: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_RETIRED.csv\r\n", - "DEBUG - PTYPE_RETIRED: write_table cache: choosers\r\n", - "DEBUG - PTYPE_RETIRED: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_RETIRED.interaction_simulate Running adaptive_chunked_choosers with 357 choosers\r\n", - "INFO - Running chunk 1 of 1 with 357 of 357 choosers\r\n", - "INFO - Running eval_interaction_utilities on 34272 rows\r\n", - "INFO - PTYPE_RETIRED: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_RETIRED.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_RETIRED: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_RETIRED: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_RETIRED: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_RETIRED: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_RETIRED: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_RETIRED/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_RETIRED: end estimation\r\n", - "INFO - Running segment 'PTYPE_DRIVING' of size 50\r\n", - "INFO - Initialize Estimator for'PTYPE_DRIVING'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_DRIVING: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_DRIVING: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_DRIVING.csv\r\n", - "DEBUG - PTYPE_DRIVING: write_table cache: choosers\r\n", - "DEBUG - PTYPE_DRIVING: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_DRIVING.interaction_simulate Running adaptive_chunked_choosers with 50 choosers\r\n", - "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", - "INFO - Running eval_interaction_utilities on 4800 rows\r\n", - "INFO - PTYPE_DRIVING: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_DRIVING.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_DRIVING: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_DRIVING: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_DRIVING: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_DRIVING: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_DRIVING: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_DRIVING/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_DRIVING: end estimation\r\n", - "INFO - Running segment 'PTYPE_SCHOOL' of size 313\r\n", - "INFO - Initialize Estimator for'PTYPE_SCHOOL'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_SCHOOL: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_SCHOOL: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_SCHOOL.csv\r\n", - "DEBUG - PTYPE_SCHOOL: write_table cache: choosers\r\n", - "DEBUG - PTYPE_SCHOOL: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_SCHOOL.interaction_simulate Running adaptive_chunked_choosers with 313 choosers\r\n", - "INFO - Running chunk 1 of 1 with 313 of 313 choosers\r\n", - "INFO - Running eval_interaction_utilities on 30048 rows\r\n", - "INFO - PTYPE_SCHOOL: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_SCHOOL.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_SCHOOL: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_SCHOOL: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_SCHOOL: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_SCHOOL: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_SCHOOL: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_SCHOOL/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_SCHOOL: end estimation\r\n", - "INFO - Running segment 'PTYPE_PRESCHOOL' of size 184\r\n", - "INFO - Initialize Estimator for'PTYPE_PRESCHOOL'\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_alternatives.csv\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_model_settings.yaml\r\n", - "deleted output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_PRESCHOOL: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_frequency/non_mandatory_tour_frequency_SPEC.csv\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_table write: non_mandatory_tour_frequency_coefficients_PTYPE_PRESCHOOL.csv\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_table cache: choosers\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_table write: alternatives\r\n", - "INFO - non_mandatory_tour_frequency.PTYPE_PRESCHOOL.interaction_simulate Running adaptive_chunked_choosers with 184 choosers\r\n", - "INFO - Running chunk 1 of 1 with 184 of 184 choosers\r\n", - "INFO - Running eval_interaction_utilities on 17664 rows\r\n", - "INFO - PTYPE_PRESCHOOL: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_frequency.PTYPE_PRESCHOOL.interaction_simulate.interaction_simulate.eval_interaction_utils\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_table write: interaction_expression_values\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_table cache: override_choices\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - PTYPE_PRESCHOOL: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_frequency/PTYPE_PRESCHOOL/non_mandatory_tour_frequency_choosers_combined.csv\r\n", - "INFO - PTYPE_PRESCHOOL: end estimation\r\n", - "INFO - extend_tour_counts increased tour count by 166 from 2324 to 2490\r\n", - "DEBUG - get_survey_values: reindexing using persons.index\r\n", - "INFO - estimation get_survey_values override_tour_counts 0 changed cells\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "INFO - non_mandatory_tour_frequency top 10 value counts:\r\n", - "0 2718\r\n", - "16 374\r\n", - "1 282\r\n", - "8 174\r\n", - "4 154\r\n", - "32 132\r\n", - "2 84\r\n", - "17 61\r\n", - "24 50\r\n", - "9 32\r\n", - "Name: non_mandatory_tour_frequency, dtype: int64\r\n", - "Time to execute step 'non_mandatory_tour_frequency': 188.21 s\r\n", - "Total time to execute iteration 1 with iteration value None: 188.21 s\r\n", - "INFO - #run_model running step non_mandatory_tour_destination\r\n", - "Running step 'non_mandatory_tour_destination'\r\n", - "INFO - Initialize Estimator for'non_mandatory_tour_destination'\r\n", - "DEBUG - non_mandatory_tour_destination: write_table write: non_mandatory_tour_destination_coefficients.csv\r\n", - "DEBUG - non_mandatory_tour_destination: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_destination/non_mandatory_tour_destination_SPEC.csv\r\n", - "DEBUG - non_mandatory_tour_destination: write_table write: size_terms\r\n", - "DEBUG - non_mandatory_tour_destination: write_table write: landuse\r\n", - "INFO - running non_mandatory_tour_destination.shopping.sample with 705 tours\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.shopping.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - non_mandatory_tour_destination.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 705 choosers\r\n", - "INFO - Running chunk 1 of 1 with 705 of 705 choosers\r\n", - "INFO - Running eval_interaction_utilities on 133950 rows\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.shopping.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running non_mandatory_tour_destination.shopping.logsums with 133950 rows\r\n", - "INFO - non_mandatory_tour_destination.shopping.logsums.compute_logsums Running adaptive_chunked_choosers with 133950 choosers\r\n", - "INFO - Running chunk 1 of 1 with 133950 of 133950 choosers\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 705 persons\r\n", - "INFO - non_mandatory_tour_destination.shopping.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 705 choosers and 133950 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 705 of 705 choosers\r\n", - "INFO - Running eval_interaction_utilities on 133950 rows\r\n", - "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.shopping.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running non_mandatory_tour_destination.othmaint.sample with 367 tours\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - non_mandatory_tour_destination.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 367 choosers\r\n", - "INFO - Running chunk 1 of 1 with 367 of 367 choosers\r\n", - "INFO - Running eval_interaction_utilities on 69730 rows\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.othmaint.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running non_mandatory_tour_destination.othmaint.logsums with 69730 rows\r\n", - "INFO - non_mandatory_tour_destination.othmaint.logsums.compute_logsums Running adaptive_chunked_choosers with 69730 choosers\r\n", - "INFO - Running chunk 1 of 1 with 69730 of 69730 choosers\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 367 persons\r\n", - "INFO - non_mandatory_tour_destination.othmaint.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 367 choosers and 69730 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 367 of 367 choosers\r\n", - "INFO - Running eval_interaction_utilities on 69730 rows\r\n", - "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.othmaint.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running non_mandatory_tour_destination.othdiscr.sample with 544 tours\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - non_mandatory_tour_destination.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 544 choosers\r\n", - "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", - "INFO - Running eval_interaction_utilities on 103360 rows\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running non_mandatory_tour_destination.othdiscr.logsums with 103360 rows\r\n", - "INFO - non_mandatory_tour_destination.othdiscr.logsums.compute_logsums Running adaptive_chunked_choosers with 103360 choosers\r\n", - "INFO - Running chunk 1 of 1 with 103360 of 103360 choosers\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 544 persons\r\n", - "INFO - non_mandatory_tour_destination.othdiscr.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 544 choosers and 103360 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 544 of 544 choosers\r\n", - "INFO - Running eval_interaction_utilities on 103360 rows\r\n", - "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.othdiscr.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running non_mandatory_tour_destination.eatout.sample with 301 tours\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.eatout.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - non_mandatory_tour_destination.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 301 choosers\r\n", - "INFO - Running chunk 1 of 1 with 301 of 301 choosers\r\n", - "INFO - Running eval_interaction_utilities on 57190 rows\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.eatout.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running non_mandatory_tour_destination.eatout.logsums with 57190 rows\r\n", - "INFO - non_mandatory_tour_destination.eatout.logsums.compute_logsums Running adaptive_chunked_choosers with 57190 choosers\r\n", - "INFO - Running chunk 1 of 1 with 57190 of 57190 choosers\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 301 persons\r\n", - "INFO - non_mandatory_tour_destination.eatout.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 301 choosers and 57190 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 301 of 301 choosers\r\n", - "INFO - Running eval_interaction_utilities on 57190 rows\r\n", - "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.eatout.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running non_mandatory_tour_destination.social.sample with 170 tours\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.social.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - non_mandatory_tour_destination.social.sample.interaction_sample Running adaptive_chunked_choosers with 170 choosers\r\n", - "INFO - Running chunk 1 of 1 with 170 of 170 choosers\r\n", - "INFO - Running eval_interaction_utilities on 32300 rows\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.social.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running non_mandatory_tour_destination.social.logsums with 32300 rows\r\n", - "INFO - non_mandatory_tour_destination.social.logsums.compute_logsums Running adaptive_chunked_choosers with 32300 choosers\r\n", - "INFO - Running chunk 1 of 1 with 32300 of 32300 choosers\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 170 persons\r\n", - "INFO - non_mandatory_tour_destination.social.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 170 choosers and 32300 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 170 of 170 choosers\r\n", - "INFO - Running eval_interaction_utilities on 32300 rows\r\n", - "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.social.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", - "INFO - running non_mandatory_tour_destination.escort.sample with 403 tours\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.escort.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - non_mandatory_tour_destination.escort.sample.interaction_sample Running adaptive_chunked_choosers with 403 choosers\r\n", - "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", - "INFO - Running eval_interaction_utilities on 76570 rows\r\n", - "INFO - Estimation mode for non_mandatory_tour_destination.escort.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running non_mandatory_tour_destination.escort.logsums with 76570 rows\r\n", - "INFO - non_mandatory_tour_destination.escort.logsums.compute_logsums Running adaptive_chunked_choosers with 76570 choosers\r\n", - "INFO - Running chunk 1 of 1 with 76570 of 76570 choosers\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 403 persons\r\n", - "INFO - non_mandatory_tour_destination.escort.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 403 choosers and 76570 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", - "INFO - Running eval_interaction_utilities on 76570 rows\r\n", - "INFO - non_mandatory_tour_destination: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_destination.escort.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - non_mandatory_tour_destination: write_table cache: override_choices\r\n", - "DEBUG - non_mandatory_tour_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - non_mandatory_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_destination/non_mandatory_tour_destination_choosers_combined.csv\r\n", - "DEBUG - non_mandatory_tour_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "WARNING - non_mandatory_tour_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", - "DEBUG - non_mandatory_tour_destination: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_destination/non_mandatory_tour_destination_alternatives_combined.csv\r\n", - "INFO - non_mandatory_tour_destination: end estimation\r\n", - "Time to execute step 'non_mandatory_tour_destination': 38.82 s\r\n", - "Total time to execute iteration 1 with iteration value None: 38.82 s\r\n", - "INFO - #run_model running step non_mandatory_tour_scheduling\r\n", - "Running step 'non_mandatory_tour_scheduling'\r\n", - "DEBUG - @inject timetable\r\n", - "INFO - Initialize Estimator for'non_mandatory_tour_scheduling'\r\n", - "DEBUG - non_mandatory_tour_scheduling: estimate.write_spec: output/estimation_data_bundle/non_mandatory_tour_scheduling/non_mandatory_tour_scheduling_SPEC.csv\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table write: tour_scheduling_nonmandatory_coefficients.csv\r\n", - "INFO - non_mandatory_tour_scheduling: timetable.begin_transaction person_windows\r\n", - "INFO - Running non_mandatory_tour_scheduling with 2490 tours\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1 schedule_tours running 1687 tour choices\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1 Running adaptive_chunked_choosers with 1687 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1687 of 1687 choosers\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1 schedule_tours running 1687 tour choices\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 1687 choosers and 241779 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 1687 of 1687 choosers\r\n", - "INFO - Running eval_interaction_utilities on 241779 rows\r\n", - "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2 schedule_tours running 571 tour choices\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2 Running adaptive_chunked_choosers with 571 choosers\r\n", - "INFO - Running chunk 1 of 1 with 571 of 571 choosers\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2 schedule_tours running 571 tour choices\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 571 choosers and 47267 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 571 of 571 choosers\r\n", - "INFO - Running eval_interaction_utilities on 47267 rows\r\n", - "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_2.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3 schedule_tours running 179 tour choices\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3 Running adaptive_chunked_choosers with 179 choosers\r\n", - "INFO - Running chunk 1 of 1 with 179 of 179 choosers\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3 schedule_tours running 179 tour choices\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 179 choosers and 12032 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 179 of 179 choosers\r\n", - "INFO - Running eval_interaction_utilities on 12032 rows\r\n", - "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_3.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4 schedule_tours running 50 tour choices\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4 Running adaptive_chunked_choosers with 50 choosers\r\n", - "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4 schedule_tours running 50 tour choices\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 50 choosers and 2697 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2697 rows\r\n", - "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_4.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5 schedule_tours running 3 tour choices\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5 Running adaptive_chunked_choosers with 3 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5 schedule_tours running 3 tour choices\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: choosers\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 3 choosers and 146 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", - "INFO - Running eval_interaction_utilities on 146 rows\r\n", - "INFO - non_mandatory_tour_scheduling: eval_interaction_utilities write_interaction_expression_values non_mandatory_tour_scheduling.vectorize_tour_scheduling.tour_5.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: interaction_expression_values\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_table cache: override_choices\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_scheduling/non_mandatory_tour_scheduling_choosers_combined.csv\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - non_mandatory_tour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/non_mandatory_tour_scheduling/non_mandatory_tour_scheduling_alternatives_combined.csv\r\n", - "INFO - non_mandatory_tour_scheduling: end estimation\r\n", - "INFO - non_mandatory_tour_scheduling: timetable.rollback person_windows\r\n", - "Time to execute step 'non_mandatory_tour_scheduling': 75.98 s\r\n", - "Total time to execute iteration 1 with iteration value None: 75.98 s\r\n", - "INFO - #run_model running step tour_mode_choice_simulate\r\n", - "Running step 'tour_mode_choice_simulate'\r\n", - "INFO - Running tour_mode_choice with 5314 tours\r\n", - "INFO - tour_types top 10 value counts:\r\n", - "work 2124\r\n", - "shopping 723\r\n", - "school 609\r\n", - "othdiscr 564\r\n", - "escort 403\r\n", - "othmaint 393\r\n", - "eatout 316\r\n", - "social 182\r\n", - "Name: tour_type, dtype: int64\r\n", - "INFO - Initialize Estimator for'tour_mode_choice'\r\n", - "DEBUG - tour_mode_choice: write_table write: tour_mode_choice_coefficients.csv\r\n", - "DEBUG - tour_mode_choice: write_table write: coefficients_template\r\n", - "DEBUG - tour_mode_choice: estimate.write_spec: output/estimation_data_bundle/tour_mode_choice/tour_mode_choice_SPEC.csv\r\n", - "INFO - tour_mode_choice_simulate tour_type 'eatout' (316 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.eatout.simple_simulate Running adaptive_chunked_choosers with 316 choosers\r\n", - "INFO - Running chunk 1 of 1 with 316 of 316 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate eatout choices_df top 10 value counts:\r\n", - "DRIVEALONEFREE 77\r\n", - "WALK 73\r\n", - "SHARED3FREE 53\r\n", - "SHARED2FREE 53\r\n", - "WALK_LOC 24\r\n", - "WALK_LRF 13\r\n", - "WALK_HVY 7\r\n", - "BIKE 6\r\n", - "TNC_SINGLE 5\r\n", - "DRIVE_LOC 3\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'escort' (403 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.escort.simple_simulate Running adaptive_chunked_choosers with 403 choosers\r\n", - "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate escort choices_df top 10 value counts:\r\n", - "SHARED2FREE 175\r\n", - "SHARED3FREE 151\r\n", - "TNC_SINGLE 45\r\n", - "TNC_SHARED 15\r\n", - "TAXI 6\r\n", - "BIKE 4\r\n", - "WALK 4\r\n", - "DRIVE_HVY 1\r\n", - "WALK_HVY 1\r\n", - "WALK_LRF 1\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'othdiscr' (564 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.othdiscr.simple_simulate Running adaptive_chunked_choosers with 564 choosers\r\n", - "INFO - Running chunk 1 of 1 with 564 of 564 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate othdiscr choices_df top 10 value counts:\r\n", - "DRIVEALONEFREE 151\r\n", - "SHARED3FREE 104\r\n", - "SHARED2FREE 83\r\n", - "WALK 73\r\n", - "WALK_LOC 65\r\n", - "WALK_LRF 31\r\n", - "WALK_HVY 19\r\n", - "BIKE 19\r\n", - "TNC_SINGLE 12\r\n", - "TAXI 2\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'othmaint' (393 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.othmaint.simple_simulate Running adaptive_chunked_choosers with 393 choosers\r\n", - "INFO - Running chunk 1 of 1 with 393 of 393 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate othmaint choices_df top 10 value counts:\r\n", - "DRIVEALONEFREE 141\r\n", - "SHARED2FREE 62\r\n", - "WALK_LOC 36\r\n", - "SHARED3FREE 35\r\n", - "BIKE 28\r\n", - "WALK 23\r\n", - "WALK_LRF 23\r\n", - "TNC_SINGLE 21\r\n", - "TNC_SHARED 11\r\n", - "WALK_HVY 9\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'school' (460 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.school.simple_simulate Running adaptive_chunked_choosers with 460 choosers\r\n", - "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate school choices_df top 10 value counts:\r\n", - "SHARED3FREE 126\r\n", - "WALK_LRF 101\r\n", - "WALK_LOC 92\r\n", - "SHARED2FREE 59\r\n", - "WALK 48\r\n", - "WALK_HVY 23\r\n", - "BIKE 7\r\n", - "DRIVEALONEFREE 2\r\n", - "TNC_SHARED 1\r\n", - "TAXI 1\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'shopping' (723 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.shopping.simple_simulate Running adaptive_chunked_choosers with 723 choosers\r\n", - "INFO - Running chunk 1 of 1 with 723 of 723 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate shopping choices_df top 10 value counts:\r\n", - "DRIVEALONEFREE 270\r\n", - "SHARED2FREE 107\r\n", - "WALK 82\r\n", - "SHARED3FREE 75\r\n", - "WALK_LOC 46\r\n", - "TNC_SINGLE 44\r\n", - "WALK_LRF 35\r\n", - "BIKE 27\r\n", - "TNC_SHARED 17\r\n", - "WALK_HVY 9\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'social' (182 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.social.simple_simulate Running adaptive_chunked_choosers with 182 choosers\r\n", - "INFO - Running chunk 1 of 1 with 182 of 182 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate social choices_df top 10 value counts:\r\n", - "SHARED2FREE 43\r\n", - "DRIVEALONEFREE 29\r\n", - "SHARED3FREE 28\r\n", - "WALK 23\r\n", - "WALK_LRF 17\r\n", - "WALK_LOC 16\r\n", - "BIKE 15\r\n", - "TNC_SINGLE 8\r\n", - "WALK_HVY 2\r\n", - "TNC_SHARED 1\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'univ' (149 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.univ.simple_simulate Running adaptive_chunked_choosers with 149 choosers\r\n", - "INFO - Running chunk 1 of 1 with 149 of 149 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate univ choices_df top 10 value counts:\r\n", - "DRIVEALONEFREE 40\r\n", - "WALK_LRF 38\r\n", - "WALK_LOC 38\r\n", - "WALK_HVY 9\r\n", - "SHARED3FREE 7\r\n", - "TNC_SHARED 6\r\n", - "TAXI 4\r\n", - "SHARED2FREE 3\r\n", - "TNC_SINGLE 2\r\n", - "BIKE 1\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - tour_mode_choice_simulate tour_type 'work' (2124 tours)\r\n", - "DEBUG - tour_mode_choice: write_table cache: choosers\r\n", - "INFO - tour_mode_choice.work.simple_simulate Running adaptive_chunked_choosers with 2124 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2124 of 2124 choosers\r\n", - "DEBUG - tour_mode_choice: write_table cache: expression_values\r\n", - "INFO - tour_mode_choice_simulate work choices_df top 10 value counts:\r\n", - "DRIVEALONEFREE 665\r\n", - "WALK_LOC 324\r\n", - "WALK_LRF 254\r\n", - "SHARED2FREE 216\r\n", - "TNC_SINGLE 177\r\n", - "SHARED3FREE 138\r\n", - "WALK 137\r\n", - "BIKE 109\r\n", - "WALK_HVY 82\r\n", - "TAXI 10\r\n", - "Name: tour_mode, dtype: int64\r\n", - "DEBUG - tour_mode_choice: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - tour_mode_choice: write_table cache: override_choices\r\n", - "DEBUG - tour_mode_choice: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - tour_mode_choice: write_omnibus_choosers: output/estimation_data_bundle/tour_mode_choice/tour_mode_choice_values_combined.csv\r\n", - "INFO - tour_mode_choice: end estimation\r\n", - "INFO - tour_mode_choice_simulate all tour type choices top 10 value counts:\r\n", - "DRIVEALONEFREE 1357\r\n", - "SHARED2FREE 801\r\n", - "SHARED3FREE 701\r\n", - "WALK_LOC 601\r\n", - "WALK_LRF 593\r\n", - "WALK 456\r\n", - "TNC_SINGLE 312\r\n", - "BIKE 200\r\n", - "WALK_HVY 194\r\n", - "TNC_SHARED 53\r\n", - "Name: tour_mode, dtype: int64\r\n", - "Time to execute step 'tour_mode_choice_simulate': 16.74 s\r\n", - "Total time to execute iteration 1 with iteration value None: 16.74 s\r\n", - "INFO - #run_model running step atwork_subtour_frequency\r\n", - "Running step 'atwork_subtour_frequency'\r\n", - "INFO - Initialize Estimator for'atwork_subtour_frequency'\r\n", - "INFO - Running atwork_subtour_frequency with 2124 work tours\r\n", - "DEBUG - atwork_subtour_frequency: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_frequency/atwork_subtour_frequency_SPEC.csv\r\n", - "DEBUG - atwork_subtour_frequency: write_table write: atwork_subtour_frequency_coefficients.csv\r\n", - "DEBUG - atwork_subtour_frequency: write_table cache: choosers\r\n", - "INFO - atwork_subtour_frequency.simple_simulate Running adaptive_chunked_choosers with 2124 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2124 of 2124 choosers\r\n", - "DEBUG - atwork_subtour_frequency: write_table cache: expression_values\r\n", - "DEBUG - atwork_subtour_frequency: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - atwork_subtour_frequency: write_table cache: override_choices\r\n", - "DEBUG - atwork_subtour_frequency: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - atwork_subtour_frequency: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_frequency/atwork_subtour_frequency_values_combined.csv\r\n", - "INFO - atwork_subtour_frequency: end estimation\r\n", - "INFO - atwork_subtour_frequency top 10 value counts:\r\n", - " 3654\r\n", - "no_subtours 1664\r\n", - "eat 346\r\n", - "maint 62\r\n", - "business1 48\r\n", - "eat_business 3\r\n", - "business2 1\r\n", - "Name: atwork_subtour_frequency, dtype: int64\r\n", - "Time to execute step 'atwork_subtour_frequency': 0.75 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.75 s\r\n", - "INFO - #run_model running step atwork_subtour_destination\r\n", - "Running step 'atwork_subtour_destination'\r\n", - "INFO - Initialize Estimator for'atwork_subtour_destination'\r\n", - "DEBUG - atwork_subtour_destination: write_table write: atwork_subtour_destination_coefficients.csv\r\n", - "DEBUG - atwork_subtour_destination: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_destination/atwork_subtour_destination_SPEC.csv\r\n", - "DEBUG - atwork_subtour_destination: write_table write: size_terms\r\n", - "DEBUG - atwork_subtour_destination: write_table write: landuse\r\n", - "INFO - running atwork_subtour_destination.atwork.sample with 464 tours\r\n", - "INFO - Estimation mode for atwork_subtour_destination.atwork.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - atwork_subtour_destination.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 464 choosers\r\n", - "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", - "INFO - Running eval_interaction_utilities on 88160 rows\r\n", - "INFO - Estimation mode for atwork_subtour_destination.atwork.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Running atwork_subtour_destination.atwork.logsums with 88160 rows\r\n", - "INFO - atwork_subtour_destination.atwork.logsums.compute_logsums Running adaptive_chunked_choosers with 88160 choosers\r\n", - "INFO - Running chunk 1 of 1 with 88160 of 88160 choosers\r\n", - "DEBUG - atwork_subtour_destination: write_table cache: choosers\r\n", - "INFO - Running tour_destination_simulate with 464 persons\r\n", - "INFO - atwork_subtour_destination.atwork.simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 464 choosers and 88160 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", - "INFO - Running eval_interaction_utilities on 88160 rows\r\n", - "INFO - atwork_subtour_destination: eval_interaction_utilities write_interaction_expression_values atwork_subtour_destination.atwork.simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - atwork_subtour_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - atwork_subtour_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - atwork_subtour_destination: write_table cache: override_choices\r\n", - "DEBUG - atwork_subtour_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - atwork_subtour_destination: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_destination/atwork_subtour_destination_choosers_combined.csv\r\n", - "DEBUG - atwork_subtour_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "WARNING - atwork_subtour_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", - "DEBUG - atwork_subtour_destination: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_destination/atwork_subtour_destination_alternatives_combined.csv\r\n", - "INFO - atwork_subtour_destination: end estimation\r\n", - "INFO - destination summary:\r\n", - "count 464.000000\r\n", - "mean 92.616379\r\n", - "std 54.698856\r\n", - "min 1.000000\r\n", - "25% 44.750000\r\n", - "50% 90.000000\r\n", - "75% 139.250000\r\n", - "max 190.000000\r\n", - "Name: destination, dtype: float64\r\n", - "Time to execute step 'atwork_subtour_destination': 9.38 s\r\n", - "Total time to execute iteration 1 with iteration value None: 9.38 s\r\n", - "INFO - #run_model running step atwork_subtour_scheduling\r\n", - "Running step 'atwork_subtour_scheduling'\r\n", - "INFO - Initialize Estimator for'atwork_subtour_scheduling'\r\n", - "INFO - Running atwork_subtour_scheduling with 464 tours\r\n", - "DEBUG - atwork_subtour_scheduling: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_scheduling/atwork_subtour_scheduling_SPEC.csv\r\n", - "DEBUG - atwork_subtour_scheduling: write_table write: tour_scheduling_atwork_coefficients.csv\r\n", - "INFO - atwork_subtour_scheduling.tour_1 schedule_tours running 460 tour choices\r\n", - "INFO - atwork_subtour_scheduling.tour_1 Running adaptive_chunked_choosers with 460 choosers\r\n", - "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", - "INFO - atwork_subtour_scheduling.tour_1 schedule_tours running 460 tour choices\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: choosers\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - atwork_subtour_scheduling.tour_1.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 460 choosers and 33700 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", - "INFO - Running eval_interaction_utilities on 33700 rows\r\n", - "INFO - atwork_subtour_scheduling: eval_interaction_utilities write_interaction_expression_values atwork_subtour_scheduling.tour_1.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_expression_values\r\n", - "INFO - atwork_subtour_scheduling.tour_2 schedule_tours running 4 tour choices\r\n", - "INFO - atwork_subtour_scheduling.tour_2 Running adaptive_chunked_choosers with 4 choosers\r\n", - "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", - "INFO - atwork_subtour_scheduling.tour_2 schedule_tours running 4 tour choices\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: choosers\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_sample_alternatives\r\n", - "INFO - atwork_subtour_scheduling.tour_2.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 4 choosers and 223 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", - "INFO - Running eval_interaction_utilities on 223 rows\r\n", - "INFO - atwork_subtour_scheduling: eval_interaction_utilities write_interaction_expression_values atwork_subtour_scheduling.tour_2.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: interaction_expression_values\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - atwork_subtour_scheduling: write_table cache: override_choices\r\n", - "DEBUG - atwork_subtour_scheduling: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - atwork_subtour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_scheduling/atwork_subtour_scheduling_choosers_combined.csv\r\n", - "DEBUG - atwork_subtour_scheduling: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "DEBUG - atwork_subtour_scheduling: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_scheduling/atwork_subtour_scheduling_alternatives_combined.csv\r\n", - "INFO - atwork_subtour_scheduling: end estimation\r\n", - "Time to execute step 'atwork_subtour_scheduling': 5.79 s\r\n", - "Total time to execute iteration 1 with iteration value None: 5.79 s\r\n", - "INFO - #run_model running step atwork_subtour_mode_choice\r\n", - "Running step 'atwork_subtour_mode_choice'\r\n", - "INFO - Running atwork_subtour_mode_choice with 464 subtours\r\n", - "INFO - atwork_subtour_mode_choice tour_type top 10 value counts:\r\n", - "eat 349\r\n", - "maint 62\r\n", - "business 53\r\n", - "Name: tour_type, dtype: int64\r\n", - "INFO - Initialize Estimator for'atwork_subtour_mode_choice'\r\n", - "DEBUG - atwork_subtour_mode_choice: write_table write: tour_mode_choice_coefficients.csv\r\n", - "DEBUG - atwork_subtour_mode_choice: write_table write: coefficients_template\r\n", - "DEBUG - atwork_subtour_mode_choice: estimate.write_spec: output/estimation_data_bundle/atwork_subtour_mode_choice/atwork_subtour_mode_choice_SPEC.csv\r\n", - "WARNING - /mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/pandas/core/frame.py:3636: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", - " self[k1] = value[k2]\r\n", - "\r\n", - "WARNING - /mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/util/mode.py:129: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\r\n", - " choosers[trace_column_names] = choosers.index\r\n", - "\r\n", - "DEBUG - atwork_subtour_mode_choice: write_table cache: choosers\r\n", - "INFO - atwork_subtour_mode_choice.simple_simulate Running adaptive_chunked_choosers with 464 choosers\r\n", - "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", - "DEBUG - atwork_subtour_mode_choice: write_table cache: expression_values\r\n", - "DEBUG - atwork_subtour_mode_choice: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - atwork_subtour_mode_choice: write_table cache: override_choices\r\n", - "DEBUG - atwork_subtour_mode_choice: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - atwork_subtour_mode_choice: write_omnibus_choosers: output/estimation_data_bundle/atwork_subtour_mode_choice/atwork_subtour_mode_choice_values_combined.csv\r\n", - "INFO - atwork_subtour_mode_choice: end estimation\r\n", - "INFO - atwork_subtour_mode_choice choices top 10 value counts:\r\n", - "DRIVEALONEFREE 133\r\n", - "WALK 99\r\n", - "SHARED2FREE 95\r\n", - "SHARED3FREE 67\r\n", - "TNC_SINGLE 27\r\n", - "WALK_LOC 13\r\n", - "TNC_SHARED 11\r\n", - "BIKE 10\r\n", - "WALK_LRF 5\r\n", - "TAXI 2\r\n", - "Name: tour_mode, dtype: int64\r\n", - "Time to execute step 'atwork_subtour_mode_choice': 2.23 s\r\n", - "Total time to execute iteration 1 with iteration value None: 2.23 s\r\n", - "INFO - #run_model running step stop_frequency\r\n", - "Running step 'stop_frequency'\r\n", - "INFO - stop_frequency segments top 10 value counts:\r\n", - "work 2124\r\n", - "shopping 723\r\n", - "othdiscr 564\r\n", - "atwork 464\r\n", - "school 460\r\n", - "escort 403\r\n", - "othmaint 393\r\n", - "eatout 316\r\n", - "social 182\r\n", - "univ 149\r\n", - "Name: primary_purpose, dtype: int64\r\n", - "INFO - stop_frequency running segment work with 2124 chooser rows\r\n", - "INFO - Initialize Estimator for'work'\r\n", - "DEBUG - work: estimate.write_spec: output/estimation_data_bundle/stop_frequency/work/stop_frequency_SPEC.csv\r\n", - "DEBUG - work: write_table write: stop_frequency_coefficients_work.csv\r\n", - "DEBUG - work: write_table cache: choosers\r\n", - "INFO - stop_frequency.work.simple_simulate Running adaptive_chunked_choosers with 2124 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2124 of 2124 choosers\r\n", - "DEBUG - work: write_table cache: expression_values\r\n", - "DEBUG - work: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - work: write_table cache: override_choices\r\n", - "DEBUG - work: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - work: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/work/stop_frequency_values_combined.csv\r\n", - "INFO - work: end estimation\r\n", - "INFO - stop_frequency running segment school with 460 chooser rows\r\n", - "INFO - Initialize Estimator for'school'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - school: estimate.write_spec: output/estimation_data_bundle/stop_frequency/school/stop_frequency_SPEC.csv\r\n", - "DEBUG - school: write_table write: stop_frequency_coefficients_school.csv\r\n", - "DEBUG - school: write_table cache: choosers\r\n", - "INFO - stop_frequency.school.simple_simulate Running adaptive_chunked_choosers with 460 choosers\r\n", - "INFO - Running chunk 1 of 1 with 460 of 460 choosers\r\n", - "DEBUG - school: write_table cache: expression_values\r\n", - "DEBUG - school: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - school: write_table cache: override_choices\r\n", - "DEBUG - school: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - school: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/school/stop_frequency_values_combined.csv\r\n", - "INFO - school: end estimation\r\n", - "INFO - stop_frequency running segment univ with 149 chooser rows\r\n", - "INFO - Initialize Estimator for'univ'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - univ: estimate.write_spec: output/estimation_data_bundle/stop_frequency/univ/stop_frequency_SPEC.csv\r\n", - "DEBUG - univ: write_table write: stop_frequency_coefficients_univ.csv\r\n", - "DEBUG - univ: write_table cache: choosers\r\n", - "INFO - stop_frequency.univ.simple_simulate Running adaptive_chunked_choosers with 149 choosers\r\n", - "INFO - Running chunk 1 of 1 with 149 of 149 choosers\r\n", - "DEBUG - univ: write_table cache: expression_values\r\n", - "DEBUG - univ: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - univ: write_table cache: override_choices\r\n", - "DEBUG - univ: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - univ: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/univ/stop_frequency_values_combined.csv\r\n", - "INFO - univ: end estimation\r\n", - "INFO - stop_frequency running segment social with 182 chooser rows\r\n", - "INFO - Initialize Estimator for'social'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - social: estimate.write_spec: output/estimation_data_bundle/stop_frequency/social/stop_frequency_SPEC.csv\r\n", - "DEBUG - social: write_table write: stop_frequency_coefficients_social.csv\r\n", - "DEBUG - social: write_table cache: choosers\r\n", - "INFO - stop_frequency.social.simple_simulate Running adaptive_chunked_choosers with 182 choosers\r\n", - "INFO - Running chunk 1 of 1 with 182 of 182 choosers\r\n", - "DEBUG - social: write_table cache: expression_values\r\n", - "DEBUG - social: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - social: write_table cache: override_choices\r\n", - "DEBUG - social: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - social: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/social/stop_frequency_values_combined.csv\r\n", - "INFO - social: end estimation\r\n", - "INFO - stop_frequency running segment shopping with 723 chooser rows\r\n", - "INFO - Initialize Estimator for'shopping'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - shopping: estimate.write_spec: output/estimation_data_bundle/stop_frequency/shopping/stop_frequency_SPEC.csv\r\n", - "DEBUG - shopping: write_table write: stop_frequency_coefficients_shopping.csv\r\n", - "DEBUG - shopping: write_table cache: choosers\r\n", - "INFO - stop_frequency.shopping.simple_simulate Running adaptive_chunked_choosers with 723 choosers\r\n", - "INFO - Running chunk 1 of 1 with 723 of 723 choosers\r\n", - "DEBUG - shopping: write_table cache: expression_values\r\n", - "DEBUG - shopping: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - shopping: write_table cache: override_choices\r\n", - "DEBUG - shopping: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - shopping: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/shopping/stop_frequency_values_combined.csv\r\n", - "INFO - shopping: end estimation\r\n", - "INFO - stop_frequency running segment eatout with 316 chooser rows\r\n", - "INFO - Initialize Estimator for'eatout'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - eatout: estimate.write_spec: output/estimation_data_bundle/stop_frequency/eatout/stop_frequency_SPEC.csv\r\n", - "DEBUG - eatout: write_table write: stop_frequency_coefficients_eatout.csv\r\n", - "DEBUG - eatout: write_table cache: choosers\r\n", - "INFO - stop_frequency.eatout.simple_simulate Running adaptive_chunked_choosers with 316 choosers\r\n", - "INFO - Running chunk 1 of 1 with 316 of 316 choosers\r\n", - "DEBUG - eatout: write_table cache: expression_values\r\n", - "DEBUG - eatout: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - eatout: write_table cache: override_choices\r\n", - "DEBUG - eatout: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - eatout: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/eatout/stop_frequency_values_combined.csv\r\n", - "INFO - eatout: end estimation\r\n", - "INFO - stop_frequency running segment escort with 403 chooser rows\r\n", - "INFO - Initialize Estimator for'escort'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - escort: estimate.write_spec: output/estimation_data_bundle/stop_frequency/escort/stop_frequency_SPEC.csv\r\n", - "DEBUG - escort: write_table write: stop_frequency_coefficients_escort.csv\r\n", - "DEBUG - escort: write_table cache: choosers\r\n", - "INFO - stop_frequency.escort.simple_simulate Running adaptive_chunked_choosers with 403 choosers\r\n", - "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", - "DEBUG - escort: write_table cache: expression_values\r\n", - "DEBUG - escort: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - escort: write_table cache: override_choices\r\n", - "DEBUG - escort: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - escort: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/escort/stop_frequency_values_combined.csv\r\n", - "INFO - escort: end estimation\r\n", - "INFO - stop_frequency running segment othmaint with 393 chooser rows\r\n", - "INFO - Initialize Estimator for'othmaint'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - othmaint: estimate.write_spec: output/estimation_data_bundle/stop_frequency/othmaint/stop_frequency_SPEC.csv\r\n", - "DEBUG - othmaint: write_table write: stop_frequency_coefficients_othmaint.csv\r\n", - "DEBUG - othmaint: write_table cache: choosers\r\n", - "INFO - stop_frequency.othmaint.simple_simulate Running adaptive_chunked_choosers with 393 choosers\r\n", - "INFO - Running chunk 1 of 1 with 393 of 393 choosers\r\n", - "DEBUG - othmaint: write_table cache: expression_values\r\n", - "DEBUG - othmaint: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - othmaint: write_table cache: override_choices\r\n", - "DEBUG - othmaint: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - othmaint: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/othmaint/stop_frequency_values_combined.csv\r\n", - "INFO - othmaint: end estimation\r\n", - "INFO - stop_frequency running segment othdiscr with 564 chooser rows\r\n", - "INFO - Initialize Estimator for'othdiscr'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - othdiscr: estimate.write_spec: output/estimation_data_bundle/stop_frequency/othdiscr/stop_frequency_SPEC.csv\r\n", - "DEBUG - othdiscr: write_table write: stop_frequency_coefficients_othdiscr.csv\r\n", - "DEBUG - othdiscr: write_table cache: choosers\r\n", - "INFO - stop_frequency.othdiscr.simple_simulate Running adaptive_chunked_choosers with 564 choosers\r\n", - "INFO - Running chunk 1 of 1 with 564 of 564 choosers\r\n", - "DEBUG - othdiscr: write_table cache: expression_values\r\n", - "DEBUG - othdiscr: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - othdiscr: write_table cache: override_choices\r\n", - "DEBUG - othdiscr: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - othdiscr: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/othdiscr/stop_frequency_values_combined.csv\r\n", - "INFO - othdiscr: end estimation\r\n", - "INFO - stop_frequency running segment atwork with 464 chooser rows\r\n", - "INFO - Initialize Estimator for'atwork'\r\n", - "deleted output/estimation_data_bundle/stop_frequency/stop_frequency_model_settings.yaml\r\n", - "DEBUG - atwork: estimate.write_spec: output/estimation_data_bundle/stop_frequency/atwork/stop_frequency_SPEC.csv\r\n", - "DEBUG - atwork: write_table write: stop_frequency_coefficients_atwork.csv\r\n", - "DEBUG - atwork: write_table cache: choosers\r\n", - "INFO - stop_frequency.atwork.simple_simulate Running adaptive_chunked_choosers with 464 choosers\r\n", - "INFO - Running chunk 1 of 1 with 464 of 464 choosers\r\n", - "DEBUG - atwork: write_table cache: expression_values\r\n", - "DEBUG - atwork: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using tours.index\r\n", - "DEBUG - atwork: write_table cache: override_choices\r\n", - "DEBUG - atwork: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - atwork: write_omnibus_choosers: output/estimation_data_bundle/stop_frequency/atwork/stop_frequency_values_combined.csv\r\n", - "INFO - atwork: end estimation\r\n", - "INFO - stop_frequency top 10 value counts:\r\n", - "0out_0in 4040\r\n", - "0out_1in 636\r\n", - "1out_0in 421\r\n", - "0out_2in 178\r\n", - "1out_1in 164\r\n", - "0out_3in 91\r\n", - "2out_0in 62\r\n", - "1out_3in 50\r\n", - "1out_2in 44\r\n", - "2out_1in 31\r\n", - "Name: stop_frequency, dtype: int64\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "Time to execute step 'stop_frequency': 6.05 s\r\n", - "Total time to execute iteration 1 with iteration value None: 6.05 s\r\n", - "INFO - #run_model running step trip_purpose\r\n", - "Running step 'trip_purpose'\r\n", - "INFO - Initialize Estimator for'trip_purpose'\r\n", - "DEBUG - trip_purpose: write_table cache: choosers\r\n", - "DEBUG - trip_purpose: estimate.write_spec: output/estimation_data_bundle/trip_purpose/trip_purpose_PROBS_SPEC.csv\r\n", - "INFO - assign purpose to 5778 last outbound trips\r\n", - "INFO - assign purpose to 5778 last inbound trips\r\n", - "INFO - assign purpose to 2796 intermediate trips\r\n", - "INFO - trip_purpose Running adaptive_chunked_choosers with 2796 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2796 of 2796 choosers\r\n", - " primary_purpose outbound ... social othdiscr\r\n", - "trip_id ... \r\n", - "206561 univ True ... 0.048 0.050\r\n", - "9188185 univ True ... 0.048 0.050\r\n", - "9668377 univ True ... 0.048 0.050\r\n", - "9806797 univ False ... 0.067 0.113\r\n", - "10849837 univ False ... 0.067 0.113\r\n", - "... ... ... ... ... ...\r\n", - "2414259747 atwork True ... 0.004 0.014\r\n", - "2414259749 atwork False ... 0.019 0.018\r\n", - "2414488393 atwork True ... 0.004 0.014\r\n", - "2423389333 atwork False ... 0.019 0.018\r\n", - "2440963725 atwork False ... 0.019 0.018\r\n", - "\r\n", - "[2796 rows x 14 columns]\r\n", - "DEBUG - trip_purpose: write_table cache: probs\r\n", - "DEBUG - trip_purpose: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_purpose: write_table cache: override_choices\r\n", - "DEBUG - trip_purpose: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'choosers', 'probs']\r\n", - "DEBUG - trip_purpose: write_omnibus_choosers: output/estimation_data_bundle/trip_purpose/trip_purpose_values_combined.csv\r\n", - "INFO - trip_purpose: end estimation\r\n", - "Time to execute step 'trip_purpose': 0.33 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.33 s\r\n", - "INFO - #run_model running step trip_destination\r\n", - "Running step 'trip_destination'\r\n", - "INFO - Initialize Estimator for'trip_destination'\r\n", - "DEBUG - trip_destination: write_table write: trip_destination_coefficients.csv\r\n", - "DEBUG - trip_destination: estimate.write_spec: output/estimation_data_bundle/trip_destination/trip_destination_SPEC.csv\r\n", - "DEBUG - trip_destination: write_table write: size_terms\r\n", - "DEBUG - trip_destination: write_table write: landuse\r\n", - "INFO - Running trip_destination with 14352 trips\r\n", - "INFO - Running trip_destination.trip_num_1 with 2061 trips\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.atwork with 124 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.atwork.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 124 choosers\r\n", - "INFO - Running chunk 1 of 1 with 124 of 124 choosers\r\n", - "INFO - Running eval_interaction_utilities on 23560 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.atwork.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.atwork.trip_destination_sample : 0.09 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.atwork.compute_logsums with 23560 samples\r\n", - "INFO - trip_destination.trip_num_1.atwork.compute_logsums.od Running adaptive_chunked_choosers with 23560 choosers\r\n", - "INFO - Running chunk 1 of 1 with 23560 of 23560 choosers\r\n", - "INFO - trip_destination.trip_num_1.atwork.compute_logsums.dp Running adaptive_chunked_choosers with 23560 choosers\r\n", - "INFO - Running chunk 1 of 1 with 23560 of 23560 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.atwork.compute_logsums : 4.409 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 124 trips\r\n", - "INFO - trip_destination.trip_num_1.atwork.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 124 choosers and 23560 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 124 of 124 choosers\r\n", - "INFO - Running eval_interaction_utilities on 23560 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.atwork.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.atwork.trip_destination_simulate : 0.372 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.eatout with 66 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.eatout.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 66 choosers\r\n", - "INFO - Running chunk 1 of 1 with 66 of 66 choosers\r\n", - "INFO - Running eval_interaction_utilities on 12540 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.eatout.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.eatout.trip_destination_sample : 0.086 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.eatout.compute_logsums with 12540 samples\r\n", - "INFO - trip_destination.trip_num_1.eatout.compute_logsums.od Running adaptive_chunked_choosers with 12540 choosers\r\n", - "INFO - Running chunk 1 of 1 with 12540 of 12540 choosers\r\n", - "INFO - trip_destination.trip_num_1.eatout.compute_logsums.dp Running adaptive_chunked_choosers with 12540 choosers\r\n", - "INFO - Running chunk 1 of 1 with 12540 of 12540 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.eatout.compute_logsums : 3.345 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 66 trips\r\n", - "INFO - trip_destination.trip_num_1.eatout.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 66 choosers and 12540 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 66 of 66 choosers\r\n", - "INFO - Running eval_interaction_utilities on 12540 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.eatout.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.eatout.trip_destination_simulate : 0.266 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.escort with 117 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.escort.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.escort.sample.interaction_sample Running adaptive_chunked_choosers with 117 choosers\r\n", - "INFO - Running chunk 1 of 1 with 117 of 117 choosers\r\n", - "INFO - Running eval_interaction_utilities on 22230 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.escort.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.escort.trip_destination_sample : 0.119 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.escort.compute_logsums with 22230 samples\r\n", - "INFO - trip_destination.trip_num_1.escort.compute_logsums.od Running adaptive_chunked_choosers with 22230 choosers\r\n", - "INFO - Running chunk 1 of 1 with 22230 of 22230 choosers\r\n", - "INFO - trip_destination.trip_num_1.escort.compute_logsums.dp Running adaptive_chunked_choosers with 22230 choosers\r\n", - "INFO - Running chunk 1 of 1 with 22230 of 22230 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.escort.compute_logsums : 4.465 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 117 trips\r\n", - "INFO - trip_destination.trip_num_1.escort.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 117 choosers and 22230 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 117 of 117 choosers\r\n", - "INFO - Running eval_interaction_utilities on 22230 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.escort.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.escort.trip_destination_simulate : 0.431 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.othdiscr with 207 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 207 choosers\r\n", - "INFO - Running chunk 1 of 1 with 207 of 207 choosers\r\n", - "INFO - Running eval_interaction_utilities on 39330 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.othdiscr.trip_destination_sample : 0.117 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.othdiscr.compute_logsums with 39330 samples\r\n", - "INFO - trip_destination.trip_num_1.othdiscr.compute_logsums.od Running adaptive_chunked_choosers with 39330 choosers\r\n", - "INFO - Running chunk 1 of 1 with 39330 of 39330 choosers\r\n", - "INFO - trip_destination.trip_num_1.othdiscr.compute_logsums.dp Running adaptive_chunked_choosers with 39330 choosers\r\n", - "INFO - Running chunk 1 of 1 with 39330 of 39330 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.othdiscr.compute_logsums : 6.014 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 207 trips\r\n", - "INFO - trip_destination.trip_num_1.othdiscr.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 207 choosers and 39330 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 207 of 207 choosers\r\n", - "INFO - Running eval_interaction_utilities on 39330 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.othdiscr.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.othdiscr.trip_destination_simulate : 0.567 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.othmaint with 122 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 122 choosers\r\n", - "INFO - Running chunk 1 of 1 with 122 of 122 choosers\r\n", - "INFO - Running eval_interaction_utilities on 23180 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.othmaint.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.othmaint.trip_destination_sample : 0.095 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.othmaint.compute_logsums with 23180 samples\r\n", - "INFO - trip_destination.trip_num_1.othmaint.compute_logsums.od Running adaptive_chunked_choosers with 23180 choosers\r\n", - "INFO - Running chunk 1 of 1 with 23180 of 23180 choosers\r\n", - "INFO - trip_destination.trip_num_1.othmaint.compute_logsums.dp Running adaptive_chunked_choosers with 23180 choosers\r\n", - "INFO - Running chunk 1 of 1 with 23180 of 23180 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.othmaint.compute_logsums : 4.333 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 122 trips\r\n", - "INFO - trip_destination.trip_num_1.othmaint.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 122 choosers and 23180 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 122 of 122 choosers\r\n", - "INFO - Running eval_interaction_utilities on 23180 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.othmaint.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.othmaint.trip_destination_simulate : 0.417 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.school with 119 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.school.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.school.sample.interaction_sample Running adaptive_chunked_choosers with 119 choosers\r\n", - "INFO - Running chunk 1 of 1 with 119 of 119 choosers\r\n", - "INFO - Running eval_interaction_utilities on 22610 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.school.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.school.trip_destination_sample : 0.091 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.school.compute_logsums with 22610 samples\r\n", - "INFO - trip_destination.trip_num_1.school.compute_logsums.od Running adaptive_chunked_choosers with 22610 choosers\r\n", - "INFO - Running chunk 1 of 1 with 22610 of 22610 choosers\r\n", - "INFO - trip_destination.trip_num_1.school.compute_logsums.dp Running adaptive_chunked_choosers with 22610 choosers\r\n", - "INFO - Running chunk 1 of 1 with 22610 of 22610 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.school.compute_logsums : 4.473 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 119 trips\r\n", - "INFO - trip_destination.trip_num_1.school.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 119 choosers and 22610 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 119 of 119 choosers\r\n", - "INFO - Running eval_interaction_utilities on 22610 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.school.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.school.trip_destination_simulate : 0.423 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.shopping with 322 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.shopping.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 322 choosers\r\n", - "INFO - Running chunk 1 of 1 with 322 of 322 choosers\r\n", - "INFO - Running eval_interaction_utilities on 61180 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.shopping.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.shopping.trip_destination_sample : 0.16 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.shopping.compute_logsums with 61180 samples\r\n", - "INFO - trip_destination.trip_num_1.shopping.compute_logsums.od Running adaptive_chunked_choosers with 61180 choosers\r\n", - "INFO - Running chunk 1 of 1 with 61180 of 61180 choosers\r\n", - "INFO - trip_destination.trip_num_1.shopping.compute_logsums.dp Running adaptive_chunked_choosers with 61180 choosers\r\n", - "INFO - Running chunk 1 of 1 with 61180 of 61180 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.shopping.compute_logsums : 8.168 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 322 trips\r\n", - "INFO - trip_destination.trip_num_1.shopping.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 322 choosers and 61180 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 322 of 322 choosers\r\n", - "INFO - Running eval_interaction_utilities on 61180 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.shopping.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.shopping.trip_destination_simulate : 0.892 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.social with 50 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.social.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.social.sample.interaction_sample Running adaptive_chunked_choosers with 50 choosers\r\n", - "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", - "INFO - Running eval_interaction_utilities on 9500 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.social.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.social.trip_destination_sample : 0.074 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.social.compute_logsums with 9500 samples\r\n", - "INFO - trip_destination.trip_num_1.social.compute_logsums.od Running adaptive_chunked_choosers with 9500 choosers\r\n", - "INFO - Running chunk 1 of 1 with 9500 of 9500 choosers\r\n", - "INFO - trip_destination.trip_num_1.social.compute_logsums.dp Running adaptive_chunked_choosers with 9500 choosers\r\n", - "INFO - Running chunk 1 of 1 with 9500 of 9500 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.social.compute_logsums : 2.802 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 50 trips\r\n", - "INFO - trip_destination.trip_num_1.social.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 50 choosers and 9500 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 50 of 50 choosers\r\n", - "INFO - Running eval_interaction_utilities on 9500 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.social.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.social.trip_destination_simulate : 0.285 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.univ with 75 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.univ.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.univ.sample.interaction_sample Running adaptive_chunked_choosers with 75 choosers\r\n", - "INFO - Running chunk 1 of 1 with 75 of 75 choosers\r\n", - "INFO - Running eval_interaction_utilities on 14250 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.univ.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.univ.trip_destination_sample : 0.084 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.univ.compute_logsums with 14250 samples\r\n", - "INFO - trip_destination.trip_num_1.univ.compute_logsums.od Running adaptive_chunked_choosers with 14250 choosers\r\n", - "INFO - Running chunk 1 of 1 with 14250 of 14250 choosers\r\n", - "INFO - trip_destination.trip_num_1.univ.compute_logsums.dp Running adaptive_chunked_choosers with 14250 choosers\r\n", - "INFO - Running chunk 1 of 1 with 14250 of 14250 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.univ.compute_logsums : 3.17 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 75 trips\r\n", - "INFO - trip_destination.trip_num_1.univ.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 75 choosers and 14250 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 75 of 75 choosers\r\n", - "INFO - Running eval_interaction_utilities on 14250 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.univ.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.univ.trip_destination_simulate : 0.31 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_1.work with 859 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.work.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_1.work.sample.interaction_sample Running adaptive_chunked_choosers with 859 choosers\r\n", - "INFO - Running chunk 1 of 1 with 859 of 859 choosers\r\n", - "INFO - Running eval_interaction_utilities on 163210 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_1.work.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_1.work.trip_destination_sample : 0.278 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_1.work.compute_logsums with 163210 samples\r\n", - "INFO - trip_destination.trip_num_1.work.compute_logsums.od Running adaptive_chunked_choosers with 163210 choosers\r\n", - "INFO - Running chunk 1 of 1 with 163210 of 163210 choosers\r\n", - "INFO - trip_destination.trip_num_1.work.compute_logsums.dp Running adaptive_chunked_choosers with 163210 choosers\r\n", - "INFO - Running chunk 1 of 1 with 163210 of 163210 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_1.work.compute_logsums : 17.277 seconds (0.3 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 859 trips\r\n", - "INFO - trip_destination.trip_num_1.work.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 859 choosers and 163210 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 859 of 859 choosers\r\n", - "INFO - Running eval_interaction_utilities on 163210 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_1.work.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_1.work.trip_destination_simulate : 3.053 seconds (0.1 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2 with 542 trips\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.atwork with 18 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.atwork.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 18 choosers\r\n", - "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3420 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.atwork.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.atwork.trip_destination_sample : 0.07 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.atwork.compute_logsums with 3420 samples\r\n", - "INFO - trip_destination.trip_num_2.atwork.compute_logsums.od Running adaptive_chunked_choosers with 3420 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3420 of 3420 choosers\r\n", - "INFO - trip_destination.trip_num_2.atwork.compute_logsums.dp Running adaptive_chunked_choosers with 3420 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3420 of 3420 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.atwork.compute_logsums : 2.279 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 18 trips\r\n", - "INFO - trip_destination.trip_num_2.atwork.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 18 choosers and 3420 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 18 of 18 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3420 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.atwork.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.atwork.trip_destination_simulate : 0.383 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.eatout with 11 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.eatout.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 11 choosers\r\n", - "INFO - Running chunk 1 of 1 with 11 of 11 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2090 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.eatout.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.eatout.trip_destination_sample : 0.084 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.eatout.compute_logsums with 2090 samples\r\n", - "INFO - trip_destination.trip_num_2.eatout.compute_logsums.od Running adaptive_chunked_choosers with 2090 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2090 of 2090 choosers\r\n", - "INFO - trip_destination.trip_num_2.eatout.compute_logsums.dp Running adaptive_chunked_choosers with 2090 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2090 of 2090 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.eatout.compute_logsums : 2.19 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 11 trips\r\n", - "INFO - trip_destination.trip_num_2.eatout.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 11 choosers and 2090 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 11 of 11 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2090 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.eatout.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.eatout.trip_destination_simulate : 0.486 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.escort with 32 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.escort.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.escort.sample.interaction_sample Running adaptive_chunked_choosers with 32 choosers\r\n", - "INFO - Running chunk 1 of 1 with 32 of 32 choosers\r\n", - "INFO - Running eval_interaction_utilities on 6080 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.escort.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.escort.trip_destination_sample : 0.092 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.escort.compute_logsums with 6080 samples\r\n", - "INFO - trip_destination.trip_num_2.escort.compute_logsums.od Running adaptive_chunked_choosers with 6080 choosers\r\n", - "INFO - Running chunk 1 of 1 with 6080 of 6080 choosers\r\n", - "INFO - trip_destination.trip_num_2.escort.compute_logsums.dp Running adaptive_chunked_choosers with 6080 choosers\r\n", - "INFO - Running chunk 1 of 1 with 6080 of 6080 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.escort.compute_logsums : 2.883 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 32 trips\r\n", - "INFO - trip_destination.trip_num_2.escort.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 32 choosers and 6080 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 32 of 32 choosers\r\n", - "INFO - Running eval_interaction_utilities on 6080 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.escort.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.escort.trip_destination_simulate : 0.844 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.othdiscr with 56 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 56 choosers\r\n", - "INFO - Running chunk 1 of 1 with 56 of 56 choosers\r\n", - "INFO - Running eval_interaction_utilities on 10640 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.othdiscr.trip_destination_sample : 0.15 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.othdiscr.compute_logsums with 10640 samples\r\n", - "INFO - trip_destination.trip_num_2.othdiscr.compute_logsums.od Running adaptive_chunked_choosers with 10640 choosers\r\n", - "INFO - Running chunk 1 of 1 with 10640 of 10640 choosers\r\n", - "INFO - trip_destination.trip_num_2.othdiscr.compute_logsums.dp Running adaptive_chunked_choosers with 10640 choosers\r\n", - "INFO - Running chunk 1 of 1 with 10640 of 10640 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.othdiscr.compute_logsums : 4.791 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 56 trips\r\n", - "INFO - trip_destination.trip_num_2.othdiscr.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 56 choosers and 10640 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 56 of 56 choosers\r\n", - "INFO - Running eval_interaction_utilities on 10640 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.othdiscr.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.othdiscr.trip_destination_simulate : 0.442 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.othmaint with 53 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 53 choosers\r\n", - "INFO - Running chunk 1 of 1 with 53 of 53 choosers\r\n", - "INFO - Running eval_interaction_utilities on 10070 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.othmaint.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.othmaint.trip_destination_sample : 0.07 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.othmaint.compute_logsums with 10070 samples\r\n", - "INFO - trip_destination.trip_num_2.othmaint.compute_logsums.od Running adaptive_chunked_choosers with 10070 choosers\r\n", - "INFO - Running chunk 1 of 1 with 10070 of 10070 choosers\r\n", - "INFO - trip_destination.trip_num_2.othmaint.compute_logsums.dp Running adaptive_chunked_choosers with 10070 choosers\r\n", - "INFO - Running chunk 1 of 1 with 10070 of 10070 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.othmaint.compute_logsums : 2.728 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 53 trips\r\n", - "INFO - trip_destination.trip_num_2.othmaint.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 53 choosers and 10070 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 53 of 53 choosers\r\n", - "INFO - Running eval_interaction_utilities on 10070 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.othmaint.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.othmaint.trip_destination_simulate : 0.486 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.school with 29 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.school.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.school.sample.interaction_sample Running adaptive_chunked_choosers with 29 choosers\r\n", - "INFO - Running chunk 1 of 1 with 29 of 29 choosers\r\n", - "INFO - Running eval_interaction_utilities on 5510 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.school.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.school.trip_destination_sample : 0.055 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.school.compute_logsums with 5510 samples\r\n", - "INFO - trip_destination.trip_num_2.school.compute_logsums.od Running adaptive_chunked_choosers with 5510 choosers\r\n", - "INFO - Running chunk 1 of 1 with 5510 of 5510 choosers\r\n", - "INFO - trip_destination.trip_num_2.school.compute_logsums.dp Running adaptive_chunked_choosers with 5510 choosers\r\n", - "INFO - Running chunk 1 of 1 with 5510 of 5510 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.school.compute_logsums : 2.617 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 29 trips\r\n", - "INFO - trip_destination.trip_num_2.school.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 29 choosers and 5510 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 29 of 29 choosers\r\n", - "INFO - Running eval_interaction_utilities on 5510 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.school.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.school.trip_destination_simulate : 0.609 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.shopping with 80 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.shopping.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 80 choosers\r\n", - "INFO - Running chunk 1 of 1 with 80 of 80 choosers\r\n", - "INFO - Running eval_interaction_utilities on 15200 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.shopping.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.shopping.trip_destination_sample : 0.073 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.shopping.compute_logsums with 15200 samples\r\n", - "INFO - trip_destination.trip_num_2.shopping.compute_logsums.od Running adaptive_chunked_choosers with 15200 choosers\r\n", - "INFO - Running chunk 1 of 1 with 15200 of 15200 choosers\r\n", - "INFO - trip_destination.trip_num_2.shopping.compute_logsums.dp Running adaptive_chunked_choosers with 15200 choosers\r\n", - "INFO - Running chunk 1 of 1 with 15200 of 15200 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.shopping.compute_logsums : 3.393 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 80 trips\r\n", - "INFO - trip_destination.trip_num_2.shopping.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 80 choosers and 15200 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 80 of 80 choosers\r\n", - "INFO - Running eval_interaction_utilities on 15200 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.shopping.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.shopping.trip_destination_simulate : 0.622 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.social with 7 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.social.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.social.sample.interaction_sample Running adaptive_chunked_choosers with 7 choosers\r\n", - "INFO - Running chunk 1 of 1 with 7 of 7 choosers\r\n", - "INFO - Running eval_interaction_utilities on 1330 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.social.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.social.trip_destination_sample : 0.076 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.social.compute_logsums with 1330 samples\r\n", - "INFO - trip_destination.trip_num_2.social.compute_logsums.od Running adaptive_chunked_choosers with 1330 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1330 of 1330 choosers\r\n", - "INFO - trip_destination.trip_num_2.social.compute_logsums.dp Running adaptive_chunked_choosers with 1330 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1330 of 1330 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.social.compute_logsums : 2.247 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 7 trips\r\n", - "INFO - trip_destination.trip_num_2.social.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 7 choosers and 1330 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 7 of 7 choosers\r\n", - "INFO - Running eval_interaction_utilities on 1330 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.social.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.social.trip_destination_simulate : 0.414 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.univ with 20 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.univ.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.univ.sample.interaction_sample Running adaptive_chunked_choosers with 20 choosers\r\n", - "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3800 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.univ.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.univ.trip_destination_sample : 0.054 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.univ.compute_logsums with 3800 samples\r\n", - "INFO - trip_destination.trip_num_2.univ.compute_logsums.od Running adaptive_chunked_choosers with 3800 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3800 of 3800 choosers\r\n", - "INFO - trip_destination.trip_num_2.univ.compute_logsums.dp Running adaptive_chunked_choosers with 3800 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3800 of 3800 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.univ.compute_logsums : 2.624 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 20 trips\r\n", - "INFO - trip_destination.trip_num_2.univ.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 20 choosers and 3800 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 20 of 20 choosers\r\n", - "INFO - Running eval_interaction_utilities on 3800 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.univ.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.univ.trip_destination_simulate : 0.485 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_2.work with 236 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.work.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_2.work.sample.interaction_sample Running adaptive_chunked_choosers with 236 choosers\r\n", - "INFO - Running chunk 1 of 1 with 236 of 236 choosers\r\n", - "INFO - Running eval_interaction_utilities on 44840 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_2.work.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_2.work.trip_destination_sample : 0.121 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_2.work.compute_logsums with 44840 samples\r\n", - "INFO - trip_destination.trip_num_2.work.compute_logsums.od Running adaptive_chunked_choosers with 44840 choosers\r\n", - "INFO - Running chunk 1 of 1 with 44840 of 44840 choosers\r\n", - "INFO - trip_destination.trip_num_2.work.compute_logsums.dp Running adaptive_chunked_choosers with 44840 choosers\r\n", - "INFO - Running chunk 1 of 1 with 44840 of 44840 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_2.work.compute_logsums : 6.123 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 236 trips\r\n", - "INFO - trip_destination.trip_num_2.work.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 236 choosers and 44840 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 236 of 236 choosers\r\n", - "INFO - Running eval_interaction_utilities on 44840 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_2.work.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_2.work.trip_destination_simulate : 0.961 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3 with 193 trips\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.atwork with 3 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.atwork.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.atwork.sample.interaction_sample Running adaptive_chunked_choosers with 3 choosers\r\n", - "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", - "INFO - Running eval_interaction_utilities on 570 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.atwork.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.atwork.trip_destination_sample : 0.062 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.atwork.compute_logsums with 570 samples\r\n", - "INFO - trip_destination.trip_num_3.atwork.compute_logsums.od Running adaptive_chunked_choosers with 570 choosers\r\n", - "INFO - Running chunk 1 of 1 with 570 of 570 choosers\r\n", - "INFO - trip_destination.trip_num_3.atwork.compute_logsums.dp Running adaptive_chunked_choosers with 570 choosers\r\n", - "INFO - Running chunk 1 of 1 with 570 of 570 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.atwork.compute_logsums : 1.823 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 3 trips\r\n", - "INFO - trip_destination.trip_num_3.atwork.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 3 choosers and 570 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 3 of 3 choosers\r\n", - "INFO - Running eval_interaction_utilities on 570 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.atwork.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.atwork.trip_destination_simulate : 0.433 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.eatout with 4 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.eatout.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.eatout.sample.interaction_sample Running adaptive_chunked_choosers with 4 choosers\r\n", - "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", - "INFO - Running eval_interaction_utilities on 760 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.eatout.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.eatout.trip_destination_sample : 0.069 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.eatout.compute_logsums with 760 samples\r\n", - "INFO - trip_destination.trip_num_3.eatout.compute_logsums.od Running adaptive_chunked_choosers with 760 choosers\r\n", - "INFO - Running chunk 1 of 1 with 760 of 760 choosers\r\n", - "INFO - trip_destination.trip_num_3.eatout.compute_logsums.dp Running adaptive_chunked_choosers with 760 choosers\r\n", - "INFO - Running chunk 1 of 1 with 760 of 760 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.eatout.compute_logsums : 1.92 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 4 trips\r\n", - "INFO - trip_destination.trip_num_3.eatout.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 4 choosers and 760 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 4 of 4 choosers\r\n", - "INFO - Running eval_interaction_utilities on 760 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.eatout.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.eatout.trip_destination_simulate : 0.447 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.escort with 12 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.escort.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.escort.sample.interaction_sample Running adaptive_chunked_choosers with 12 choosers\r\n", - "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2280 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.escort.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.escort.trip_destination_sample : 0.048 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.escort.compute_logsums with 2280 samples\r\n", - "INFO - trip_destination.trip_num_3.escort.compute_logsums.od Running adaptive_chunked_choosers with 2280 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2280 of 2280 choosers\r\n", - "INFO - trip_destination.trip_num_3.escort.compute_logsums.dp Running adaptive_chunked_choosers with 2280 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2280 of 2280 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.escort.compute_logsums : 2.102 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 12 trips\r\n", - "INFO - trip_destination.trip_num_3.escort.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 12 choosers and 2280 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 12 of 12 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2280 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.escort.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.escort.trip_destination_simulate : 0.462 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.othdiscr with 22 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.othdiscr.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.othdiscr.sample.interaction_sample Running adaptive_chunked_choosers with 22 choosers\r\n", - "INFO - Running chunk 1 of 1 with 22 of 22 choosers\r\n", - "INFO - Running eval_interaction_utilities on 4180 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.othdiscr.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.othdiscr.trip_destination_sample : 0.071 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.othdiscr.compute_logsums with 4180 samples\r\n", - "INFO - trip_destination.trip_num_3.othdiscr.compute_logsums.od Running adaptive_chunked_choosers with 4180 choosers\r\n", - "INFO - Running chunk 1 of 1 with 4180 of 4180 choosers\r\n", - "INFO - trip_destination.trip_num_3.othdiscr.compute_logsums.dp Running adaptive_chunked_choosers with 4180 choosers\r\n", - "INFO - Running chunk 1 of 1 with 4180 of 4180 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.othdiscr.compute_logsums : 2.22 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 22 trips\r\n", - "INFO - trip_destination.trip_num_3.othdiscr.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 22 choosers and 4180 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 22 of 22 choosers\r\n", - "INFO - Running eval_interaction_utilities on 4180 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.othdiscr.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.othdiscr.trip_destination_simulate : 0.506 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.othmaint with 14 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.othmaint.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.othmaint.sample.interaction_sample Running adaptive_chunked_choosers with 14 choosers\r\n", - "INFO - Running chunk 1 of 1 with 14 of 14 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2660 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.othmaint.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.othmaint.trip_destination_sample : 0.066 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.othmaint.compute_logsums with 2660 samples\r\n", - "INFO - trip_destination.trip_num_3.othmaint.compute_logsums.od Running adaptive_chunked_choosers with 2660 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2660 of 2660 choosers\r\n", - "INFO - trip_destination.trip_num_3.othmaint.compute_logsums.dp Running adaptive_chunked_choosers with 2660 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2660 of 2660 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.othmaint.compute_logsums : 2.088 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 14 trips\r\n", - "INFO - trip_destination.trip_num_3.othmaint.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 14 choosers and 2660 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 14 of 14 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2660 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.othmaint.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.othmaint.trip_destination_simulate : 0.483 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.school with 13 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.school.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.school.sample.interaction_sample Running adaptive_chunked_choosers with 13 choosers\r\n", - "INFO - Running chunk 1 of 1 with 13 of 13 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2470 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.school.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.school.trip_destination_sample : 0.052 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.school.compute_logsums with 2470 samples\r\n", - "INFO - trip_destination.trip_num_3.school.compute_logsums.od Running adaptive_chunked_choosers with 2470 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2470 of 2470 choosers\r\n", - "INFO - trip_destination.trip_num_3.school.compute_logsums.dp Running adaptive_chunked_choosers with 2470 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2470 of 2470 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.school.compute_logsums : 2.017 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 13 trips\r\n", - "INFO - trip_destination.trip_num_3.school.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 13 choosers and 2470 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 13 of 13 choosers\r\n", - "INFO - Running eval_interaction_utilities on 2470 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.school.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.school.trip_destination_simulate : 0.458 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.shopping with 28 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.shopping.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 28 choosers\r\n", - "INFO - Running chunk 1 of 1 with 28 of 28 choosers\r\n", - "INFO - Running eval_interaction_utilities on 5320 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.shopping.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.shopping.trip_destination_sample : 0.078 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.shopping.compute_logsums with 5320 samples\r\n", - "INFO - trip_destination.trip_num_3.shopping.compute_logsums.od Running adaptive_chunked_choosers with 5320 choosers\r\n", - "INFO - Running chunk 1 of 1 with 5320 of 5320 choosers\r\n", - "INFO - trip_destination.trip_num_3.shopping.compute_logsums.dp Running adaptive_chunked_choosers with 5320 choosers\r\n", - "INFO - Running chunk 1 of 1 with 5320 of 5320 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.shopping.compute_logsums : 2.804 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 28 trips\r\n", - "INFO - trip_destination.trip_num_3.shopping.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 28 choosers and 5320 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 28 of 28 choosers\r\n", - "INFO - Running eval_interaction_utilities on 5320 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.shopping.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.shopping.trip_destination_simulate : 0.513 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.social with 2 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.social.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.social.sample.interaction_sample Running adaptive_chunked_choosers with 2 choosers\r\n", - "INFO - Running chunk 1 of 1 with 2 of 2 choosers\r\n", - "INFO - Running eval_interaction_utilities on 380 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.social.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.social.trip_destination_sample : 0.07 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.social.compute_logsums with 380 samples\r\n", - "INFO - trip_destination.trip_num_3.social.compute_logsums.od Running adaptive_chunked_choosers with 380 choosers\r\n", - "INFO - Running chunk 1 of 1 with 380 of 380 choosers\r\n", - "INFO - trip_destination.trip_num_3.social.compute_logsums.dp Running adaptive_chunked_choosers with 380 choosers\r\n", - "INFO - Running chunk 1 of 1 with 380 of 380 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.social.compute_logsums : 1.945 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 2 trips\r\n", - "INFO - trip_destination.trip_num_3.social.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 2 choosers and 380 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 2 of 2 choosers\r\n", - "INFO - Running eval_interaction_utilities on 380 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.social.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.social.trip_destination_simulate : 0.472 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.univ with 10 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.univ.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.univ.sample.interaction_sample Running adaptive_chunked_choosers with 10 choosers\r\n", - "INFO - Running chunk 1 of 1 with 10 of 10 choosers\r\n", - "INFO - Running eval_interaction_utilities on 1900 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.univ.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.univ.trip_destination_sample : 0.075 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.univ.compute_logsums with 1900 samples\r\n", - "INFO - trip_destination.trip_num_3.univ.compute_logsums.od Running adaptive_chunked_choosers with 1900 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1900 of 1900 choosers\r\n", - "INFO - trip_destination.trip_num_3.univ.compute_logsums.dp Running adaptive_chunked_choosers with 1900 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1900 of 1900 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.univ.compute_logsums : 1.925 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 10 trips\r\n", - "INFO - trip_destination.trip_num_3.univ.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 10 choosers and 1900 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 10 of 10 choosers\r\n", - "INFO - Running eval_interaction_utilities on 1900 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.univ.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.univ.trip_destination_simulate : 0.53 seconds (0.0 minutes)\r\n", - "INFO - choose_trip_destination trip_destination.trip_num_3.work with 85 trips\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.work.sample using unsampled alternatives short_circuit_choices\r\n", - "INFO - trip_destination.trip_num_3.work.sample.interaction_sample Running adaptive_chunked_choosers with 85 choosers\r\n", - "INFO - Running chunk 1 of 1 with 85 of 85 choosers\r\n", - "INFO - Running eval_interaction_utilities on 16150 rows\r\n", - "INFO - Estimation mode for trip_destination.trip_num_3.work.sample.interaction_sample using unsampled alternatives\r\n", - "INFO - Time to execute trip_destination.trip_num_3.work.trip_destination_sample : 0.083 seconds (0.0 minutes)\r\n", - "INFO - Running trip_destination.trip_num_3.work.compute_logsums with 16150 samples\r\n", - "INFO - trip_destination.trip_num_3.work.compute_logsums.od Running adaptive_chunked_choosers with 16150 choosers\r\n", - "INFO - Running chunk 1 of 1 with 16150 of 16150 choosers\r\n", - "INFO - trip_destination.trip_num_3.work.compute_logsums.dp Running adaptive_chunked_choosers with 16150 choosers\r\n", - "INFO - Running chunk 1 of 1 with 16150 of 16150 choosers\r\n", - "INFO - Time to execute trip_destination.trip_num_3.work.compute_logsums : 3.316 seconds (0.1 minutes)\r\n", - "DEBUG - trip_destination: write_table cache: choosers\r\n", - "INFO - Running trip_destination_simulate with 85 trips\r\n", - "INFO - trip_destination.trip_num_3.work.trip_dest_simulate.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 85 choosers and 16150 alternatives\r\n", - "INFO - Running chunk 1 of 1 with 85 of 85 choosers\r\n", - "INFO - Running eval_interaction_utilities on 16150 rows\r\n", - "INFO - trip_destination: eval_interaction_utilities write_interaction_expression_values trip_destination.trip_num_3.work.trip_dest_simulate.interaction_sample_simulate.eval_interaction_utils\r\n", - "DEBUG - trip_destination: write_table cache: interaction_expression_values\r\n", - "DEBUG - trip_destination: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_destination: write_table cache: override_choices\r\n", - "INFO - Time to execute trip_destination.trip_num_3.work.trip_destination_simulate : 0.63 seconds (0.0 minutes)\r\n", - "DEBUG - trip_destination: write_omnibus_table: choosers_combined table_names: ['choices', 'override_choices', 'choosers']\r\n", - "DEBUG - trip_destination: write_omnibus_choosers: output/estimation_data_bundle/trip_destination/trip_destination_choosers_combined.csv\r\n", - "DEBUG - trip_destination: write_omnibus_table: alternatives_combined table_names: ['interaction_sample_alternatives', 'interaction_expression_values']\r\n", - "WARNING - trip_destination: write_omnibus_table: alternatives_combined table 'interaction_sample_alternatives' not found\r\n", - "DEBUG - trip_destination: write_omnibus_choosers: output/estimation_data_bundle/trip_destination/trip_destination_alternatives_combined.csv\r\n", - "INFO - trip_destination: end estimation\r\n", - "Time to execute step 'trip_destination': 141.72 s\r\n", - "Total time to execute iteration 1 with iteration value None: 141.72 s\r\n", - "INFO - #run_model running step trip_scheduling\r\n", - "Running step 'trip_scheduling'\r\n", - "INFO - Initialize Estimator for'trip_scheduling'\r\n", - "DEBUG - trip_scheduling: estimate.write_spec: output/estimation_data_bundle/trip_scheduling/trip_scheduling_PROBS_SPEC.csv\r\n", - "DEBUG - trip_scheduling: write_table cache: choosers\r\n", - "INFO - trip_scheduling Running chunk 1 of 1 with 5778 of 5778 choosers\r\n", - "INFO - trip_scheduling.i1 scheduling 14352 trips within chunk 1\r\n", - "INFO - trip_scheduling.i1 236 failed\r\n", - "INFO - trip_scheduling.i2 scheduling 686 trips within chunk 1\r\n", - "INFO - trip_scheduling.i2 388 failed\r\n", - "INFO - trip_scheduling.i3 scheduling 622 trips within chunk 1\r\n", - "INFO - trip_scheduling.i3 376 failed\r\n", - "INFO - trip_scheduling.i4 scheduling 585 trips within chunk 1\r\n", - "INFO - trip_scheduling.i4 373 failed\r\n", - "INFO - trip_scheduling.i5 scheduling 563 trips within chunk 1\r\n", - "INFO - trip_scheduling.i5 370 failed\r\n", - "INFO - trip_scheduling.i6 scheduling 553 trips within chunk 1\r\n", - "INFO - trip_scheduling.i6 367 failed\r\n", - "INFO - trip_scheduling.i7 scheduling 546 trips within chunk 1\r\n", - "INFO - trip_scheduling.i7 367 failed\r\n", - "INFO - trip_scheduling.i8 scheduling 546 trips within chunk 1\r\n", - "INFO - trip_scheduling.i8 366 failed\r\n", - "INFO - trip_scheduling.i9 scheduling 542 trips within chunk 1\r\n", - "INFO - trip_scheduling.i9 365 failed\r\n", - "INFO - trip_scheduling.i10 scheduling 542 trips within chunk 1\r\n", - "INFO - trip_scheduling.i10 366 failed\r\n", - "INFO - trip_scheduling.i11 scheduling 542 trips within chunk 1\r\n", - "INFO - trip_scheduling.i11 363 failed\r\n", - "INFO - trip_scheduling.i12 scheduling 539 trips within chunk 1\r\n", - "INFO - trip_scheduling.i12 366 failed\r\n", - "INFO - trip_scheduling.i13 scheduling 539 trips within chunk 1\r\n", - "INFO - trip_scheduling.i13 364 failed\r\n", - "INFO - trip_scheduling.i14 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i14 361 failed\r\n", - "INFO - trip_scheduling.i15 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i15 363 failed\r\n", - "INFO - trip_scheduling.i16 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i16 364 failed\r\n", - "INFO - trip_scheduling.i17 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i17 365 failed\r\n", - "INFO - trip_scheduling.i18 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i18 363 failed\r\n", - "INFO - trip_scheduling.i19 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i19 363 failed\r\n", - "INFO - trip_scheduling.i20 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i20 362 failed\r\n", - "INFO - trip_scheduling.i21 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i21 365 failed\r\n", - "INFO - trip_scheduling.i22 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i22 363 failed\r\n", - "INFO - trip_scheduling.i23 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i23 365 failed\r\n", - "INFO - trip_scheduling.i24 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i24 365 failed\r\n", - "INFO - trip_scheduling.i25 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i25 363 failed\r\n", - "INFO - trip_scheduling.i26 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i26 364 failed\r\n", - "INFO - trip_scheduling.i27 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i27 365 failed\r\n", - "INFO - trip_scheduling.i28 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i28 365 failed\r\n", - "INFO - trip_scheduling.i29 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i29 363 failed\r\n", - "INFO - trip_scheduling.i30 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i30 363 failed\r\n", - "INFO - trip_scheduling.i31 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i31 365 failed\r\n", - "INFO - trip_scheduling.i32 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i32 364 failed\r\n", - "INFO - trip_scheduling.i33 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i33 364 failed\r\n", - "INFO - trip_scheduling.i34 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i34 364 failed\r\n", - "INFO - trip_scheduling.i35 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i35 363 failed\r\n", - "INFO - trip_scheduling.i36 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i36 363 failed\r\n", - "INFO - trip_scheduling.i37 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i37 364 failed\r\n", - "INFO - trip_scheduling.i38 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i38 364 failed\r\n", - "INFO - trip_scheduling.i39 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i39 364 failed\r\n", - "INFO - trip_scheduling.i40 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i40 365 failed\r\n", - "INFO - trip_scheduling.i41 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i41 365 failed\r\n", - "INFO - trip_scheduling.i42 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i42 365 failed\r\n", - "INFO - trip_scheduling.i43 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i43 366 failed\r\n", - "INFO - trip_scheduling.i44 scheduling 536 trips within chunk 1\r\n", - "INFO - trip_scheduling.i44 363 failed\r\n", - "INFO - trip_scheduling.i45 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i45 363 failed\r\n", - "INFO - trip_scheduling.i46 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i46 362 failed\r\n", - "INFO - trip_scheduling.i47 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i47 363 failed\r\n", - "INFO - trip_scheduling.i48 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i48 364 failed\r\n", - "INFO - trip_scheduling.i49 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i49 362 failed\r\n", - "INFO - trip_scheduling.i50 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i50 363 failed\r\n", - "INFO - trip_scheduling.i51 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i51 363 failed\r\n", - "INFO - trip_scheduling.i52 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i52 363 failed\r\n", - "INFO - trip_scheduling.i53 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i53 363 failed\r\n", - "INFO - trip_scheduling.i54 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i54 362 failed\r\n", - "INFO - trip_scheduling.i55 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i55 363 failed\r\n", - "INFO - trip_scheduling.i56 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i56 364 failed\r\n", - "INFO - trip_scheduling.i57 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i57 364 failed\r\n", - "INFO - trip_scheduling.i58 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i58 363 failed\r\n", - "INFO - trip_scheduling.i59 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i59 364 failed\r\n", - "INFO - trip_scheduling.i60 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i60 362 failed\r\n", - "INFO - trip_scheduling.i61 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i61 365 failed\r\n", - "INFO - trip_scheduling.i62 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i62 361 failed\r\n", - "INFO - trip_scheduling.i63 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i63 362 failed\r\n", - "INFO - trip_scheduling.i64 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i64 364 failed\r\n", - "INFO - trip_scheduling.i65 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i65 363 failed\r\n", - "INFO - trip_scheduling.i66 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i66 364 failed\r\n", - "INFO - trip_scheduling.i67 scheduling 533 trips within chunk 1\r\n", - "INFO - trip_scheduling.i67 363 failed\r\n", - "INFO - trip_scheduling.i68 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i68 364 failed\r\n", - "INFO - trip_scheduling.i69 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i69 362 failed\r\n", - "INFO - trip_scheduling.i70 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i70 360 failed\r\n", - "INFO - trip_scheduling.i71 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i71 361 failed\r\n", - "INFO - trip_scheduling.i72 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i72 363 failed\r\n", - "INFO - trip_scheduling.i73 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i73 363 failed\r\n", - "INFO - trip_scheduling.i74 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i74 363 failed\r\n", - "INFO - trip_scheduling.i75 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i75 362 failed\r\n", - "INFO - trip_scheduling.i76 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i76 362 failed\r\n", - "INFO - trip_scheduling.i77 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i77 363 failed\r\n", - "INFO - trip_scheduling.i78 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i78 361 failed\r\n", - "INFO - trip_scheduling.i79 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i79 363 failed\r\n", - "INFO - trip_scheduling.i80 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i80 361 failed\r\n", - "INFO - trip_scheduling.i81 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i81 363 failed\r\n", - "INFO - trip_scheduling.i82 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i82 362 failed\r\n", - "INFO - trip_scheduling.i83 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i83 363 failed\r\n", - "INFO - trip_scheduling.i84 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i84 363 failed\r\n", - "INFO - trip_scheduling.i85 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i85 362 failed\r\n", - "INFO - trip_scheduling.i86 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i86 358 failed\r\n", - "INFO - trip_scheduling.i87 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i87 361 failed\r\n", - "INFO - trip_scheduling.i88 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i88 361 failed\r\n", - "INFO - trip_scheduling.i89 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i89 362 failed\r\n", - "INFO - trip_scheduling.i90 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i90 362 failed\r\n", - "INFO - trip_scheduling.i91 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i91 362 failed\r\n", - "INFO - trip_scheduling.i92 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i92 362 failed\r\n", - "INFO - trip_scheduling.i93 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i93 362 failed\r\n", - "INFO - trip_scheduling.i94 scheduling 530 trips within chunk 1\r\n", - "INFO - trip_scheduling.i94 361 failed\r\n", - "INFO - trip_scheduling.i95 scheduling 526 trips within chunk 1\r\n", - "INFO - trip_scheduling.i95 360 failed\r\n", - "INFO - trip_scheduling.i96 scheduling 526 trips within chunk 1\r\n", - "INFO - trip_scheduling.i96 359 failed\r\n", - "INFO - trip_scheduling.i97 scheduling 526 trips within chunk 1\r\n", - "INFO - trip_scheduling.i97 359 failed\r\n", - "INFO - trip_scheduling.i98 scheduling 526 trips within chunk 1\r\n", - "INFO - trip_scheduling.i98 359 failed\r\n", - "INFO - trip_scheduling.i99 scheduling 526 trips within chunk 1\r\n", - "INFO - trip_scheduling.i99 359 failed\r\n", - "INFO - trip_scheduling.i100 scheduling 526 trips within chunk 1\r\n", - "INFO - dumping trip_scheduling.i100.outbound.num_2.failed_choosers\r\n", - "WARNING - trip_scheduling.i100.outbound.num_2 coercing 2 depart choices to most initial\r\n", - "INFO - dumping trip_scheduling.i100.outbound.num_3.failed_choosers\r\n", - "WARNING - trip_scheduling.i100.outbound.num_3 coercing 11 depart choices to most initial\r\n", - "INFO - dumping trip_scheduling.i100.outbound.num_4.failed_choosers\r\n", - "WARNING - trip_scheduling.i100.outbound.num_4 coercing 9 depart choices to most initial\r\n", - "INFO - dumping trip_scheduling.i100.inbound.num_1.failed_choosers\r\n", - "WARNING - trip_scheduling.i100.inbound.num_1 coercing 133 depart choices to most initial\r\n", - "INFO - dumping trip_scheduling.i100.inbound.num_2.failed_choosers\r\n", - "WARNING - trip_scheduling.i100.inbound.num_2 coercing 133 depart choices to most initial\r\n", - "INFO - dumping trip_scheduling.i100.inbound.num_3.failed_choosers\r\n", - "WARNING - trip_scheduling.i100.inbound.num_3 coercing 73 depart choices to most initial\r\n", - "INFO - trip_scheduling.i100 0 failed\r\n", - "DEBUG - trip_scheduling: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_scheduling: write_table cache: override_choices\r\n", - "DEBUG - trip_scheduling: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'choosers', 'probs']\r\n", - "WARNING - trip_scheduling: write_omnibus_table: values_combined table 'probs' not found\r\n", - "DEBUG - trip_scheduling: write_omnibus_choosers: output/estimation_data_bundle/trip_scheduling/trip_scheduling_values_combined.csv\r\n", - "INFO - trip_scheduling: end estimation\r\n", - "Time to execute step 'trip_scheduling': 19.54 s\r\n", - "Total time to execute iteration 1 with iteration value None: 19.54 s\r\n", - "INFO - #run_model running step trip_mode_choice\r\n", - "Running step 'trip_mode_choice'\r\n", - "INFO - Running trip_mode_choice with 14352 trips\r\n", - "INFO - primary_purpose top 10 value counts:\r\n", - "work 5428\r\n", - "shopping 1876\r\n", - "othdiscr 1413\r\n", - "school 1081\r\n", - "atwork 1073\r\n", - "othmaint 975\r\n", - "escort 967\r\n", - "eatout 713\r\n", - "social 423\r\n", - "univ 403\r\n", - "Name: primary_purpose, dtype: int64\r\n", - "INFO - Initialize Estimator for'trip_mode_choice'\r\n", - "DEBUG - trip_mode_choice: write_table write: trip_mode_choice_coefficients.csv\r\n", - "DEBUG - trip_mode_choice: write_table write: coefficients_template\r\n", - "DEBUG - trip_mode_choice: estimate.write_spec: output/estimation_data_bundle/trip_mode_choice/trip_mode_choice_SPEC.csv\r\n", - "INFO - trip_mode_choice tour_type 'atwork' (1073 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.atwork.simple_simulate Running adaptive_chunked_choosers with 1073 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1073 of 1073 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'eatout' (713 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.eatout.simple_simulate Running adaptive_chunked_choosers with 713 choosers\r\n", - "INFO - Running chunk 1 of 1 with 713 of 713 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'escort' (967 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.escort.simple_simulate Running adaptive_chunked_choosers with 967 choosers\r\n", - "INFO - Running chunk 1 of 1 with 967 of 967 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'othdiscr' (1413 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.othdiscr.simple_simulate Running adaptive_chunked_choosers with 1413 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1413 of 1413 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'othmaint' (975 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.othmaint.simple_simulate Running adaptive_chunked_choosers with 975 choosers\r\n", - "INFO - Running chunk 1 of 1 with 975 of 975 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'school' (1081 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.school.simple_simulate Running adaptive_chunked_choosers with 1081 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1081 of 1081 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'shopping' (1876 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.shopping.simple_simulate Running adaptive_chunked_choosers with 1876 choosers\r\n", - "INFO - Running chunk 1 of 1 with 1876 of 1876 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'social' (423 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.social.simple_simulate Running adaptive_chunked_choosers with 423 choosers\r\n", - "INFO - Running chunk 1 of 1 with 423 of 423 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'univ' (403 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.univ.simple_simulate Running adaptive_chunked_choosers with 403 choosers\r\n", - "INFO - Running chunk 1 of 1 with 403 of 403 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "INFO - trip_mode_choice tour_type 'work' (5428 trips)\r\n", - "DEBUG - trip_mode_choice: write_table cache: choosers\r\n", - "INFO - trip_mode_choice.work.simple_simulate Running adaptive_chunked_choosers with 5428 choosers\r\n", - "INFO - Running chunk 1 of 1 with 5428 of 5428 choosers\r\n", - "DEBUG - trip_mode_choice: write_table cache: expression_values\r\n", - "DEBUG - trip_mode_choice: write_table cache: choices\r\n", - "DEBUG - get_survey_values: reindexing using trips.index\r\n", - "DEBUG - trip_mode_choice: write_table cache: override_choices\r\n", - "DEBUG - trip_mode_choice: write_omnibus_table: values_combined table_names: ['choices', 'override_choices', 'expression_values', 'choosers']\r\n", - "DEBUG - trip_mode_choice: write_omnibus_choosers: output/estimation_data_bundle/trip_mode_choice/trip_mode_choice_values_combined.csv\r\n", - "INFO - trip_mode_choice: end estimation\r\n", - "INFO - trip_modes top 10 value counts:\r\n", - "DRIVEALONEFREE 3864\r\n", - "SHARED2FREE 2283\r\n", - "SHARED3FREE 1921\r\n", - "WALK_LOC 1524\r\n", - "WALK_LRF 1427\r\n", - "WALK 1259\r\n", - "TNC_SINGLE 871\r\n", - "WALK_HVY 473\r\n", - "BIKE 469\r\n", - "TNC_SHARED 149\r\n", - "Name: tour_mode, dtype: int64\r\n", - "INFO - trip_mode_choice choices top 10 value counts:\r\n", - "DRIVEALONEFREE 4317\r\n", - "SHARED2FREE 1872\r\n", - "WALK_LOC 1828\r\n", - "WALK 1621\r\n", - "SHARED3FREE 1345\r\n", - "WALK_LRF 1189\r\n", - "TNC_SINGLE 1038\r\n", - "BIKE 434\r\n", - "TNC_SHARED 361\r\n", - "WALK_HVY 242\r\n", - "Name: trip_mode, dtype: int64\r\n", - "Time to execute step 'trip_mode_choice': 18.31 s\r\n", - "Total time to execute iteration 1 with iteration value None: 18.31 s\r\n", - "INFO - #run_model running step write_tables\r\n", - "Running step 'write_tables'\r\n", - "Time to execute step 'write_tables': 0.41 s\r\n", - "Total time to execute iteration 1 with iteration value None: 0.41 s\r\n", - "INFO - Time to execute run_model (29 models) : 713.015 seconds (11.9 minutes)\r\n", - "INFO - MainProcess high water mark rss: 1_529_487_360 (1.5 GB) timestamp: 09/05/2022 14:57:08 label:pipeline.run_model trip_destination finished\r\n", - "INFO - MainProcess high water mark uss: 0 (0 B) timestamp: 09/05/2022 14:45:55 label:pipeline.run before preload_injectables\r\n", - "INFO - Time to execute all models : 713.145 seconds (11.9 minutes)\r\n" - ] - } - ], + "execution_count": 4, + "outputs": [], "source": [ - "# run estimation mode TODO: smaller data set?\n", - "!activitysim run -c configs_estimation/configs -c configs -o output -d data_sf" + "# run estimation mode\n", + "#!activitysim run -c configs_estimation/configs -c configs -o output_sf -d data_sf" ], "metadata": { "collapsed": false, @@ -2827,12 +93,14 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 5, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ + "/mnt/c/Users/jan.zill/code/activitysim/activitysim/estimation/larch/simple_simulate.py:92: DtypeWarning: Columns (411) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " chooser_data = _read_csv(chooser_data_file, index_col=values_index_col,)\n", "req_data does not request avail_ca or avail_co but it is set and being provided\n", "req_data does not request avail_ca or avail_co but it is set and being provided\n", "req_data does not request avail_ca or avail_co but it is set and being provided\n", @@ -2859,14 +127,6 @@ "problem: chosen-but-not-available (2 issues)\n", "problem: low-variance-data-co (1 issues)\n" ] - }, - { - "data": { - "text/plain": "-10094.898223413013" - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -2893,13 +153,15 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 16, "outputs": [], "source": [ "probs_0 = pd.DataFrame(probs[0])\n", "m_0 = model._k_models[0]\n", "\n", - "probs_0.index = m_0.dataframes.data_ch.index" + "probs_0.index = m_0.dataframes.data_ch.index\n", + "\n", + "probs_0 = probs_0.rename(columns={v-1: k for k,v in data.settings[\"CONSTANTS\"][\"I_MODE_MAP\"].items()})" ], "metadata": { "collapsed": false, @@ -2910,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 17, "outputs": [ { "data": { @@ -2940,13 +202,13 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 18, "outputs": [], "source": [ "#data.chooser_data[['override_choice', 'override_choice_code']].drop_duplicates().reset_index(drop=True).set_index\\\n", "# ('override_choice_code').to_dict()['override_choice']\n", - "mode_map = pd.DataFrame.from_dict(data.settings[\"CONSTANTS\"][\"I_MODE_MAP\"], orient='index', columns=[\"mode_code\"])\\\n", - " .reset_index().rename(columns={\"index\": \"mode_name\"})" + "#mode_map = pd.DataFrame.from_dict(data.settings[\"CONSTANTS\"][\"I_MODE_MAP\"], orient='index', columns=[\"mode_code\"])\\\n", + "# .reset_index().rename(columns={\"index\": \"mode_name\"})" ], "metadata": { "collapsed": false, @@ -2957,39 +219,83 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 19, "outputs": [ { "data": { - "text/plain": " trip_id model_choice override_choice \\\ntour_id \n3021985 24175881 DRIVEALONEFREE DRIVEALONEFREE \n3021985 24175885 DRIVEALONEFREE DRIVEALONEFREE \n\n util_DRIVEALONEFREE_Unavailable \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_zero_auto_households \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_persons_less_than_16 \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_joint_tours \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_work \\\ntour_id \n3021985 0.0 \n3021985 0.0 \n\n util_DRIVEALONEFREE_In_vehicle_time \\\ntour_id \n3021985 11.50 \n3021985 11.81 \n\n util_DRIVEALONEFREE_Terminal_time ... \\\ntour_id ... \n3021985 4.14944 ... \n3021985 4.14944 ... \n\n drive_heavyrail_available_outbound \\\ntour_id \n3021985 False \n3021985 False \n\n drive_heavyrail_available_inbound drive_commuter_available_outbound \\\ntour_id \n3021985 False False \n3021985 False False \n\n drive_commuter_available_inbound walk_ferry_available \\\ntour_id \n3021985 False False \n3021985 False False \n\n drive_ferry_available distance distance_walk_od distance_bike_od \\\ntour_id \n3021985 False 5.96 5.99 5.99 \n3021985 False 5.99 5.99 5.99 \n\n override_choice_code \ntour_id \n3021985 1 \n3021985 1 \n\n[2 rows x 478 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_idmodel_choiceoverride_choiceutil_DRIVEALONEFREE_Unavailableutil_DRIVEALONEFREE_Unavailable_for_zero_auto_householdsutil_DRIVEALONEFREE_Unavailable_for_persons_less_than_16util_DRIVEALONEFREE_Unavailable_for_joint_toursutil_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_workutil_DRIVEALONEFREE_In_vehicle_timeutil_DRIVEALONEFREE_Terminal_time...drive_heavyrail_available_outbounddrive_heavyrail_available_inbounddrive_commuter_available_outbounddrive_commuter_available_inboundwalk_ferry_availabledrive_ferry_availabledistancedistance_walk_oddistance_bike_odoverride_choice_code
tour_id
302198524175881DRIVEALONEFREEDRIVEALONEFREE0.00.00.00.00.011.504.14944...FalseFalseFalseFalseFalseFalse5.965.995.991
302198524175885DRIVEALONEFREEDRIVEALONEFREE0.00.00.00.00.011.814.14944...FalseFalseFalseFalseFalseFalse5.995.995.991
\n

2 rows × 478 columns

\n
" + "text/plain": "Int64Index([ 2961920, 2970120, 2998943, 3013252, 3015794, 3021985,\n 3022354, 3025019, 3055523, 3060361,\n ...\n 308028857, 308037836, 308057680, 308057681, 308070308, 308070309,\n 308073875, 308090603, 308105896, 308122624],\n dtype='int64', name='_caseid_', length=2124)" + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "m_0.dataframes.data_ch.index.unique()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 34, + "outputs": [ + { + "data": { + "text/plain": " trip_id model_choice override_choice \\\ntour_id \n2961920 23695361 WALK_LRF WALK_LRF \n2961920 23695365 WALK WALK_LRF \n\n util_DRIVEALONEFREE_Unavailable \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_zero_auto_households \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_persons_less_than_16 \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_joint_tours \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_work \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_In_vehicle_time \\\ntour_id \n2961920 5.36 \n2961920 4.91 \n\n util_DRIVEALONEFREE_Terminal_time ... \\\ntour_id ... \n2961920 10.45084 ... \n2961920 10.45084 ... \n\n drive_heavyrail_available_outbound \\\ntour_id \n2961920 False \n2961920 False \n\n drive_heavyrail_available_inbound drive_commuter_available_outbound \\\ntour_id \n2961920 False False \n2961920 False False \n\n drive_commuter_available_inbound walk_ferry_available \\\ntour_id \n2961920 False False \n2961920 False False \n\n drive_ferry_available distance distance_walk_od distance_bike_od \\\ntour_id \n2961920 False 1.98 1.98 1.98 \n2961920 False 1.96 1.96 1.96 \n\n override_choice_code \ntour_id \n2961920 10 \n2961920 10 \n\n[2 rows x 478 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_idmodel_choiceoverride_choiceutil_DRIVEALONEFREE_Unavailableutil_DRIVEALONEFREE_Unavailable_for_zero_auto_householdsutil_DRIVEALONEFREE_Unavailable_for_persons_less_than_16util_DRIVEALONEFREE_Unavailable_for_joint_toursutil_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_workutil_DRIVEALONEFREE_In_vehicle_timeutil_DRIVEALONEFREE_Terminal_time...drive_heavyrail_available_outbounddrive_heavyrail_available_inbounddrive_commuter_available_outbounddrive_commuter_available_inboundwalk_ferry_availabledrive_ferry_availabledistancedistance_walk_oddistance_bike_odoverride_choice_code
tour_id
296192023695361WALK_LRFWALK_LRF0.00.00.00.00.05.3610.45084...FalseFalseFalseFalseFalseFalse1.981.981.9810
296192023695365WALKWALK_LRF0.00.00.00.00.04.9110.45084...FalseFalseFalseFalseFalseFalse1.961.961.9610
\n

2 rows × 478 columns

\n
" }, "metadata": {}, "output_type": "display_data" }, { "data": { - "text/plain": " 1 2 3 4 5 6 7 8 9 10 ... 12 13 \\\n_caseid_ ... \n3021985 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 \n3021985 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 \n\n 14 15 16 17 18 19 20 21 \n_caseid_ \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n\n[2 rows x 21 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
12345678910...12131415161718192021
_caseid_
30219851.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
30219851.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n

2 rows × 21 columns

\n
" + "text/plain": " 1 2 3 4 5 6 7 8 9 10 ... 12 13 \\\n_caseid_ ... \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 ... 0.0 0.0 \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 ... 0.0 0.0 \n\n 14 15 16 17 18 19 20 21 \n_caseid_ \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n\n[2 rows x 21 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
12345678910...12131415161718192021
_caseid_
29619200.00.00.00.00.00.00.00.00.01.0...0.00.00.00.00.00.00.00.00.00.0
29619200.00.00.00.00.00.00.00.00.01.0...0.00.00.00.00.00.00.00.00.00.0
\n

2 rows × 21 columns

\n
" }, "metadata": {}, "output_type": "display_data" }, { "data": { - "text/plain": " 0 1 2 3 4 5 6 7 8 9 ... \\\n_caseid_ ... \n3021985 0.999871 0.0 0.0 0.0 0.0 0.0 0.000006 0.0 0.0 0.0 ... \n3021985 0.999886 0.0 0.0 0.0 0.0 0.0 0.000006 0.0 0.0 0.0 ... \n\n 11 12 13 14 15 16 17 18 19 20 \n_caseid_ \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.368530e-06 0.000021 0.000100 \n3021985 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.692194e-07 0.000011 0.000096 \n\n[2 rows x 21 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
0123456789...11121314151617181920
_caseid_
30219850.9998710.00.00.00.00.00.0000060.00.00.0...0.00.00.00.00.00.00.01.368530e-060.0000210.000100
30219850.9998860.00.00.00.00.00.0000060.00.00.0...0.00.00.00.00.00.00.03.692194e-070.0000110.000096
\n

2 rows × 21 columns

\n
" + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n_caseid_ \n2961920 0.0 0.0 2.027363e-06 0.0 \n2961920 0.0 0.0 2.943297e-09 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF ... \\\n_caseid_ ... \n2961920 1.061530e-06 0.0 0.111823 0.0 0.263130 0.625043 ... \n2961920 1.967559e-08 0.0 0.119603 0.0 0.272258 0.608139 ... \n\n WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY \\\n_caseid_ \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n_caseid_ \n2961920 0.0 2.489336e-08 4.965881e-08 4.072015e-07 \n2961920 0.0 2.435815e-08 4.996681e-08 4.311721e-07 \n\n[2 rows x 21 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRF...WALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
_caseid_
29619200.00.02.027363e-060.01.061530e-060.00.1118230.00.2631300.625043...0.00.00.00.00.00.00.02.489336e-084.965881e-084.072015e-07
29619200.00.02.943297e-090.01.967559e-080.00.1196030.00.2722580.608139...0.00.00.00.00.00.00.02.435815e-084.996681e-084.311721e-07
\n

2 rows × 21 columns

\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "23695361\n" + ] + }, + { + "data": { + "text/plain": " SHARED2FREE SHARED3FREE WALK WALK_LOC WALK_LRF \\\n_caseid_ \n2961920 2.027363e-06 1.061530e-06 0.111823 0.263130 0.625043 \n2961920 2.943297e-09 1.967559e-08 0.119603 0.272258 0.608139 \n\n TAXI TNC_SINGLE TNC_SHARED \n_caseid_ \n2961920 2.489336e-08 4.965881e-08 4.072015e-07 \n2961920 2.435815e-08 4.996681e-08 4.311721e-07 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREEWALKWALK_LOCWALK_LRFTAXITNC_SINGLETNC_SHARED
_caseid_
29619202.027363e-061.061530e-060.1118230.2631300.6250432.489336e-084.965881e-084.072015e-07
29619202.943297e-091.967559e-080.1196030.2722580.6081392.435815e-084.996681e-084.311721e-07
\n
" }, "metadata": {}, "output_type": "display_data" } ], "source": [ - "tour_id_to_check = 3021985\n", + "tour_id_to_check = 2961920\n", "\n", "display(data.chooser_data.loc[data.chooser_data.index == tour_id_to_check])\n", "display(m_0.dataframes.data_ch.loc[m_0.dataframes.data_ch.index == tour_id_to_check])\n", - "display(probs_0.loc[probs_0.index == tour_id_to_check])" + "display(probs_0.loc[probs_0.index == tour_id_to_check])\n", + "\n", + "# choose the first trip to look into probabilities below\n", + "trip_id_to_check = data.chooser_data.loc[data.chooser_data.index == tour_id_to_check].trip_id.values[0]\n", + "print(trip_id_to_check)\n", + "res_ = probs_0.loc[(probs_0.index == tour_id_to_check)]\n", + "display(res_.loc[:, (res_ != 0).any(axis=0)])" ], "metadata": { "collapsed": false, @@ -2999,12 +305,18 @@ } }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 46, + "outputs": [], "source": [ - "## now use my code to calculate probabilities" + "data.chooser_data.loc[data.chooser_data.index.isin(m_0.dataframes.data_ch.index)].to_csv(os.path.join(example_dir, \"choosers_larch.csv\"))\n", + "probs_0.to_csv(os.path.join(example_dir, \"probabilities_larch.csv\"))" ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } } }, { diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 84605dcd55..0843c4a2d6 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 24, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "execution": { @@ -26,14 +26,23 @@ }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 25, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "execution": { @@ -52,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 43, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -82,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 27, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -101,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 28, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -121,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 29, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -140,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 30, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -167,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 31, "outputs": [], "source": [ "## need to do this before loading checkpoint tables\n", @@ -185,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 45, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -202,7 +211,6 @@ "name": "stderr", "output_type": "stream", "text": [ - "setting trace_hh_id is wrong type, should be an int, but was \n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -247,8 +255,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 16.9 s, sys: 2.78 s, total: 19.7 s\n", - "Wall time: 14.7 s\n" + "CPU times: user 16.3 s, sys: 2.48 s, total: 18.8 s\n", + "Wall time: 14.1 s\n" ] } ], @@ -260,14 +268,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 46, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 308 304\nWALK_LOC 87 85\nWALK_LRF 54 53\nBIKE 15 17\nTNC_SINGLE 7 4\nDRIVEALONEFREE 5 4\nSHARED2FREE 3 4\nSHARED3FREE 2 1\nWALK_HVY 1 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK308304
WALK_LOC8785
WALK_LRF5453
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n
" }, - "execution_count": 10, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -287,7 +295,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 47, "outputs": [ { "data": { @@ -315,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 48, "outputs": [], "source": [ "#ls_ = np.abs(((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru))\n", @@ -349,7 +357,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 49, "outputs": [ { "name": "stderr", @@ -390,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 51, "outputs": [ { "name": "stderr", @@ -403,11 +411,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'social' (100000 trips)\n", - "Done\n", - "CPU times: user 1min 1s, sys: 3.78 s, total: 1min 5s\n", - "Wall time: 1min 2s\n" + "trip_mode_choice tour_type 'social' (10000 trips)\n", + "Done\n" + ] + }, + { + "data": { + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.004600 0.000000e+00 0.000000 0.000300 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.042100 0.120600 0.832400 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0046000.000000e+000.0000000.0003000.0000000.0421000.1206000.832400
\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 7.16 s, sys: 594 ms, total: 7.75 s\n", + "Wall time: 6.98 s\n" ] } ], @@ -415,7 +436,7 @@ "%%time\n", "#prun\n", "\n", - "num_samples = 100000\n", + "num_samples = 10000\n", "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like 65s and 1e4 takes 8s\n", "# work 615236801 also in trace log\n", "# social 2464104885 642446345 1767182945\n", @@ -743,6 +764,108 @@ } } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 52, + "outputs": [], + "source": [ + "from activitysim.core import config\n", + "resume_after = \"trip_scheduling\"\n", + "model_name = \"trip_mode_choice\"\n", + "\n", + "pipeline.open_pipeline(resume_after)\n", + "# preload any bulky injectables (e.g. skims) not in pipeline\n", + "inject.get_injectable('preload_injectables', None)\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "#step_name = model_name\n", + "args = {}\n", + "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "trips = inject.get_table('trips')\n", + "tours_merged = inject.get_table('tours_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "\n", + "trace_label = 'trip_mode_choice'\n", + "model_settings_file_name = 'trip_mode_choice.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", + "mode_column_name = 'trip_mode'\n", + "\n", + "trips_df = trips.to_frame()\n", + "\n", + "tours_merged = tours_merged.to_frame()\n", + "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", + "\n", + "# - trips_merged - merge trips and tours_merged\n", + "trips_merged = pd.merge(\n", + " trips_df,\n", + " tours_merged,\n", + " left_on='tour_id',\n", + " right_index=True,\n", + " how=\"left\")\n", + "assert trips_merged.index.equals(trips.index)\n", + "\n", + "# setup skim keys\n", + "assert ('trip_period' not in trips_merged)\n", + "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", + "pipeline.close_pipeline()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 56, + "outputs": [ + { + "data": { + "text/plain": "Index(['person_id', 'household_id', 'primary_purpose', 'trip_num', 'outbound',\n 'trip_count', 'destination', 'origin', 'tour_id', 'purpose',\n 'destination_logsum', 'depart', 'hhsize', 'age', 'auto_ownership',\n 'number_of_participants', 'tour_category', 'parent_tour_id',\n 'tour_mode', 'duration', 'value_of_time', 'tour_type',\n 'free_parking_at_work', 'trip_period'],\n dtype='object')" + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trips_merged.columns" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 4ee74ea7b9a291ca3ab413f4644698d423f99d2c Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 10 May 2022 00:40:11 +1000 Subject: [PATCH 024/135] clean up --- notebooks/check_sf_probs.ipynb | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/notebooks/check_sf_probs.ipynb b/notebooks/check_sf_probs.ipynb index de5d82d521..c14f1578b6 100644 --- a/notebooks/check_sf_probs.ipynb +++ b/notebooks/check_sf_probs.ipynb @@ -234,7 +234,7 @@ "num_samples = 10000\n", "\n", "comp = compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs)\n", - "comp.loc[(comp != 0).any(axis=1), :].applymap(lambda x: f\"{x:.3%}\")" + "comp.loc[(comp > 1e-3).any(axis=1), :].applymap(lambda x: f\"{x:.3%}\")" ], "metadata": { "collapsed": false, @@ -244,6 +244,30 @@ } } }, + { + "cell_type": "code", + "execution_count": 95, + "outputs": [ + { + "data": { + "text/plain": " me asim larch\nSHARED2FREE 0.0000 0.0000 2.027363e-06\nSHARED3FREE 0.0000 0.0000 1.061530e-06\nTAXI 0.0000 0.0000 2.489336e-08\nTNC_SHARED 0.0000 0.0000 4.072015e-07\nTNC_SINGLE 0.0000 0.0000 4.965881e-08\nWALK 0.1085 0.1166 1.118235e-01\nWALK_LOC 0.2615 0.2015 2.631296e-01\nWALK_LRF 0.6300 0.6819 6.250434e-01", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
measimlarch
SHARED2FREE0.00000.00002.027363e-06
SHARED3FREE0.00000.00001.061530e-06
TAXI0.00000.00002.489336e-08
TNC_SHARED0.00000.00004.072015e-07
TNC_SINGLE0.00000.00004.965881e-08
WALK0.10850.11661.118235e-01
WALK_LOC0.26150.20152.631296e-01
WALK_LRF0.63000.68196.250434e-01
\n
" + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "comp.loc[(comp > 0).any(axis=1), :]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 4d0d8232aa5ed9b97340c6f683163140e4f19272 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 10 May 2022 01:10:24 +1000 Subject: [PATCH 025/135] inspecting a couple of test cases --- notebooks/check_sf_probs.ipynb | 42 ++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/notebooks/check_sf_probs.ipynb b/notebooks/check_sf_probs.ipynb index c14f1578b6..623b946bc5 100644 --- a/notebooks/check_sf_probs.ipynb +++ b/notebooks/check_sf_probs.ipynb @@ -208,6 +208,30 @@ } } }, + { + "cell_type": "code", + "execution_count": 124, + "outputs": [ + { + "data": { + "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n23695361 0.000000 0.0 2.027363e-06 0.0 \n23695365 0.000000 0.0 2.943297e-09 0.0 \n23760961 0.999664 0.0 0.000000e+00 0.0 \n23760962 0.998934 0.0 0.000000e+00 0.0 \n23760963 0.997988 0.0 0.000000e+00 0.0 \n... ... ... ... ... \n2464724829 0.000000 0.0 3.268925e-05 0.0 \n2464847169 0.000000 0.0 2.321209e-03 0.0 \n2464847173 0.000000 0.0 1.269812e-03 0.0 \n2464980993 0.000000 0.0 8.280461e-05 0.0 \n2464980997 0.000000 0.0 6.298188e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n23695361 1.061530e-06 0.0 0.111823 0.0 0.263130 0.625043 \n23695365 1.967559e-08 0.0 0.119603 0.0 0.272258 0.608139 \n23760961 0.000000e+00 0.0 0.000010 0.0 0.000000 0.000000 \n23760962 0.000000e+00 0.0 0.000579 0.0 0.000000 0.000000 \n23760963 0.000000e+00 0.0 0.001509 0.0 0.000000 0.000000 \n... ... ... ... ... ... ... \n2464724829 3.022226e-05 0.0 0.691838 0.0 0.308044 0.000000 \n2464847169 1.213314e-03 0.0 0.249261 0.0 0.745641 0.000000 \n2464847173 7.535248e-04 0.0 0.186394 0.0 0.810411 0.000000 \n2464980993 4.326905e-05 0.0 0.003217 0.0 0.996572 0.000000 \n2464980997 3.791076e-05 0.0 0.003749 0.0 0.996046 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n23695361 0.0 0.0 0.0 0.0 0.0 0.0 \n23695365 0.0 0.0 0.0 0.0 0.0 0.0 \n23760961 0.0 0.0 0.0 0.0 0.0 0.0 \n23760962 0.0 0.0 0.0 0.0 0.0 0.0 \n23760963 0.0 0.0 0.0 0.0 0.0 0.0 \n... ... ... ... ... ... ... \n2464724829 0.0 0.0 0.0 0.0 0.0 0.0 \n2464847169 0.0 0.0 0.0 0.0 0.0 0.0 \n2464847173 0.0 0.0 0.0 0.0 0.0 0.0 \n2464980993 0.0 0.0 0.0 0.0 0.0 0.0 \n2464980997 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n23695361 0.0 0.0 2.489336e-08 4.965881e-08 4.072015e-07 \n23695365 0.0 0.0 2.435815e-08 4.996681e-08 4.311721e-07 \n23760961 0.0 0.0 4.724941e-05 1.754653e-04 1.035425e-04 \n23760962 0.0 0.0 1.066593e-04 2.780937e-04 1.021060e-04 \n23760963 0.0 0.0 1.039326e-04 2.850044e-04 1.143830e-04 \n... ... ... ... ... ... \n2464724829 0.0 0.0 3.606576e-05 4.836729e-07 1.874068e-05 \n2464847169 0.0 0.0 4.274883e-04 5.407950e-04 5.947334e-04 \n2464847173 0.0 0.0 3.063342e-04 4.098183e-04 4.563525e-04 \n2464980993 0.0 0.0 1.593300e-05 4.967854e-05 1.954461e-05 \n2464980997 0.0 0.0 2.239090e-05 6.033135e-05 2.206105e-05 \n\n[5428 rows x 21 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
236953610.0000000.02.027363e-060.01.061530e-060.00.1118230.00.2631300.6250430.00.00.00.00.00.00.00.02.489336e-084.965881e-084.072015e-07
236953650.0000000.02.943297e-090.01.967559e-080.00.1196030.00.2722580.6081390.00.00.00.00.00.00.00.02.435815e-084.996681e-084.311721e-07
237609610.9996640.00.000000e+000.00.000000e+000.00.0000100.00.0000000.0000000.00.00.00.00.00.00.00.04.724941e-051.754653e-041.035425e-04
237609620.9989340.00.000000e+000.00.000000e+000.00.0005790.00.0000000.0000000.00.00.00.00.00.00.00.01.066593e-042.780937e-041.021060e-04
237609630.9979880.00.000000e+000.00.000000e+000.00.0015090.00.0000000.0000000.00.00.00.00.00.00.00.01.039326e-042.850044e-041.143830e-04
..................................................................
24647248290.0000000.03.268925e-050.03.022226e-050.00.6918380.00.3080440.0000000.00.00.00.00.00.00.00.03.606576e-054.836729e-071.874068e-05
24648471690.0000000.02.321209e-030.01.213314e-030.00.2492610.00.7456410.0000000.00.00.00.00.00.00.00.04.274883e-045.407950e-045.947334e-04
24648471730.0000000.01.269812e-030.07.535248e-040.00.1863940.00.8104110.0000000.00.00.00.00.00.00.00.03.063342e-044.098183e-044.563525e-04
24649809930.0000000.08.280461e-050.04.326905e-050.00.0032170.00.9965720.0000000.00.00.00.00.00.00.00.01.593300e-054.967854e-051.954461e-05
24649809970.0000000.06.298188e-050.03.791076e-050.00.0037490.00.9960460.0000000.00.00.00.00.00.00.00.02.239090e-056.033135e-052.206105e-05
\n

5428 rows × 21 columns

\n
" + }, + "execution_count": 124, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "larch_probs" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, @@ -230,8 +254,10 @@ "source": [ "%%time\n", "\n", - "trip_id_to_check = 23695361\n", - "num_samples = 10000\n", + "trip_id_to_check = 23695361 # obvious results with 1e4 samples: 23695361 24106017\n", + "# better with 1e5 samples: 1932588125\n", + "# requires higher precision, check when time: 38280865\n", + "num_samples = 10000 # 1e5 takes about 80s\n", "\n", "comp = compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs)\n", "comp.loc[(comp > 1e-3).any(axis=1), :].applymap(lambda x: f\"{x:.3%}\")" @@ -244,6 +270,18 @@ } } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": 95, From fba872882d5221fc21775f8e6a1cc0a90e59c9b3 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 10 May 2022 13:10:43 +1000 Subject: [PATCH 026/135] nest scales in asim seem to be all between 0 and 1, so not like in larch where they are non-increasing - re-adjust util calc. should really document this somewhere --- activitysim/core/logit.py | 3 - activitysim/core/simulate.py | 7 +- notebooks/check_sf_probs.ipynb | 75 ++++++++++++++++++---- notebooks/validate_frozen_impl.ipynb | 95 +++++++++++++++++----------- 4 files changed, 125 insertions(+), 55 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 40a3a8671f..adfafcef2e 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -386,7 +386,6 @@ def __init__(self, name=None, level=0): self.ancestors = [] self.alternatives = None self.coefficient = 0 - self.parent_scale = 1 def print(self): print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s " % @@ -461,7 +460,6 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients * coefficient nest.alternatives = alternatives nest.ancestors = parent_nest.ancestors + [name] - nest.parent_scale = parent_nest.coefficient if pre_order: yield spec, nest @@ -482,7 +480,6 @@ def _each_nest(spec, parent_nest, post_order): nest.product_of_coefficients = parent_nest.product_of_coefficients nest.ancestors = parent_nest.ancestors + [name] nest.coefficient = parent_nest.coefficient - nest.parent_scale = parent_nest.coefficient yield spec, nest diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index be03116968..3bfd4f7761 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -737,16 +737,13 @@ def compute_nested_utilities(raw_utilities, nest_spec): name = nest.name if nest.is_leaf: # do not scale here, do afterwards so recursive structure works - nested_utilities[name] = raw_utilities[name].astype(float) + nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients else: # the alternative nested_utilities will already have been computed due to post_order with np.errstate(divide='ignore'): nested_utilities[name] = \ nest.coefficient * np.log( - np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) - - if nest.level > 1: # root does not have parent. also, parent_scale seems to be 0. - nested_utilities[name] = nested_utilities[name] / nest.parent_scale + np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) return nested_utilities diff --git a/notebooks/check_sf_probs.ipynb b/notebooks/check_sf_probs.ipynb index 623b946bc5..dfa75b24a2 100644 --- a/notebooks/check_sf_probs.ipynb +++ b/notebooks/check_sf_probs.ipynb @@ -234,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 142, "outputs": [ { "name": "stderr", @@ -247,14 +247,41 @@ "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'work' (10000 trips)\n" + "trip_mode_choice tour_type 'work' (10000 trips)\n", + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'work' (10000 trips)\n", + "Done\n", + "CPU times: user 9.94 s, sys: 844 ms, total: 10.8 s\n", + "Wall time: 9.56 s\n" ] + }, + { + "data": { + "text/plain": " me asim larch\nSHARED2FREE 0.450% 0.400% 0.359%\nSHARED3FREE 0.220% 0.140% 0.188%\nTAXI 0.110% 0.160% 0.080%\nTNC_SINGLE 0.280% 0.270% 0.202%\nWALK 0.520% 0.480% 0.435%\nWALK_HVY 69.070% 70.030% 60.165%\nWALK_LOC 9.540% 9.310% 14.513%\nWALK_LRF 19.710% 19.120% 23.977%", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
measimlarch
SHARED2FREE0.450%0.400%0.359%
SHARED3FREE0.220%0.140%0.188%
TAXI0.110%0.160%0.080%
TNC_SINGLE0.280%0.270%0.202%
WALK0.520%0.480%0.435%
WALK_HVY69.070%70.030%60.165%
WALK_LOC9.540%9.310%14.513%
WALK_LRF19.710%19.120%23.977%
\n
" + }, + "execution_count": 142, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "%%time\n", "\n", - "trip_id_to_check = 23695361 # obvious results with 1e4 samples: 23695361 24106017\n", + "trip_id_to_check = 24106017 # obvious results with 1e4 samples: 23695361 24106017\n", "# better with 1e5 samples: 1932588125\n", "# requires higher precision, check when time: 38280865\n", "num_samples = 10000 # 1e5 takes about 80s\n", @@ -265,8 +292,19 @@ "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n", - "is_executing": true + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" } } }, @@ -284,20 +322,33 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 139, + "outputs": [], + "source": [ + "#comp.loc[(comp > 0).any(axis=1), :]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 140, "outputs": [ { "data": { - "text/plain": " me asim larch\nSHARED2FREE 0.0000 0.0000 2.027363e-06\nSHARED3FREE 0.0000 0.0000 1.061530e-06\nTAXI 0.0000 0.0000 2.489336e-08\nTNC_SHARED 0.0000 0.0000 4.072015e-07\nTNC_SINGLE 0.0000 0.0000 4.965881e-08\nWALK 0.1085 0.1166 1.118235e-01\nWALK_LOC 0.2615 0.2015 2.631296e-01\nWALK_LRF 0.6300 0.6819 6.250434e-01", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
measimlarch
SHARED2FREE0.00000.00002.027363e-06
SHARED3FREE0.00000.00001.061530e-06
TAXI0.00000.00002.489336e-08
TNC_SHARED0.00000.00004.072015e-07
TNC_SINGLE0.00000.00004.965881e-08
WALK0.10850.11661.118235e-01
WALK_LOC0.26150.20152.631296e-01
WALK_LRF0.63000.68196.250434e-01
\n
" + "text/plain": "0.252" }, - "execution_count": 95, + "execution_count": 140, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "comp.loc[(comp > 0).any(axis=1), :]" + "0.72 * 0.35" ], "metadata": { "collapsed": false, @@ -310,7 +361,9 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [], + "source": [ + "# check what happens if you multiply them - like Train at the lower level? but then 0\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK308304
WALK_LOC8785
WALK_LRF5453
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n" + "text/plain": " trip_mode_fru trip_mode_asim\nWALK 309 304\nWALK_LOC 85 85\nWALK_LRF 55 53\nBIKE 15 17\nTNC_SINGLE 7 4\nDRIVEALONEFREE 5 4\nSHARED2FREE 3 4\nSHARED3FREE 2 1\nWALK_HVY 1 4", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK309304
WALK_LOC8585
WALK_LRF5553
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n
" }, - "execution_count": 46, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -295,17 +286,14 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 16, "outputs": [ { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN+ElEQVR4nO3df2xd9XnH8fdTXCgiG2kb5rEkmpGIOiHc0WIBU/+YA/sRoFrYBBUdKkmXKZoEUqdmak2RVk3btFSIoXarKkWlWjqhGcRWgQjVygCr6h9hS1pK+NEOF4U1EQXR0nQGusnrsz984NqpzXXiY9/cx++XZOV8v9/jr5/7CH18cnLuJTITSVItb+t1AZKk9hnuklSQ4S5JBRnuklSQ4S5JBQ30ugCAdevW5dDQUCt7vfrqq5x11lmt7NXv7EWHveiwFx393ouDBw++nJnnzLd2SoT70NAQBw4caGWviYkJRkdHW9mr39mLDnvRYS86+r0XEfH8QmvelpGkggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekgk6Jd6hK0koYGts3Z7xreJrtY/s4vPvqHlW0fLxyl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKmjR4R4Rp0XEtyLigWZ8XkQ8FhGTEXF3RJzezJ/RjCeb9aFlql2StIATuXL/GPDMrPFngDsy83zgFWBHM78DeKWZv6M5T5K0ghYV7hGxAbga+GIzDuBy4N7mlL3ANc3x1mZMs35Fc74kaYVEZnY/KeJe4G+AXwD+DNgO7G+uzomIjcBXM/PCiHgS2JKZR5q17wGXZubLx+25E9gJMDg4ePH4+HgrL2hqaoo1a9a0sle/sxcd9qJjNffi0NFjc8aDZ8KLr8Pw+rN7VNHSbN68+WBmjsy3NtDtmyPig8BLmXkwIkbbKioz9wB7AEZGRnJ0tJ2tJyYmaGuvfmcvOuxFx2ruxfaxfXPGu4anuf3QAIdvGO1NQcuoa7gDHwB+LyKuAt4B/CLwWWBtRAxk5jSwATjanH8U2AgciYgB4Gzgh61XLklaUNd77pl5S2ZuyMwh4Hrgkcy8AXgUuLY5bRtwX3N8fzOmWX8kF3PvR5LUmqU85/5J4OMRMQm8G7izmb8TeHcz/3FgbGklSpJO1GJuy7wpMyeAieb4OeCSec75KXBdC7VJkk6S71CVpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIK6hntEvCMi/j0ivh0RT0XEXzTz50XEYxExGRF3R8TpzfwZzXiyWR9a5tcgSTrOYq7c/we4PDN/HbgI2BIRlwGfAe7IzPOBV4Adzfk7gFea+Tua8yRJK6hruOeMqWb49uYrgcuBe5v5vcA1zfHWZkyzfkVERFsFS5K6i8zsflLEacBB4Hzg88BtwP7m6pyI2Ah8NTMvjIgngS2ZeaRZ+x5waWa+fNyeO4GdAIODgxePj4+38oKmpqZYs2ZNK3v1O3vRYS86VnMvDh09Nmc8eCa8+DoMrz+7RxUtzebNmw9m5sh8awOL2SAz/w+4KCLWAl8Bfm2pRWXmHmAPwMjISI6Oji51SwAmJiZoa69+Zy867EXHau7F9rF9c8a7hqe5/dAAh28Y7U1By+iEnpbJzB8DjwK/AayNiDd+OWwAjjbHR4GNAM362cAP2yhWkrQ4i3la5pzmip2IOBP4beAZZkL+2ua0bcB9zfH9zZhm/ZFczL0fSVJrFnNb5lxgb3Pf/W3APZn5QEQ8DYxHxF8B3wLubM6/E/jHiJgEfgRcvwx1S5LeQtdwz8wngPfNM/8ccMk88z8FrmulOknSSfEdqpJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUUNdwj4iNEfFoRDwdEU9FxMea+XdFxEMR8Wzz5zub+YiIz0XEZEQ8ERHvX+4XIUmaazFX7tPArsy8ALgMuCkiLgDGgIczcxPwcDMGuBLY1HztBL7QetWSpLfUNdwz84XM/GZz/N/AM8B6YCuwtzltL3BNc7wV+HLO2A+sjYhz2y5ckrSwE7rnHhFDwPuAx4DBzHyhWfoBMNgcrwe+P+vbjjRzkqQVMrDYEyNiDfDPwJ9m5k8i4s21zMyIyBP5wRGxk5nbNgwODjIxMXEi376gqamp1vbqd/aiw150rOZe7BqenjMePHNmrmI/FhXuEfF2ZoL9rsz8l2b6xYg4NzNfaG67vNTMHwU2zvr2Dc3cHJm5B9gDMDIykqOjoyf3Co4zMTFBW3v1O3vRYS86VnMvto/tmzPeNTzN7YcGOHzDaG8KWkaLeVomgDuBZzLzb2ct3Q9sa463AffNmr+xeWrmMuDYrNs3kqQVsJgr9w8AHwEORcTjzdyngN3APRGxA3ge+FCz9iBwFTAJvAZ8tM2CJUnddQ33zPwGEAssXzHP+QnctMS6JElL4DtUJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCuoa7hHxpYh4KSKenDX3roh4KCKebf58ZzMfEfG5iJiMiCci4v3LWbwkaX6LuXL/B2DLcXNjwMOZuQl4uBkDXAlsar52Al9op0xJ0onoGu6Z+XXgR8dNbwX2Nsd7gWtmzX85Z+wH1kbEuS3VKklapMjM7idFDAEPZOaFzfjHmbm2OQ7glcxcGxEPALsz8xvN2sPAJzPzwDx77mTm6p7BwcGLx8fHW3lBU1NTrFmzppW9+p296LAXHau5F4eOHpszHjwTXnwdhtef3aOKlmbz5s0HM3NkvrWBpW6emRkR3X9D/Pz37QH2AIyMjOTo6OhSSwFgYmKCtvbqd/aiw150rOZebB/bN2e8a3ia2w8NcPiG0d4UtIxO9mmZF9+43dL8+VIzfxTYOOu8Dc2cJGkFnWy43w9sa463AffNmr+xeWrmMuBYZr6wxBolSSeo622ZiPgnYBRYFxFHgE8Du4F7ImIH8Dzwoeb0B4GrgEngNeCjy1CzJKmLruGemR9eYOmKec5N4KalFiVJWhrfoSpJBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBQ30ugBJ6rWhsX3zzh/effUKV9Ier9wlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaBlec49IrYAnwVOA76YmbuX4+dI0nLq5+ffW79yj4jTgM8DVwIXAB+OiAva/jmSpIUtx5X7JcBkZj4HEBHjwFbg6WX4WT/3m3XX8DTbx/b1xW9Wqbp+vvJ9Kyfzula6F5GZ7W4YcS2wJTP/uBl/BLg0M28+7rydwM5m+B7guy2VsA54uaW9+p296LAXHfaio9978auZec58Cz37bJnM3APsaXvfiDiQmSNt79uP7EWHveiwFx2Ve7EcT8scBTbOGm9o5iRJK2Q5wv0/gE0RcV5EnA5cD9y/DD9HkrSA1m/LZOZ0RNwM/Cszj0J+KTOfavvnvIXWb/X0MXvRYS867EVH2V60/g+qkqTe8x2qklSQ4S5JBZUN94jYFREZEet6XUsvRcRfRsQTEfF4RHwtIn6l1zX1SkTcFhHfafrxlYhY2+uaeiUirouIpyLiZxFR8lHAbiJiS0R8NyImI2Ks1/W0rWS4R8RG4HeA/+p1LaeA2zLzvZl5EfAA8Oc9rqeXHgIuzMz3Av8J3NLjenrpSeAPgK/3upBeWA0fk1Iy3IE7gE8Aq/5fizPzJ7OGZ7GKe5KZX8vM6Wa4n5n3YKxKmflMZrb1rvB+9ObHpGTm/wJvfExKGT17h+pyiYitwNHM/HZE9LqcU0JE/DVwI3AM2Nzjck4VfwTc3esi1DPrge/PGh8BLu1RLcuiL8M9Iv4N+OV5lm4FPsXMLZlV4636kZn3ZeatwK0RcQtwM/DpFS1wBXXrRXPOrcA0cNdK1rbSFtML1dWX4Z6ZvzXffEQMA+cBb1y1bwC+GRGXZOYPVrDEFbVQP+ZxF/AghcO9Wy8iYjvwQeCKLP4mjxP472I1Kv8xKX0Z7gvJzEPAL70xjojDwEhm9vOnvi1JRGzKzGeb4VbgO72sp5ea/4nMJ4DfzMzXel2PeurNj0lhJtSvB/6wtyW1q1S4a167I+I9wM+A54E/6XE9vfT3wBnAQ83f7PZn5qrsR0T8PvB3wDnAvoh4PDN/t8dlrZhT4GNSlp0fPyBJBVV9FFKSVjXDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqaD/B/tepd0H3PK6AAAAAElFTkSuQmCC\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" + "name": "stdout", + "output_type": "stream", + "text": [ + "Logsums agree\n" + ] } ], "source": [ @@ -396,6 +384,41 @@ } } }, + { + "cell_type": "code", + "execution_count": 58, + "outputs": [ + { + "data": { + "text/plain": "{'name': 'root',\n 'coefficient': 1.0,\n 'alternatives': [{'name': 'AUTO',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'DRIVEALONE',\n 'coefficient': 0.35,\n 'alternatives': ['DRIVEALONEFREE', 'DRIVEALONEPAY']},\n {'name': 'SHAREDRIDE2',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED2FREE', 'SHARED2PAY']},\n {'name': 'SHAREDRIDE3',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED3FREE', 'SHARED3PAY']}]},\n {'name': 'NONMOTORIZED',\n 'coefficient': 0.72,\n 'alternatives': ['WALK', 'BIKE']},\n {'name': 'TRANSIT',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'WALKACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['WALK_LOC',\n 'WALK_LRF',\n 'WALK_EXP',\n 'WALK_HVY',\n 'WALK_COM']},\n {'name': 'DRIVEACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['DRIVE_LOC',\n 'DRIVE_LRF',\n 'DRIVE_EXP',\n 'DRIVE_HVY',\n 'DRIVE_COM']}]},\n {'name': 'RIDEHAIL',\n 'coefficient': 0.36,\n 'alternatives': ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']}]}" + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ns[0]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": 51, From e821b57b9dd6e9cd9596a3c3453cb96b5129482b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 10 May 2022 14:12:29 +1000 Subject: [PATCH 027/135] nest scaling comment --- activitysim/core/simulate.py | 15 +++ notebooks/check_sf_probs.ipynb | 18 +-- .../larch_probabilities_via_estimation.ipynb | 72 ++++++++++++ notebooks/validate_frozen_impl.ipynb | 107 +++++++----------- 4 files changed, 138 insertions(+), 74 deletions(-) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 3bfd4f7761..aba8d2340c 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -746,6 +746,21 @@ def compute_nested_utilities(raw_utilities, nest_spec): np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) return nested_utilities +# So it looks like TM1 was estimated such that the nest coefficient is the ratio of lower level and upper level +# nest in larch. This means the values can all be between 0 and 1, unlike for larch where they need to be decreasing +# going down the tree. In that world, the above would read +# for nest in logit.each_nest(nest_spec, post_order=True): +# name = nest.name +# if nest.is_leaf: +# # do not scale here, do afterwards so recursive structure works +# nested_utilities[name] = raw_utilities[name].astype(float) +# else: +# # the alternative nested_utilities will already have been computed due to post_order +# with np.errstate(divide='ignore'): +# nested_utilities[name] = \ +# nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) +# nested_utilities[name] /= parent_nest_scale # parent_nest_scale would need to be defined as part of nest +# # and would be = coeffiecient for leaves and for nests it would be that of the parent nest def compute_nested_exp_utilities(raw_utilities, nest_spec): diff --git a/notebooks/check_sf_probs.ipynb b/notebooks/check_sf_probs.ipynb index dfa75b24a2..eb5c355cad 100644 --- a/notebooks/check_sf_probs.ipynb +++ b/notebooks/check_sf_probs.ipynb @@ -234,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 144, "outputs": [ { "name": "stderr", @@ -247,7 +247,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'work' (10000 trips)\n", + "trip_mode_choice tour_type 'work' (100000 trips)\n", "Done\n" ] }, @@ -262,18 +262,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "trip_mode_choice tour_type 'work' (10000 trips)\n", + "trip_mode_choice tour_type 'work' (100000 trips)\n", "Done\n", - "CPU times: user 9.94 s, sys: 844 ms, total: 10.8 s\n", - "Wall time: 9.56 s\n" + "CPU times: user 1min 10s, sys: 3.98 s, total: 1min 14s\n", + "Wall time: 1min 9s\n" ] }, { "data": { - "text/plain": " me asim larch\nSHARED2FREE 0.450% 0.400% 0.359%\nSHARED3FREE 0.220% 0.140% 0.188%\nTAXI 0.110% 0.160% 0.080%\nTNC_SINGLE 0.280% 0.270% 0.202%\nWALK 0.520% 0.480% 0.435%\nWALK_HVY 69.070% 70.030% 60.165%\nWALK_LOC 9.540% 9.310% 14.513%\nWALK_LRF 19.710% 19.120% 23.977%", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
measimlarch
SHARED2FREE0.450%0.400%0.359%
SHARED3FREE0.220%0.140%0.188%
TAXI0.110%0.160%0.080%
TNC_SINGLE0.280%0.270%0.202%
WALK0.520%0.480%0.435%
WALK_HVY69.070%70.030%60.165%
WALK_LOC9.540%9.310%14.513%
WALK_LRF19.710%19.120%23.977%
\n
" + "text/plain": " me asim larch\nSHARED2FREE 0.395% 0.420% 0.359%\nSHARED3FREE 0.214% 0.226% 0.188%\nTNC_SINGLE 0.212% 0.231% 0.202%\nWALK 0.488% 0.466% 0.435%\nWALK_HVY 69.439% 69.422% 60.165%\nWALK_LOC 9.697% 9.596% 14.513%\nWALK_LRF 19.390% 19.469% 23.977%", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
measimlarch
SHARED2FREE0.395%0.420%0.359%
SHARED3FREE0.214%0.226%0.188%
TNC_SINGLE0.212%0.231%0.202%
WALK0.488%0.466%0.435%
WALK_HVY69.439%69.422%60.165%
WALK_LOC9.697%9.596%14.513%
WALK_LRF19.390%19.469%23.977%
\n
" }, - "execution_count": 142, + "execution_count": 144, "metadata": {}, "output_type": "execute_result" } @@ -284,7 +284,7 @@ "trip_id_to_check = 24106017 # obvious results with 1e4 samples: 23695361 24106017\n", "# better with 1e5 samples: 1932588125\n", "# requires higher precision, check when time: 38280865\n", - "num_samples = 10000 # 1e5 takes about 80s\n", + "num_samples = 100000 # 1e5 takes about 80s\n", "\n", "comp = compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs)\n", "comp.loc[(comp > 1e-3).any(axis=1), :].applymap(lambda x: f\"{x:.3%}\")" diff --git a/notebooks/larch_probabilities_via_estimation.ipynb b/notebooks/larch_probabilities_via_estimation.ipynb index db362e994c..f6105fb11a 100644 --- a/notebooks/larch_probabilities_via_estimation.ipynb +++ b/notebooks/larch_probabilities_via_estimation.ipynb @@ -319,6 +319,78 @@ } } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# see if larch scales nest scale" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 46, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 46, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 46, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 0e80ea7dc5..cb12d91201 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -345,7 +345,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 17, "outputs": [ { "name": "stderr", @@ -386,42 +386,7 @@ }, { "cell_type": "code", - "execution_count": 58, - "outputs": [ - { - "data": { - "text/plain": "{'name': 'root',\n 'coefficient': 1.0,\n 'alternatives': [{'name': 'AUTO',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'DRIVEALONE',\n 'coefficient': 0.35,\n 'alternatives': ['DRIVEALONEFREE', 'DRIVEALONEPAY']},\n {'name': 'SHAREDRIDE2',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED2FREE', 'SHARED2PAY']},\n {'name': 'SHAREDRIDE3',\n 'coefficient': 0.35,\n 'alternatives': ['SHARED3FREE', 'SHARED3PAY']}]},\n {'name': 'NONMOTORIZED',\n 'coefficient': 0.72,\n 'alternatives': ['WALK', 'BIKE']},\n {'name': 'TRANSIT',\n 'coefficient': 0.72,\n 'alternatives': [{'name': 'WALKACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['WALK_LOC',\n 'WALK_LRF',\n 'WALK_EXP',\n 'WALK_HVY',\n 'WALK_COM']},\n {'name': 'DRIVEACCESS',\n 'coefficient': 0.5,\n 'alternatives': ['DRIVE_LOC',\n 'DRIVE_LRF',\n 'DRIVE_EXP',\n 'DRIVE_HVY',\n 'DRIVE_COM']}]},\n {'name': 'RIDEHAIL',\n 'coefficient': 0.36,\n 'alternatives': ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']}]}" - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ns[0]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 51, + "execution_count": 18, "outputs": [ { "name": "stderr", @@ -440,8 +405,8 @@ }, { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.004600 0.000000e+00 0.000000 0.000300 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.042100 0.120600 0.832400 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0046000.000000e+000.0000000.0003000.0000000.0421000.1206000.832400
\n
" + "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.005000 0.000000e+00 0.000000 0.000300 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.043900 0.059400 0.891400 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0050000.000000e+000.0000000.0003000.0000000.0439000.0594000.891400
\n
" }, "metadata": {}, "output_type": "display_data" @@ -450,8 +415,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 7.16 s, sys: 594 ms, total: 7.75 s\n", - "Wall time: 6.98 s\n" + "CPU times: user 9.06 s, sys: 562 ms, total: 9.62 s\n", + "Wall time: 9.3 s\n" ] } ], @@ -493,6 +458,42 @@ } } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# Dev corner" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, { "cell_type": "markdown", "source": [ @@ -663,30 +664,6 @@ } } }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "markdown", "source": [ From 7bfb80a80839a6df6326302a0ed09fd25e223713 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 15 May 2022 22:32:41 +1000 Subject: [PATCH 028/135] adds comment regarding potential random choice runtime improvement --- activitysim/core/logit.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index adfafcef2e..de292dc3a1 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -221,7 +221,12 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter next_level_alts = nest_alternatives_by_name[choice_this_level] raise ValueError("This should never happen - no alternative found") - +# Note: this is relatively slow due to the apply. +# It could *maybe* be sped up by using the fact that the nesting structure is the same for all rows: Add ev1(0,1) to +# all entries (as is currently being done). Then, at each level, pick the maximum of the available composite +# alternatives and set the corresponding entry to 1 for each row, set all other alternatives at this level to zero. +# Once the tree is walked (all alternatives have been processed), take the product of the alternatives in each +# leaf's alternative list. Then pick the only alternative with entry 1, all others must be 0. def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_choosers=None): """ walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') From 42901bf2e7a331529ccad5420e996fbb4c676617 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 16 May 2022 16:15:51 +1000 Subject: [PATCH 029/135] adds goodness of fit test and loop over all trips --- notebooks/fru_utils.py | 11 +- .../larch_probabilities_via_estimation.ipynb | 176 ++++++- notebooks/validate_frozen_impl.ipynb | 491 ++++++++++++++++-- 3 files changed, 632 insertions(+), 46 deletions(-) diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index d466184af0..645db8f174 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -162,12 +162,13 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa return trips_merged, choices, nest_spec -def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check): - t_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, - trip_id_to_check=trip_id_to_check, num_samples=num_samples) +# def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check): +# t_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, +# trip_id_to_check=trip_id_to_check, num_samples=num_samples) +# sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0] +# return sim_mode_shares - sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0] - #sim_mode_shares.columns = ["mode_share_sim"] +def comp_mode_shares(base_probs, sim_mode_shares, trip_id_to_check): obs_probs = base_probs.loc[base_probs.index == trip_id_to_check].T obs_probs.columns = ["mode_share_obs"] ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how="outer").fillna(0) diff --git a/notebooks/larch_probabilities_via_estimation.ipynb b/notebooks/larch_probabilities_via_estimation.ipynb index f6105fb11a..1dc495ed64 100644 --- a/notebooks/larch_probabilities_via_estimation.ipynb +++ b/notebooks/larch_probabilities_via_estimation.ipynb @@ -23,6 +23,34 @@ "cell_type": "code", "execution_count": 1, "outputs": [], + "source": [ + "%load_ext autoreload" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "%autoreload 2" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [], "source": [ "import os\n", "import sys\n", @@ -47,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "outputs": [], "source": [ "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", @@ -63,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "outputs": [], "source": [ "# create estimation test example\n", @@ -78,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "outputs": [], "source": [ "# run estimation mode\n", @@ -93,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "outputs": [ { "name": "stderr", @@ -153,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 8, "outputs": [], "source": [ "probs_0 = pd.DataFrame(probs[0])\n", @@ -172,7 +200,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 9, "outputs": [ { "data": { @@ -200,6 +228,142 @@ } } }, + { + "cell_type": "code", + "execution_count": 11, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/mnt/c/Users/jan.zill/code/activitysim/test_estimation_sf\r\n" + ] + } + ], + "source": [ + "!pwd" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 14, + "outputs": [], + "source": [ + "#m_0.graph.to_png(filename=\"/mnt/c/Users/jan.zill/code/activitysim/test_estimation_sf/mtc_test.png\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 15, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'temp' is not defined", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/tmp/ipykernel_24538/1635136243.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mdisplay\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtemp\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", + "\u001B[0;31mNameError\u001B[0m: name 'temp' is not defined" + ] + } + ], + "source": [ + "display(temp)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 17, + "outputs": [], + "source": [ + "subg = m_0.graph.subgraph_from(22)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 23, + "outputs": [ + { + "data": { + "text/plain": "", + "text/html": "" + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "subg.to_png(filename=\"/mnt/c/Users/jan.zill/code/activitysim/test_estimation_sf/mtc_mode_choice_subgraph.png\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 22, + "outputs": [ + { + "data": { + "text/plain": "", + "text/html": "
\n\nTree\n\n\ncluster_elemental\n\nElemental Alternatives\n\n\n\n1\n\nDRIVEALONEFREE \n(1)\n\n\n\n2\n\nDRIVEALONEPAY \n(2)\n\n\n\n24\n\nSHAREDRIDE2 \n(24)\n\n\n\n25\n\nSHAREDRIDE3 \n(25)\n\n\n\n22\n\nAUTO \n(22)\n\n\n\n22->24\n\n\n\n\n\n22->25\n\n\n\n\n\n23\n\nDRIVEALONE \n(23)\n\n\n\n22->23\n\n\n\n\n\n23->1\n\n\n\n\n\n23->2\n\n\n\n\n\n0\n\nRoot\n\n\n
" + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "m_0.graph.partial_figure(source=22)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": 18, diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index cb12d91201..49f3ed786c 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -28,6 +28,7 @@ }, "outputs": [], "source": [ + "import numba\n", "%load_ext autoreload" ] }, @@ -52,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 40, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -73,11 +74,12 @@ "import numpy as np\n", "import pandas as pd\n", "from numpy.random import default_rng\n", + "from statsmodels.stats.gof import powerdiscrepancy\n", "\n", "from activitysim.cli import run\n", "from activitysim.core import inject\n", "\n", - "from fru_utils import run_trip_mode_choice, comp_mode_shares" + "from fru_utils import run_trip_mode_choice, comp_mode_shares, mode_choice_for_trip" ] }, { @@ -140,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 7, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -167,7 +169,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 8, "outputs": [], "source": [ "## need to do this before loading checkpoint tables\n", @@ -185,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 9, "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", "metadata": { "execution": { @@ -202,6 +204,7 @@ "name": "stderr", "output_type": "stream", "text": [ + "setting trace_hh_id is wrong type, should be an int, but was \n", "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" ] }, @@ -220,7 +223,8 @@ "trip_mode_choice tour_type 'social' (19 trips)\n", "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n" + "Done\n", + "Running with %d trips 482\n" ] }, { @@ -234,7 +238,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running with %d trips 482\n", "trip_mode_choice tour_type 'atwork' (27 trips)\n", "trip_mode_choice tour_type 'eatout' (33 trips)\n", "trip_mode_choice tour_type 'escort' (6 trips)\n", @@ -246,8 +249,8 @@ "trip_mode_choice tour_type 'univ' (26 trips)\n", "trip_mode_choice tour_type 'work' (168 trips)\n", "Done\n", - "CPU times: user 17.8 s, sys: 2.34 s, total: 20.2 s\n", - "Wall time: 16.7 s\n" + "CPU times: user 18.3 s, sys: 2.3 s, total: 20.6 s\n", + "Wall time: 17.8 s\n" ] } ], @@ -259,14 +262,14 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 10, "outputs": [ { "data": { "text/plain": " trip_mode_fru trip_mode_asim\nWALK 309 304\nWALK_LOC 85 85\nWALK_LRF 55 53\nBIKE 15 17\nTNC_SINGLE 7 4\nDRIVEALONEFREE 5 4\nSHARED2FREE 3 4\nSHARED3FREE 2 1\nWALK_HVY 1 4", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK309304
WALK_LOC8585
WALK_LRF5553
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n
" }, - "execution_count": 15, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -286,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 11, "outputs": [ { "name": "stdout", @@ -311,7 +314,30 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 12, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.000000e+00\n" + ] + } + ], + "source": [ + "x_ = np.abs((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim)).values\n", + "print(f\"{x_.max():e}\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 13, "outputs": [], "source": [ "#ls_ = np.abs(((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru))\n", @@ -345,7 +371,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "outputs": [ { "name": "stderr", @@ -375,7 +401,8 @@ ], "source": [ "from fru_utils import get_stuff\n", - "t, ru, neu, nu, ns, nested_probs, base_probs = get_stuff() #do_these_purposes=[\"social\"]) #social" + "t, ru, neu, nu, ns, nested_probs, base_probs = get_stuff() #do_these_purposes=[\"social\"]) #social\n", + "bp = pd.concat(base_probs)" ], "metadata": { "collapsed": false, @@ -386,7 +413,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 107, "outputs": [ { "name": "stderr", @@ -400,45 +427,401 @@ "output_type": "stream", "text": [ "trip_mode_choice tour_type 'social' (10000 trips)\n", - "Done\n" + "Done\n", + "CPU times: user 8.39 s, sys: 453 ms, total: 8.84 s\n", + "Wall time: 9.28 s\n" ] - }, + } + ], + "source": [ + "%%time\n", + "#prun\n", + "\n", + "num_samples = 10000\n", + "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like 65s and 1e4 takes 8s\n", + "# work 615236801 also in trace log\n", + "# social 2464104885 642446345 1767182945\n", + "\n", + "\n", + "#bp = base_probs[7] # 7 for social, 9 for work. Check t[t.index == trip_id_to_check] to see which purpose, and\n", + "# output above to see which index\n", + "# choose_individual_max_utility = False\n", + "#ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", + "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", + "\n", + "choose_individual_max_utility = True\n", + "t_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility,\n", + " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", + "sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", + "ms_comp = comp_mode_shares(bp, sim_mode_shares, trip_id_to_check)\n", + "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "#### generic power discrepancy implementation in statsmodels\n", + "\n", + "lambd{float, str}\n", + "float : exponent a for power discrepancy\n", + "\n", + "‘loglikeratio’: a = 0\n", + "\n", + "‘freeman_tukey’: a = -0.5\n", + "\n", + "‘pearson’: a = 1 (standard chisquare test statistic)\n", + "\n", + "‘modified_loglikeratio’: a = -1\n", + "\n", + "‘cressie_read’: a = 2/3\n", + "\n", + "‘neyman’ : a = -2 (Neyman-modified chisquare, reference from a book?)\n", + "\n", + "note - they are all based on continuous approximations (x^2) so technically they might be inadequate for very small\n", + "occurences of counts" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 64, + "outputs": [ { "data": { - "text/plain": " SHARED2FREE SHARED3FREE TAXI TNC_SHARED TNC_SINGLE \\\nmode_share_obs 0.004883 4.701403e-14 0.000096 0.000287 0.000131 \ntrip_mode 0.005000 0.000000e+00 0.000000 0.000300 0.000000 \n\n WALK WALK_LOC WALK_LRF \nmode_share_obs 0.044905 0.059962 0.889736 \ntrip_mode 0.043900 0.059400 0.891400 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREETAXITNC_SHAREDTNC_SINGLEWALKWALK_LOCWALK_LRF
mode_share_obs0.0048834.701403e-140.0000960.0002870.0001310.0449050.0599620.889736
trip_mode0.0050000.000000e+000.0000000.0003000.0000000.0439000.0594000.891400
\n
" + "text/plain": "(0.3439655414093617, nan)" }, + "execution_count": 64, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" + } + ], + "source": [ + "# Note power tests will be inf when expected count is zero (any non-zero count will be impossible)\n", + "eps = 0 #1e-14\n", + "non_zero_shares = ms_comp.loc[(ms_comp.mode_share_obs > eps) & (ms_comp.trip_mode > eps)][[\"mode_share_obs\", \"trip_mode\"]]\n", + "ddof = non_zero_shares.shape[0] - 1 # what to use here? num alternatives - 1? What about zero expected count?\n", + "\n", + "non_zero_shares.trip_mode *= num_samples\n", + "\n", + "sum_ms = non_zero_shares.mode_share_obs.sum()\n", + "if sum_ms != 1.0:\n", + " if np.abs(1.0 - sum_ms) > 1e-1:\n", + " print(f\"NOT GOOD {sum_ms} should be one\")\n", + " #print(f\"sum ms is not one but {sum_ms}, re-normalising\")\n", + " non_zero_shares.mode_share_obs /= sum_ms\n", + "\n", + "powerdiscrepancy(non_zero_shares.trip_mode.values, non_zero_shares.mode_share_obs.values, lambd=0, ddof=ddof)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 99, + "outputs": [ + { + "data": { + "text/plain": "array([[4.17345080e-03, 5.98926219e-03, 5.00000000e-03],\n [1.45406892e-04, 6.18851051e-04, 3.00000000e-04],\n [4.13473933e-02, 4.66025328e-02, 4.39000000e-02],\n [5.64412252e-02, 6.25036058e-02, 5.94000000e-02],\n [8.87346329e-01, 8.95325012e-01, 8.91400000e-01]])" + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from statsmodels.stats.proportion import multinomial_proportions_confint\n", + "ci = multinomial_proportions_confint(non_zero_shares.trip_mode.values, alpha=0.999, method='goodman')\n", + "vals = non_zero_shares.trip_mode.values / num_samples\n", + "np.hstack((ci, vals.reshape((len(vals),1))))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 106, + "outputs": [ + { + "data": { + "text/plain": "array([ True, True, True, True, True])" + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numba\n", + "\n", + "@numba.jit\n", + "def within(lower, upper, x):\n", + " return (x >= lower) & (x <= upper)\n", + "\n", + "@numba.jit\n", + "def numpy_within(x):\n", + " lower = x[0]\n", + " upper = x[1]\n", + " val = x[2]\n", + " return within(lower, upper, val)\n", + "\n", + "test = np.hstack((ci, vals.reshape((len(vals),1))))\n", + "np.apply_along_axis(numpy_within, 1, test)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# #from scipy.special import rel_entr\n", + "# from scipy.stats import linregress\n", + "# linregress(ms_comp.mode_share_obs.values, ms_comp.trip_mode.values)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# TODO for paper:\n", + "\n", + "automate such that for a given sample size we calculate all probs for ALL 482 trips. then scatterplot and lin reg\n", + "\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'atwork' (10000 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "trip_mode_choice tour_type 'atwork' (10000 trips)\n", + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'atwork' (10000 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "trip_mode_choice tour_type 'atwork' (10000 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "trip_mode_choice tour_type 'atwork' (10000 trips)\n", + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'atwork' (10000 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "trip_mode_choice tour_type 'atwork' (10000 trips)\n", + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 9.06 s, sys: 562 ms, total: 9.62 s\n", - "Wall time: 9.3 s\n" + "trip_mode_choice tour_type 'atwork' (10000 trips)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "trip_mode_choice tour_type 'atwork' (10000 trips)\n", + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'atwork' (10000 trips)\n", + "Done\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'atwork' (10000 trips)\n" ] } ], "source": [ "%%time\n", - "#prun\n", "\n", "num_samples = 10000\n", - "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like 65s and 1e4 takes 8s\n", - "# work 615236801 also in trace log\n", - "# social 2464104885 642446345 1767182945\n", "\n", - "bp = base_probs[7] # 7 for social, 9 for work. Check t[t.index == trip_id_to_check] to see which purpose, and\n", - "# output above to see which index\n", - "# choose_individual_max_utility = False\n", - "#ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", - "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", + "trips_to_check = bp.index.values #[100:103]\n", "\n", - "choose_individual_max_utility = True\n", - "ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", - "display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" + "res = []\n", + "\n", + "for i, trip_id_to_check in enumerate(trips_to_check):\n", + " if i == 50:\n", + " print(f\"Processing {i} out of {trips_to_check.shape[0]}\")\n", + " _, c_, _ = mode_choice_for_trip(choose_individual_max_utility=True,\n", + " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", + " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", + " res.append(sim_mode_shares.rename(trip_id_to_check).to_frame().T)\n", + "\n", + "result_sim = pd.concat(res).fillna(0)" ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + } + }, + { + "cell_type": "code", + "execution_count": 139, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { @@ -482,6 +865,44 @@ } } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# = bp.merge(result_sim, left_index=True, right_index=True, how=\"outer\").fillna(0)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "markdown", "source": [ From 3700a9e76decb7d4c0582ddf7cba759081a2edda Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 16 May 2022 17:43:32 +1000 Subject: [PATCH 030/135] some validation metrics --- notebooks/validate_frozen_impl.ipynb | 245 +++++++-------------------- 1 file changed, 60 insertions(+), 185 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 49f3ed786c..7444b9df2e 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -594,11 +594,7 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [ - "# #from scipy.special import rel_entr\n", - "# from scipy.stats import linregress\n", - "# linregress(ms_comp.mode_share_obs.values, ms_comp.trip_mode.values)" - ], + "source": [], "metadata": { "collapsed": false, "pycharm": { @@ -624,172 +620,7 @@ { "cell_type": "code", "execution_count": null, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (10000 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n", - "trip_mode_choice tour_type 'atwork' (10000 trips)\n", - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (10000 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n", - "trip_mode_choice tour_type 'atwork' (10000 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n", - "trip_mode_choice tour_type 'atwork' (10000 trips)\n", - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (10000 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n", - "trip_mode_choice tour_type 'atwork' (10000 trips)\n", - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (10000 trips)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Done\n", - "trip_mode_choice tour_type 'atwork' (10000 trips)\n", - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (10000 trips)\n", - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (10000 trips)\n" - ] - } - ], + "outputs": [], "source": [ "%%time\n", "\n", @@ -807,7 +638,8 @@ " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", " res.append(sim_mode_shares.rename(trip_id_to_check).to_frame().T)\n", "\n", - "result_sim = pd.concat(res).fillna(0)" + "result_sim = pd.concat(res).fillna(0)\n", + "result_sim.to_csv(\"temp_res_sim_10ksamples.csv\")" ], "metadata": { "collapsed": false, @@ -819,9 +651,11 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 158, "outputs": [], - "source": [], + "source": [ + "#result_sim.merge(bp, left_index=True, right_index=True, how=\"outer\", suffixes=[\"_sim\", \"_asim\"]).fillna(0)" + ], "metadata": { "collapsed": false, "pycharm": { @@ -831,9 +665,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 155, "outputs": [], - "source": [], + "source": [ + "merged_results = result_sim.reset_index().rename(columns={\"index\": \"trip_id\"}).melt(id_vars=[\"trip_id\"]).merge(\n", + " bp.reset_index().rename(columns={\"index\": \"trip_id\"}).melt(id_vars=[\"trip_id\"]),\n", + " on=[\"trip_id\", \"variable\"],\n", + " how=\"outer\",\n", + " suffixes=[\"_sim\", \"_obs\"]\n", + ").fillna(0)" + ], "metadata": { "collapsed": false, "pycharm": { @@ -843,9 +684,32 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 157, + "outputs": [ + { + "data": { + "text/plain": "" + }, + "execution_count": 157, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA14ElEQVR4nO3deXzU9Z348dd7kkkm950AISEEgkC4jXhUaZXapS6CWmutW3uslke3a6W1v23toVbs9tzaymq30mpb3W1F62rRVdsKtGgrlqCIgEhCICGQ+5hcM5nj+/n9MYdJCDBKJte8n4+HD+f7ne9kPl8C857P9X6LMQallFKxyzbWDVBKKTW2NBAopVSM00CglFIxTgOBUkrFOA0ESikV4+LHugHvRW5urikpKRnrZiil1ISye/fuVmNM3tDzEzIQlJSUUFlZOdbNUEqpCUVEaoc7r0NDSikV4zQQKKVUjNNAoJRSMU4DgVJKxTgNBEopFeOiumpIRB4GVgPNxpgFwzwvwH3AFUAf8GljzGvRbJNSSk00nS43hxp7aerqpyA9kYykOAoyHGQmOUbk50d7+eivgPuBR07x/IeBsuB/5wP/Ffy/UkopAkFgx9ttVLf0YBmobu5mVl4qnX0ezpnKiASDqAYCY8wOESk5zSVrgUdMIBf2ThHJFJGpxpiGaLZLKaUmiiPNfRzvdLFpRw1ur4XDbmP9yjKKspI41NjL8plnHwjGeo6gEDg24Lg+eO4kIrJORCpFpLKlpWVUGqeUUmPFsgw1LT109/u4b2sVbq8FgNtrcd/WKrr7fTR19Y/Ie02YncXGmE3AJoCKigqtpqOUmnQsy1DX3ktzdz8NTjc1LT3Mm5oeDgIhbq9Fn8dPQXriiLzvWAeC40DRgOPpwXNKKRVT3G4f+xu7qGnt5Y7f7wsPA/3m5vNx2G2DgoHDbiMvNZFZBckj8t5jPTS0BfikBFwAOHV+QCkVS3w+iwMnOvnb0TYanO5wEIDAN/9ut4tvX7UAhz3wce2w2/j2VQuYVZA8MVYNichvgQ8AuSJSD9wF2AGMMT8DniOwdLSawPLRz0SzPUopNZ74fBbP72+gy+3jnmcPcPMlpScNA33ql3t4fv0FPPKZ5TR1B5aPzpmSMmJBAKK/aujjZ3jeAP8azTYopdR45Hb72NvgBISirCQunJkNMOww0PP7Wpk7JZ1/XDgVm01GvC1jPUeglFIxw7IMR9t6aeh00e+zuPvZ/dS2uXDYbdy9ppxXD7dy62VlbNxWFZ4juGftAs4tzqQkNzUqQQA0ECilVNSFVgO9VtfJ1596M/whf+tlZTy6s5YGp5u7tuznwRvP5fYn32TdilLmTkljWoaD+VMySEiIi2r7xnqyWCmlJjXLMrywv5H/ff14OAhAYCJ447Yqrlk2PXzc2eelo8/DzJwULivLZ0lxdtSDAGggUEqpqPH5LF6r6+C2x/dgGYbdDyDB0R6H3caUdAfP3nIxa5cU4nCM3oCNDg0ppdQIsyzDkZYejjvdOF1ebr6klNTEuGEngo0J/H/DmgUsnpYxqgEgRAILdyaWiooKozWLlVLjjc9nsf+Ek8bufrpc3kEbw770wTk44m1894WDgyaCM5LtZCXZWTg1+kFARHYbYyqGntcegVJKjYDQnoCq5h7K8tNO2hj24xcPcculs1m3opTirGRaevpx2G1cNief+PixHaXXQKCUUmfJsgz7Tjip7whkCR1uY5jba1GUnUxWsh2fZVGWn0r5tIwxDwKggUAppd4zn89i3wknxzr6yEtLHJQldLj5gOzkBDpdHv5h3pRRWQ0UqbEPRUopNQH5fBZP7TnO9T/fyRd+u4e/HW4Lf/A/ubueWy8rG5Qf6J61CyjIsPPh8qnjKgiA9giUUupdCe0Obu3pHzQPYJl3egENTjeP7qxl3YpS5k9NJzPZzpJpmWOyIigS2iNQSqkIWZbhpepmDpzooqmrf9DQz9BeQGhj2NwpaSwvyRm3QQC0R6CUUhHx+SzePNFJr9sPAplJ9kHzAA1ON5sr69h047k0d/czMzeFpUVZUcsPNJI0ECil1Bm4XF6ee6uJbwzIE7RhzXzuXlPOXVv2h899/gOzqWvrJSM5gcWFmRMiCIBuKFNKqVMKbRDrcHn53H/vPmkV0DevmEtRTgqdfd7wstCclETKp46PZaFD6YYypZSKQGgy2Gf56Oj109TlJiHeNuy+gMZuD809Hs6fmc3UjCRKclImTC9gIA0ESikVFMoU+ueDDVSU5HHnlsCqoPUrZw+7L8AmsLQok/NLcsZlDyBSE7flSik1QizLUNvWQ+XRdrrdXj5y7oxwEAB4vLKe9SsH7wtYv7KMuQVpvK80d0IHAdAegVIqxlmWYU99Gy6Pob7DRXJCPJ0uz6Bv/w1ON4+8UsuDN55Ll8tHuiOe6ZlJzMyLXtWw0aSBQCkVk0JVwxqcbpq7+6lp6eHxyno6+jx85+qFzMhJorbNFb6+o8+DIz6OhFRhaWHWuN4X8G5NnjtRSqkIeTx+Xj/eQV2ba1Cq6FDpyK8/9SY/vWEZn//NawOWiy5genYiU9Mn5oTw6WggUErFFJ/P4s/VLSTFx52UKnrjtipuuriUB7ZX09vv45F/Xk5TVz9T0hNHpV7AWJmcd6WUUsOwLMPe4524PH46ez2nLB3psNvITk1gWVHWhJ8IjoQGAqVUTAgtDb3t8T1nXBK6Ye0ClhVmxkQQAF0+qpSKEUfbesNBAIZfEvrvVy/ksnPyuWrRNJKS7GPZ3FGlPQKl1KQT2h3c1OWmIN1BSU4KTV3uYZeE/vDaxVQ1d3NhaQ7nzciOmV7AQBoIlFKThmUZjrT28lZDF/UdfRjA5fVz3oxspmclnTQU1NHnoaq5m5k5KTEbBECHhpRSk4TH42dHVQu/f+M4B5u6+e2uOgCeqKzns49W8uZxJ/ffsHTQUNAdq+dzYWkOVy6aFrNBADT7qFJqEvB4/Dy7r4GvD0gTfetlZWyurGP1okIe2F6Nw27j/75wCQB17b0kJ8RTkJ5Icfbk2xdwKpp9VCk1Kfl8Fn870hYOAjB4T4AEP+PdXouWHjcXlOYyKz91DFs8/mggUEpNSKEUEfWdrpMmgiHwwR9nA3/wtMNuIz/NMQYtHf+iPigmIqtE5G0RqRaR24d5vlhEtovI6yKyV0SuiHablFITm89n8eqRNuo7XLR095OSGB8e+w9x2G3Mm5LO/75Wj8Nu497rllCSkzJGLR7fotojEJE44AHgcqAe2CUiW4wxBwZc9k3gcWPMf4nIfOA5oCSa7VJKTUyWZTjW0cuxdhe17X3c8+wB3F6LGTlJ3LW6nLuffads5HeuXsi8qWn8+GOLyU9zTNiiMaMh2kNDy4FqY0wNgIg8BqwFBgYCA6QHH2cAJ6LcJqXUBOLzWexvcNLR109vv0Wny0tTl5tNO2rCw0G1bS5+tqOaH1y7mENN3Zw3I4sLZ+aQkBBHSa7OB5xJtIeGCoFjA47rg+cG+hbwCRGpJ9Ab+MJwP0hE1olIpYhUtrS0RKOtSqlxxuPx89Se43xs0052HXXydlM39zx7AMtw0pxAbZuLQ03d/OKlGgozk0hIiBujVk8842Hh7MeBXxljpgNXAI+KyEntMsZsMsZUGGMq8vLyRr2RSqnRZVmG3cc6whlCRRgUAIabE7AJ3HvdEkrztBfwbkR7aOg4UDTgeHrw3EA3AasAjDGviIgDyAWao9w2pdQ4VtfeS5fbF14Cek5BGg2dfTjsNp7cXc+tl5WxcVvVoDmBZcWZMbUvYKREOxDsAspEZCaBAHA9cMOQa+qAlcCvRGQe4AB07EepGObx+Gnq7qfL5eWhl2vCH/bfurKcr394Lt95/iCP7qxl3YpSyvJTmTclndJJUjZyLEQ1EBhjfCJyC/AHIA542BizX0Q2AJXGmC3Al4Gfi8iXCEwcf9pMxO3OSqn3bGCSuMIsB9XNvbR093Oi00VWcgINzsA+gW89s59ffKqCez+6GATK8tOYpQHgrEV9Q5kx5jkCk8ADz9054PEB4H3RbodSanyyLMO2t5uoae5hRm4qx4908M2nTy4fGQoGXS4vhVlJlE/NiOn8QCNJ/xSVUmOqrr2XEx0u/Ab2n3CGgwC8kyrimmXTgcCE8Nwp6SyOkcpho0VTTCilRs3QOgHFWck0d/WTaI+jrddDvM122vKRujs4OjQQKKVGxdBSkQ67jQ1rF/DA9ipq21w47DZ+fN2SYctHVszIYu3ii3U+IEq0b6WUGhVDS0W6vRZ3/n4fqxcVho+/98Jb3LF6/qCaAd+9eiEXzcyhrCBNg0CUaI9AKRVVoeGgQ03d3HxJKU/urqfB6QbeGfYJqW1z0e32sm5FKXMK0pg3JZ2ZubovINo0ECilRtTAeYBpmQ72HHPy1Sf3DrsKyGG3MXCxuMNuY2FhBlPSkzQAjCIdGlJKjZjQPMBnfvV3dtd2UNvWFw4CMHgVkMNu467V5Ty7N5BsIDQZfGGwcIwGgdGjPQKl1Ig52tbLwy8f5taVZbR199PW6xl2FdCM7CR+cO1iHvlbDbevmoeFYW6BDgONFQ0ESqmzMnAoCIHbLp/DkTYXPR4/va29w64Cqm13UT4tjX/7h3kxVzd4PNKhIaXUexYaCrpi40t8/OevcvuTe3H7TDhV9OOV9SetArr1sjKe3XucuVPSOb80h5JcHQYaa9ojUEq9Z0OXhK5eVMieY53h444+D5v/Xse91y3hYGMXfgs2V9bx1VXzdGPYOCITMb9bRUWFqaysHOtmKBXTLMvwel0HJ5xu+vp9tPb2E2cTevr9/OKlGrKSE7jxghls3FZFVnICH62YrplCx5iI7DbGVAw9rz0CpVREPB4/e084aexyMzXdgR8/u486uW/rOzUB7lo9nxcPNIZrBQxMFT1/aoZOBo9TGgiUUmfkdvvY8mYDd255JyvohjXlvF7XNmhp6N3PHuCBG5ax4dn93HRxKXE2qJiRzUWlOZokbhzTQKCUOi3LMrxxwhkOAhBMD7FlPw/eeC5/PNAavtbttbAsw8brl9Ln8VOQ7qAkR3sB450GAqXUsELLQlu6+8O1AAZyey06+7yDzjnsNmblp2rN4AlG+2pKqUEsy3C0tYen9xznio0v8VJ1K9kp9mGLxeenJQ5aGqppoicm7REopcJC+wIONnaxaUdNuBfw3zuPcPeacu7asn/AHMECpmc7+PVnltPn8VGcnaKTwROUBgKlVFhoX8DNl5SGg8CTu+u58YIZbH2rgQdvPJfOPi/5aYlkJMVTlJVKUZYOA010OjSklMKyDNVN3VQ1dfPTG5bxvlnZVMzIAKDB6ebRnbXMnZpJY6ebQ0093P6/e3HY9XvkZKG/SaVinGUZnt/XwJefeCM87LN+ZRk3XlgCHKWy1klHn4ckexw/2VpFR59H5wImGQ0ESsUoyzLUtPTQ2OUOBwEIrAa6b2sV61aUcuvKOTR39ZOZbCclMZ4lRZm6JHQS0qEhpWKQ2+1jyxsnWH3/y+w80j7s0lDLgNPl4z+3V5GXlsj5M3O4cFaupoeYhDQQKBVj+lwe/l7Xzu3/+07BmOGWhtoEMpLi+eqqeSwszNQP/0lMA4FSMcKyDFWN3fy5qhWnyzdoVdD/+9A5g/YDrF9Zxuy8VPLSElhVPkWDwCSncwRKTXKhDWKvHevkm08H0kTcf8PScMGYBqebh14+wlf+4RymZSbhsMeRlGBDxHBOQYYGgRiggUCpScqyDLVtveyu66SuvXfQBrH6jj7WrywLZw7t6PPQ77NISYwjxR5PTloCRVk6IRwrNBAoNcmEAkBdRx9dLh+t3W7ibYPLRf76b7V8bkUp61aUYhmwCczISeb8GTkkJMSNYevVWNBAoNQk4vNZ7DzaRqPTzbH2Ph6vrKejz8NPPrZkUO3gBqebh/92hHvWLqCn38fsvBRm56drDyBGaSBQaoILZQlt6+2nvt3F1556M7wx7NbLynh0Zy3fff4t7lg9n3uePRB+7pZLy0hJiGd5cTYOh34UxLKo//ZFZBVwHxAH/MIY871hrrkO+BZggDeMMTdEu11KTQahJHG3Pb6Hmy4u5aGXawZtDNu4rYqbLi7lge3VdLu9rFtRSnF2MnlpiczITqY4W+cBVISBQERSAJcxxhKROcBc4HljjPcMr4sDHgAuB+qBXSKyxRhzYMA1ZcDXgPcZYzpEJP893otSMSPUCzja2svbjV1kJScgwrAbw0QCS0LnTkknPk6ItwnLpmfpXIAKi7RHsAO4RESygD8Cu4CPAf90htctB6qNMTUAIvIYsBY4MOCazwIPGGM6AIwxzZE3X6nYM7AXMHAIyGZj0DwAvLMx7K4ry+n3WZTmpTI9U3sBarBIN5SJMaYPuAb4qTHmo0B5BK8rBI4NOK4PnhtoDjBHRP4qIjuDQ0knN0BknYhUikhlS0tLhM1WavII7QfYdbQ9HATgnSEgn99w62VlgzaGffuqBVwwM5vZ+SlcWpZHcbamh1Ani7RHICJyIYEewE3BcyPVr4wHyoAPANOBHSKy0BjTOfAiY8wmYBNARUWFGaH3VmpC8PksXjzYxJHWXlxe/7BDQG6fxTNvHOfe65ZQ09JDSU4KUzMcZKUkaJI4dVqR9gi+SGAc/yljzH4RKQW2R/C640DRgOPpwXMD1QNbjDFeY8wR4BCBwKCUIhAEdtW243R5cXv9JCfEDZsbaG5BGqsXFfL9F94iziY4EmwsLc7SJHHqjMSYyL9ci0g6YIwx3RFeH0/gg30lgQCwC7jBGLN/wDWrgI8bYz4lIrnA68ASY0zbqX5uRUWFqaysjLjdSk1ElmWoa+/ltbpOvj5gSeiXPjgHR7yN775wMHzunrULKM5JosvlI91hpyA9UVcEqZOIyG5jTMXQ85GuGqoAfgmkBQ6lE/hnY8zu073OGOMTkVuAPxAYSno42KPYAFQaY7YEn/uQiBwA/MC/nS4IKDXZhQLAkdY+evt94SAAgSGgH794iFsunc1NF5cSZ4M5+WkkxNvIS3Vw/kwtG6nevUjnCB4GPm+MeQlARC4mEBgWnemFxpjngOeGnLtzwGMD3Bb8T6mYFUoNcbzTRVuvh5qWHuKGpIaAd+YDHnq5htsun4PXskizx2nFMPWeRRoI/KEgAGCMeVlEfFFqk1Ixx7IM295uoqqpJ5wIzmG38ePrlgy7JHR5SRYXlVbgiI/TBHHqrJ02EIjIsuDDv4jIg8BvCez+/Rjw5+g2TanYcaS1l731zkEZQt1ei++9cHJqiDtWz8dguKA0Tz/81Yg4U4/gR0OO7xrwWJdwKjUCfD6LI609WObkncG1ba5waoi5U9KwIUzPTmL+VK0ToEbOaQOBMebS0WqIUrHE4/HzZoOTlu5+khPiyEi2k54YN+wwUJ/HT3F2Mk+/foyrlhZrEFAjLtJVQxkEegMrgqf+Amwwxjij1TClJqs+l4dX6zp5va4Dy8Azbxzn+vOKmZrh4Osfnst3nh+8LHRWXgpev5/bP1yuG8NUVLybVUP7gOuCxzcSWDV0TTQapdRk5XJ5ee5AM998enCq6Md21bF2SSHzpqTzw2sX09fvo73PQ06qHZ+xOK8kVwOAippIA8EsY8xHBhzfLSJ7otAepSYlyzIc6+ilwekOBwEYnCraMvBWYxcbt1aH8wQJsGx6tgYBFVWRBgKXiFxsjHkZQETeB7ii1yylJgfLMhxp7eWthi4yk+209XqG3RcQF8wYMTs/jVtXzmZpcRYzs5MoztH0ECr6Ig0EnwMeCc4VAHQAn4pOk5Sa+CzLUNPSw4HGLg4394RLRv7y0+cNOyG8pCiT3n4/aY441iyapvmB1KiKKBAYY94AFgdzDWGM6Rr4vIh8yhjz6yi0T6kJJZQeYnddJ98YpmTkL/96mA1ryrlzy/7wc9++aiG9/R6SEuysKMvXAKBG3btKOnfKHyLymjFm2ZmvHBmadE6NR6GCMQcbuwZtDIPAt/5Qycj/uHYBeelJdPZ5KUhPJCMpnsT4eF0RpKLuVEnnIk1DfcafP0I/R6kJJzQMFCoYM9zGsIElI/PTk2h0uokTwenyUpaXrkNBakyNVCDQXcYqJoV6AVdsfImXqlvDAWC4egE2gW9dWY7ba5GSGE9RVhIrzykgPn6k/hkq9d5oj0Cps3C0rXdQ2UiH3caTu+tPKhl5x+r5LJ+Zze921+Hy+lk1fwqLi7M0CKhxIdJVQ2fy1xH6OUpNGB6PnxOdLm6+pBSAHW83c+tlZWzcVsWjO2tZt6KUmbkpJMXHkZIYR3aynR9cu0TnAtS4E2mKiQLgO8A0Y8yHRWQ+cKEx5iEAY8wtUWyjUuOGz2ex/4STE043Dnsc33pmH7VtrvDKoBf2NbBuRSlLizJJTYyn0+UlKzmBRdMySEgYqTLfSo2sSHsEvyKQUuIbweNDwGbgoSi0SalxJxQAjrb3UdPyzr6A0LLQBqebjduqgllC03n/HF0GqiaOSAcoc40xjwMWBEpQEigrqdSk53b7ePVoGweburGJ8NfqFm68YAZZyQls3FbFNcumB67zWsybms6q8ikaBNSEEmmPoFdEcgiuDhKRCwDNPKomPY/HzzP7Grjj9/vCG8DuWl3Ok6/Vcc2y6TywvRoJfuY77DbyUhM1CKgJJ9JAcBuwBZglIn8F8oBro9YqpcZQaHdwY5ebzj4v9R19ZCUn0OB04/Za3P3sfn5w7WIONXXjsNswJhAENqxZwKJpGWd+A6XGmUhTTLwmIu8HziGwVPRtY4w3qi1TapSFisc3drmo73AP6gUMnAtwey3cHh82gXvWLiAn1c6KsuUs1glhNUFFumrok0NOLRMRjDGPRKFNSo06n8/ixYNNHGntxeX1n1Q7OJQq+oHtgRTRhVlJFKQn4vZZzM5PZXqmLglVE1ekQ0PnDXjsAFYCrwEaCNSE5/NZ7Kpt50BDIEfQzZeUnjZFxIa1C0h3xCMizC1I101hasKLdGjoCwOPRSQTeCwaDVJqNHk8fv50sAljoDAjiZsvKSX1FLWDl5dkMbdgMWlJcZRPy9QegJo03uvO4l5g5kg2RKnRFEoU19Ttpq69j/u2VoXnA770wTl8bdVcvvvCO7WDN6wpJ80Rz7SMJE0QpyadSOcInuGdxHI2YD7weLQapVS0WJbhcEsPVc09WMaQkWQPBwEIDAH9+MVD3HLpbNatKGVWXiq5qQl4/Bbz8tNxOEYqK4tS40ekf6v/Y8BjH1BrjKmPQnuUigrLMtR39lJ5tJOvDygY8+2rFgw7H1CSm0K6Ix6fZeGwx3FecbauCFKTVqRzBH+JdkOUihafz+KVIy3YxMbRtl5uvqSUJ3fX0+B0c6y9b9j5gCnpDnJTEyjO1tVAavI7bSAQkW6GrzUggDHGpEelVUqNEI/Hz57jnTQ6PcPuC3i8sp47Vs/nnmcPhJ/70UeXsKw4SwOAihmnDQTGmLTRaohSI8myDIebezjY1E12ij0cBGDwvoCHXq6h1+3loU+dh8vjoyQnRSeDVcx5VzNfIpJPYB8BAMaYuhFvkVJnybIMz+9r5MtPBArG3Lpy9rDzAHE2uGP1fNId8aQ74njf7NwxarFSYyuinTAiskZEqoAjwF+Ao8DzEb52lYi8LSLVInL7aa77iIgYETmpsLJSkbIsw5v1nbzd1MXNl5QyNcOBZYYvHXn+zGyKshz8+m9HSEm0j1GLlRp7kfYI7gEuAF40xiwVkUuBT5zpRSISBzwAXA7UA7tEZIsx5sCQ69KA9cCr76bxSg3k81k8t6+Brzy5d9BcwAv7GsKVwwauFvrFjhpeOdLOvdcFqoYpFasiDQReY0ybiNhExGaM2S4iP4ngdcuBamNMDYCIPAasBQ4Mue4e4PvAv0XYHqWAdxLFnXC66Hb7ONzSMyhTaGguYHNlHT+4djHVzd1cPDuX5IQ4CjNncceV5Vo6UsW8SANBp4ikAi8B/yMizQR2F59JIXBswHE9cP7AC0RkGVBkjPk/ETllIBCRdcA6gOLi4gibrSYzj8dPZV07R9r6Bq36GZopNM4GH6so5kd/PMhX/mEuFTOy9YNfqQEizZa1HcggMHzzAnAYuPJs31xEbMC9wJfPdK0xZpMxpsIYU5GXl3e2b60mMJ/P4o1jHTx/oJH2Pi+bdhw+aUVQqGqYw27jwtIc5k9L46FPnseHF0zVIKDUEJH2COKBPwLtBGoVbzbGtEXwuuNA0YDj6cFzIWnAAuDPEijzNAXYIiJrjDGVEbZNxQiPx8++BifHOlynrBsMgzOFrl9ZBsCKMq0hrNSpRNQjMMbcbYwpB/4VmAr8RURejOClu4AyEZkpIgnA9QQqnYV+rtMYk2uMKTHGlAA7AQ0CahCfz+LN+k5+v/cEN/ziVdY/tocHd9QMWzcYAgHgnII01q0oZWZuCufpUJBSp/VuE6k3A41AG5B/pouDRe5vAf4AvAU8bozZLyIbRGTNu22sij0+n8XTbxznT281Dbsp7Jpl08PzABAIAv9+9UKykuNZu7iQD82fovUClDqDSLOPfh64jkCt4ieAzw5dAnoqxpjngOeGnLvzFNd+IJKfqSY/yzIcbeultbufbz6974zFYipmZLHx+iWU5CQzf2qGfvgr9S5EOkdQBHzRGLMnim1RCgC328dzBxr5+lNvcveV5eEAMFxyOJvA+pVlpCXGc1FprgYApd6DSOcIvqZBQI0Gj8fP3462hVNFJyfG47DbeHJ3PbdeVhbeIeyw27hj9XxSE+IozEpiYWGmBgGl3iOtsqHGhVDFsMOtPTj7vOFv/j/fcZi7Vpdz97P7eXRn7aBiMT39PhZMS6dch4KUOisaCNSY8nj8vHnCSYPTTXpSPE1dbvr6/eFhoL3Hu+DvtfzHtYvxG0Nmsp1jbb1kJSew8pwCDQBKjQANBGrMuFxentnfyJ0D6gTcdWU50zLiue3yOdz7p0O4vRaHmnvw+C2WFWfS2OXmotl5mhZCqREkxgxXd2Z8q6ioMJWVutVgIgqtBmrodBEfb+ONug66+v3himEOu43/uHYxxzr6mJ6VTFVzN4unZzI7P4UZOalj3XylJjQR2W2MOSnDs/YI1KixLMPWg028edyJZSBOoCQnmT8eqOXGC2aEdwf3eny4vH5SEuK4sDQHj99PUZZmB1UqWnSAVY0KyzIcOOGkurmHTTtquH9bNQ/uqKGxq59/+cBsNlfWcc2y6TjsNlIS4pk3JZ0ej49pmQ4uma3pIZSKJg0EKup8Potn956gubuf+7ZWDdodfN/WKlwei9WLComzwV1XlpNotzEzN4kryqcyI0fLRioVbRoIVFR5PH5213XwlSf30uvxD7s7uLffR5wNLpqVy/wpabx/dh5zp+q+AKVGi84RqBEXKhbT3O2msaufOJuQlZyAPU6G3R3c3udh0fRMRAzl03RPgFKjTQOBGlGWZdj2dhMnOly09nrCk8L/8v5SNv+9jg1ryrlzy/7wctHbLp9DXloiNgznFmVrEFBqDGggUCPCsgx17b00dbtxxMeRlmSnrdfDk7sDNQPWryzjwlk5/PfOWn5y3RIMkOqIJ8luIzc1keJs3Reg1FjRQKDOmsfj540TnTR0ujje6Q5PCA8sG3nf1ip+esMyPjDXT2aKHbtN6HB5ed+sXA0ASo0xDQTqPQnlBmrudlHf2U99Rx8Am3bUnFQz4KaLS3lgezWtPf0UZibh8viYlp/K0mItGKPUeKCBQL1rPp/FiwebONLay+KiDOo7+ijMSCI5MZ6s5IRwyUgYXDOgJDeFJHsc5+SnkZAQN4Z3oJQaSAOBiphlGY609lLV3E28TchJsXO8wx3uBYRqBD/yyjv1g0M1A77/kUVUaMlIpcYlDQQqIpZleH5fI19+Yk/4Q/+etQu4f/vJG8TWrShl49ZqHHYbd66er3WDlRrnNBCoiNS09ISDAAQ+9O/4/b7w+H+I22sxd0o6P7/xXDKT7eSl6YogpcY7DQTqlHw+i30nnLT09GMZM+yu4Lghy/4ddhtz8lOZXZA2ii1VSp0N3b2jhuXzWTy15zjrN79OVVMPB050hctEhjjsNuZOSR9UPvL7H1lEaZ6mi1ZqItEegRrWvhNO7t9exZc/NJev/O4NspITuPWyMjZuG7xH4Gd/ruami0uJs8GFs3I4T5eEKjXhaCBQYaGiMW29/bT3evhYRTHVzd24vRYNTjeP7qzlpotLSYy3saAwnbuf2U9tm4tDzT384COLOK9YU0QoNRFpIFBAYHfw34608VpdB2X5aWQkxbNxWxU3X1IaThTX4HTzwPbAaqBHb1rOvR9dgsdvUZDu0NKRSk1gGghimM9nsb/BSUdfP+29Pu7beojViwrp8/jw+CzcXosnd9efNCR0z9oFLJ2epd/+lZokNBDEIJ/P4mBjF281dnP/9iq+taac3NQE/t+H5mKPE366vZpL5+bjsNsGDQnF2eDiWbksma61ApSaTPRfc4zxePw8tec4f3yric27arlrdTnOPh+7azv4znNv8cXNe/j4+TPYe6yT9SvLwsHgoZdrmJ2XyrLiLE0PodQkoz2CGGJZhv2NTjKS7BRl5VCcncznf/PaSZlC735mPz+4djHffe4tfv7JCnx+i+LsFGbm6jyAUpORBoJJLpQf6ISzj7TEeJq7+jFAa4+Hbz6975SZQl0eHx19Hgozk3RfgFKTnA4NTWKWZXhhfyP//Ou/U9Pci8vrx+ny8sXNezjY1D3sTuFQptCUhHjuvW4JJTkpY9R6pdRoiXogEJFVIvK2iFSLyO3DPH+biBwQkb0islVEZkS7TbHA57N4ra6DbreX712zkH6/xSs17eEykcCwO4VtAhvWLmDh9HRWlU/RoSClYkBUA4GIxAEPAB8G5gMfF5H5Qy57HagwxiwCfgf8IJptmuwsy3C8s4eXDreyo6qFYx0u9hxzcu+fDmEZwkEgtCx0YHqIe9Yu4EPzCrhmSSEzclI1CCgVI6I9R7AcqDbG1ACIyGPAWuBA6AJjzPYB1+8EPhHlNk1aPp/FwSYnBxt7+c9tgT0BcTYoykoe1AsYuFN43YpS5hSkkZls54KSHF0WqlQMiva/+kLg2IDj+uC5U7kJeH64J0RknYhUikhlS0vLCDZxcrAsw3P7GnB5LVq63dxyaRnxNniisp6q5m4cdttJvYCOPg8lOSksLEznotJcDQJKxahxs2pIRD4BVADvH+55Y8wmYBNARUWFGcWmjWuh/EAdvR4anS76fdZJxeNf2NfA+pVl3Le1alAvYN6UdF0SqpSKeiA4DhQNOJ4ePDeIiHwQ+AbwfmNMf5TbNGl4PH7eONGJ2+OnsaufHo+fe18cfknoI6/U8uvPLMdgyE/T3EBKqXdEOxDsAspEZCaBAHA9cMPAC0RkKfAgsMoY0xzl9kwaHo+fPx5s4lh7Hy6vn007arj5ktJTLgnt6POQl5aoewKUUieJ6qCwMcYH3AL8AXgLeNwYs19ENojImuBlPwRSgSdEZI+IbIlmmyY6yzIcbu6h8lg7GLhva9Wg1UCnWhKqewKUUqcS9TkCY8xzwHNDzt054PEHo92GySK0Qez7L7zFuhWzaOpyDwoAw2UK/c7VC1lWnKl1g5VSpzRuJovVqQ2cED7Y2MXn3z+bE04XyQlxJwWA0GTwrLxUyqemU5qn+wGUUqengWCcCuUIqmvvRUS4f9shrl5WxKYdNeFv+1/64By+tmou333hYDgAzM5LpSDdwdLpmZolVCkVETFm4q3ErKioMJWVlWPdjKixLMNL1c10u/z09vtIccSTbI8LZwoNcdht3HLpbFxeK1AzuDSHaZkOirJ0GEgpdTIR2W2MqRh6XnsE40ygJ+CkpdvLN59+M/zt/3vXLBp2RZDbZ/HQyzWBmsEztGawUurd00Awjng8fnYebcVvSTgIQOAD/0hrTzg9RIjDbuOS2blcs7RQ9wUopd4z/fo4DliWoaa5m51H2+jzBFJEDP32/3hlPfesXTAoSdy91y3hvJJsnRBWSp0V7RGMMY/Hz47qFpq6+7nn2QO4vRbrV84+6dt/R5+HktxkNn/2Avq8fgrSdXewUmpkaCAYIx6Pn7cau/AbQ0piPE3Hndx8SSlP7q7n8cr6cG6g0BzBt69ayOJpuhJIKTXyNBCMAbfbx65j7bT2eGh0uk9KEvfozloeeaWWH167mKrmbpYWZbF8RoYGAaVUVOgcwSiyLENtWw+76zto6/GSk5LAY7vqTkoSd82y6XT0eUhPiufi2blcMCOLlKTEMW69Umqy0h7BKLAsQ117L1XNPbT3ergrWC5yYA+gwekGAsEgzgbfvXohF5bkaC9AKRV12iOIIssyHGnp4bW6dhq73KQmxoeDAAzuAYQ47DbeNyuXKxdN0yCglBoV2iOIEo/Hz8HmLvp9Fr39fnr6fcTbhKzkhPC3f3inBwAEJ4UXsHR6pm4MU0qNGg0EUeDx+PnrkRYEG609/Rxr7+Pxyno6+jzcdvkcfvnXo+FgEOoBzJuSzrQMB+XTMjQIKKVGlQaCEWRZhpqWHtr6XHT0+vj6U2+eNBdw758OsW5FKRu3VuOw29iwZgHZKXGcW1ygAUApNSY0EIwQl8vL68eddPR5SE6MDwcBGFwy8oHt1RRmJHHrytlUzMgiM8lOWUHm2DZeKRXT9CvoCOhzeXhmfyP//OtdHGjo5vW6jlOWjHTYbRx3ukhNjKezz8ucvLQxarVSSgVoj+AsuFxe3mzswuX1c+fv94U/7C3DsAnibAL/ftVCCrMcJMbZmFeQjsOhvwKl1NjST6H3qNvl5vl9Ldy5Zd9JReOfeeP4SSUjv33VApYWZTIzVxPEKaXGFw0E71Kfy8O+xm76fRZ3btk3qGaw22vx5O56brxgBpsr67jp4lLibLBoeiazcpKZma/DQEqp8UfnCN6FTpeb1487OdLai9dnkZWcABCuGeyw22hwutlcWcdXV80jyW5j3pR07DaYkZs6xq1XSqnhaY8gAj6fxeHWLt6o7w7PBTjsNtavLOORVwLpIUI1g+dNSScnJYHWnn4WTc+gKCuZGZouWik1jmmP4DQCOYK6eaWmjZoWF8kJcXzxg2VMzXDg9lrct7WKj1YE0kN09HkozEzCkSBY+CkrSOPi2XnM1KIxSqlxTnsEw7AsQ0NXLyc6+mlwujnc0hPeGbx+ZRmfW1HKz3bU0OB0U5afxn/90zKyUhJIsgtdLi8Vxbm6OUwpNWHop9UQlmU43OpkZ00nNz78d259bA8P7qjh0xeVkJWcwH1bq2jr83DNsuk47DYyk+wUpCfi81ukJdp53+x8DQJKqQlFP7GGONHVS2u3j28M2Rn84xcP8U/nF+P2WlgG4mywYc0CfMZPbloCF83KpTQ/TYeBlFITjg4NEZgMrm3vostlcbi1F79lht0ZnJeaGN4YdtGsXNIdcczOTdN00UqpCS3mA4HL5eVoZy/tvT5auvtx2OOIt8mwO4NTEuNZv7KMouxkFk5N1aphSqlJIWaHhjweP0dbO6lq7aHB2c+rR9qobunhh384SHJCHOtXBvYFAOGlogUZiVxYmsOH5hZoEFBKTRox1yNwu3283RIoGOPyWvS4fRxr7+OJ4KqgWy8rY+PWQ3zyopmsW1GKZcAmMDMnhcXTMnUYSCk16cRUj6DT5aaqrYumrn76vQZHvI3URBu/3VUXXhW0cVsV55fmYYwBYE5+KpfMzuGyc/I1CCilJqWoBwIRWSUib4tItYjcPszziSKyOfj8qyJSEo12dLrcHGro5Wiri8zkBMry7dS09FDb5uK2y+fwm7/Xcs2y6eHSkXmpDs4ryWZpcTrnztAi8kqpySuqQ0MiEgc8AFwO1AO7RGSLMebAgMtuAjqMMbNF5Hrg+8DHRrIdnS43fwxmCg2lh9iwppxLz8nkz293crill9WLCsP1ApYUZZKTGk9RRjLJSQkj2RSllBp3ot0jWA5UG2NqjDEe4DFg7ZBr1gK/Dj7+HbBSREZ0Mf6hxt5BmULdXos7t+ynrt3P1Kzk8L4Am8B3rl7IBTOyOGdKpgYBpVRMiPZkcSFwbMBxPXD+qa4xxvhExAnkAK0DLxKRdcA6gOLi4nfViKau/mH3BTR1u4mzCTaBpcVZ5KfaKctL12EgpVRMmTCrhowxm4BNABUVFebdvLYgPXHYfQEFaQ4QKJ8qLC1OJzPJMbKNVkqpCSDaQ0PHgaIBx9OD54a9RkTigQygbSQbMWdKChvWLBi0L2DDmnKKs+PIT43jklk5GgSUUjEr2j2CXUCZiMwk8IF/PXDDkGu2AJ8CXgGuBbaZ0NrNEZKZ5OBDC/IoyV1OU1c/BWmJFGfH4Uh0MCVTA4BSKrZFNRAEx/xvAf4AxAEPG2P2i8gGoNIYswV4CHhURKqBdgLBYsRlJjlYPlM/9JVSaqiozxEYY54Dnhty7s4Bj93AR6PdDqWUUsOLqZ3FSimlTqaBQCmlYpwGAqWUinEaCJRSKsbJCK/UHBUi0gLUvseX5zJk13IM0HuODXrPseFs7nmGMSZv6MkJGQjOhohUGmMqxrodo0nvOTboPceGaNyzDg0ppVSM00CglFIxLhYDwaaxbsAY0HuODXrPsWHE7znm5giUUkoNFos9AqWUUgNoIFBKqRg3aQOBiKwSkbdFpFpEbh/m+UQR2Rx8/lURKRmDZo6oCO75NhE5ICJ7RWSriMwYi3aOpDPd84DrPiIiRkQm/FLDSO5ZRK4L/q73i8hvRruNIymCv9fFIrJdRF4P/t2+YizaOZJE5GERaRaRfad4XkRkY/DPZK+ILDurNzTGTLr/CKS8PgyUAgnAG8D8Idd8HvhZ8PH1wOaxbvco3POlQHLw8b/Ewj0Hr0sDdgA7gYqxbvco/J7LgNeBrOBx/li3O8r3uwn4l+Dj+cDRsW73CNz3CmAZsO8Uz18BPA8IcAHw6tm832TtESwHqo0xNcYYD/AYsHbINWuBXwcf/w5YKSIyim0caWe8Z2PMdmNMX/BwJ4GKcRNZJL9ngHuA7wPu0WxclERyz58FHjDGdAAYY5pHuY0jKZL7NUB68HEGcGIU2xcVxpgdBOqznMpa4BETsBPIFJGp7/X9JmsgKASODTiuD54b9hpjjA9wAjmj0rroiOSeB7qJwDeKieyM9xzsMhcZY/5vNBsWRZH8nucAc0TkryKyU0RWjVrrRl4k9/st4BMiUk+g9skXRqdpY+rd/ns/rQlTvF6NHBH5BFABvH+s2xJNImID7gU+PcZNGW3xBIaHPkCg17dDRBYaYzrHslFR9HHgV8aYH4nIhQQqHi4wxlhj3bCJYrL2CI4DRQOOpwfPDXuNiMQT6FK2jUrroiOSe0ZEPgh8A1hjjOkfpbZFy5nuOQ1YAPxZRI4SGEvdMsEnjCP5PdcDW4wxXmPMEeAQgcAwEUVyvzcBjwMYY14BHAQSs01mEf17j9RkDQS7gDIRmSkiCQQmg7cMuWYL8Kng42uBbSY4CzNBnfGeRWQp8CCBIDCRx41DTnvPxhinMSbXGFNijCkhMC+yxhhTOTbNHRGR/N1+mkBvABHJJTBUVDOKbRxJkdxvHbASQETmEQgELaPaytG3BfhkcPXQBYDTGNPwXn/YpBwaMsb4ROQW4A8EVh08bIzZLyIbgEpjzBbgIQJdyGoCkzLXj12Lz16E9/xDIBV4IjgvXmeMWTNmjT5LEd7zpBLhPf8B+JCIHAD8wL8ZYyZkbzfC+/0y8HMR+RKBieNPT/AvdYjIbwkE89zg3MddgB3AGPMzAnMhVwDVQB/wmbN6vwn+56WUUuosTdahIaWUUhHSQKCUUjFOA4FSSsU4DQRKKRXjNBAopVSM00CglFIxTgOBUqchIj2j8B5rTpdCW6lo030ESp2GiPQYY1LHuh1KRZP2CFRMEZHvici/Djj+loh8M1io5zUReVNETkplLSIfEJFnBxzfLyKfDj4+V0T+IiK7ReQPp0sHLCK3DigO9Fjw3KdF5P7g41+JyH8Fs4bWBN/3YRF5S0R+NXJ/Ekq9QwOBijWbgesGHF9HoC7F1caYZQSK9/wo0toUImIH/hO41hhzLvAw8O+necntwFJjzCLgc6e4Jgu4EPgSgZwyPwbKgYUisiSSdin1bkzKXENKnYox5nURyReRaUAe0AE0Aj8WkRWARSCve0Hw/JmcQyDD6Z+CsSMOOF3yr73A/4jI0wSSww3nGWOMEZE3gSZjzJsAIrIfKAH2RNAupSKmgUDFoicIZJydQqCH8E8EgsK5xhhvMGW1Y8hrfAzuQYeeF2C/MebCCN/7HwmUIbwS+IaILBzmmlB6cGvA49Cx/ptVI06HhlQs2kwg2+y1BIJCBtAcDAKXAjOGeU0tMF9EEkUkk2DaY+BtIC9YEAURsYtI+XBvGiyUU2SM2Q58Nfi+OhGtxpx+u1AxJ5jGOA04boxpEJH/AZ4JDsVUAgeHec0xEXkc2AccIVAcHmOMR0SuBTaKSAaBf1M/AfYP89ZxwH8HrxNgozGmc2KXylaTgS4fVUqpGKdDQ0opFeN0aEipKBCRB4D3DTl9nzHml2PRHqVOR4eGlFIqxunQkFJKxTgNBEopFeM0ECilVIzTQKCUUjHu/wPOP/NCYCbGFQAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import seaborn as sns\n", + "\n", + "sns.scatterplot(data=merged_results, x=\"value_sim\", y=\"value_obs\")" + ], "metadata": { "collapsed": false, "pycharm": { @@ -855,9 +719,22 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 159, + "outputs": [ + { + "data": { + "text/plain": "LinregressResult(slope=0.999889299809943, intercept=5.271437621759412e-06, rvalue=0.9999938263765986, pvalue=0.0, stderr=3.49259911845371e-05, intercept_stderr=6.896656958760079e-06)" + }, + "execution_count": 159, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#from scipy.special import rel_entr\n", + "from scipy.stats import linregress\n", + "linregress(merged_results.value_sim.values, merged_results.value_obs.values)" + ], "metadata": { "collapsed": false, "pycharm": { @@ -881,9 +758,7 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [ - "# = bp.merge(result_sim, left_index=True, right_index=True, how=\"outer\").fillna(0)" - ], + "source": [], "metadata": { "collapsed": false, "pycharm": { From 75c16a28d4a9088ac810835efae880b3919ee879 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 16 May 2022 21:26:02 +1000 Subject: [PATCH 031/135] validation --- notebooks/validate_frozen_impl.ipynb | 352 +++++++++++++++++++++++---- 1 file changed, 304 insertions(+), 48 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 7444b9df2e..f62f539bcb 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 231, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -75,6 +75,8 @@ "import pandas as pd\n", "from numpy.random import default_rng\n", "from statsmodels.stats.gof import powerdiscrepancy\n", + "from statsmodels.stats.proportion import multinomial_proportions_confint\n", + "import numba\n", "\n", "from activitysim.cli import run\n", "from activitysim.core import inject\n", @@ -463,33 +465,6 @@ } } }, - { - "cell_type": "markdown", - "source": [ - "#### generic power discrepancy implementation in statsmodels\n", - "\n", - "lambd{float, str}\n", - "float : exponent a for power discrepancy\n", - "\n", - "‘loglikeratio’: a = 0\n", - "\n", - "‘freeman_tukey’: a = -0.5\n", - "\n", - "‘pearson’: a = 1 (standard chisquare test statistic)\n", - "\n", - "‘modified_loglikeratio’: a = -1\n", - "\n", - "‘cressie_read’: a = 2/3\n", - "\n", - "‘neyman’ : a = -2 (Neyman-modified chisquare, reference from a book?)\n", - "\n", - "note - they are all based on continuous approximations (x^2) so technically they might be inadequate for very small\n", - "occurences of counts" - ], - "metadata": { - "collapsed": false - } - }, { "cell_type": "code", "execution_count": 64, @@ -590,25 +565,10 @@ } } }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, { "cell_type": "markdown", "source": [ - "# TODO for paper:\n", - "\n", - "automate such that for a given sample size we calculate all probs for ALL 482 trips. then scatterplot and lin reg\n", - "\n" + "# All trips for paper\n" ], "metadata": { "collapsed": false, @@ -624,15 +584,14 @@ "source": [ "%%time\n", "\n", - "num_samples = 10000\n", + "num_samples = 10000 #about 8.5s per 10k per trip, so about 70mins for 10k sample. 1e5 is about 10 times as long, so\n", + "# about 11.5h\n", "\n", "trips_to_check = bp.index.values #[100:103]\n", "\n", "res = []\n", "\n", - "for i, trip_id_to_check in enumerate(trips_to_check):\n", - " if i == 50:\n", - " print(f\"Processing {i} out of {trips_to_check.shape[0]}\")\n", + "for trip_id_to_check in trips_to_check:\n", " _, c_, _ = mode_choice_for_trip(choose_individual_max_utility=True,\n", " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", @@ -742,6 +701,303 @@ } } }, + { + "cell_type": "markdown", + "source": [ + "#### generic power discrepancy implementation in statsmodels\n", + "\n", + "lambd{float, str}\n", + "float : exponent a for power discrepancy\n", + "\n", + "‘loglikeratio’: a = 0\n", + "\n", + "‘freeman_tukey’: a = -0.5\n", + "\n", + "‘pearson’: a = 1 (standard chisquare test statistic)\n", + "\n", + "‘modified_loglikeratio’: a = -1\n", + "\n", + "‘cressie_read’: a = 2/3\n", + "\n", + "‘neyman’ : a = -2 (Neyman-modified chisquare, reference from a book?)\n", + "\n", + "note - they are all based on continuous approximations (x^2) so technically they might be inadequate for very small\n", + "occurences of counts" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 64, + "outputs": [ + { + "data": { + "text/plain": "(0.3439655414093617, nan)" + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Note power tests will be inf when expected count is zero (any non-zero count will be impossible)\n", + "eps = 0 #1e-14\n", + "non_zero_shares = ms_comp.loc[(ms_comp.mode_share_obs > eps) & (ms_comp.trip_mode > eps)][[\"mode_share_obs\", \"trip_mode\"]]\n", + "ddof = non_zero_shares.shape[0] - 1 # what to use here? num alternatives - 1? What about zero expected count?\n", + "\n", + "non_zero_shares.trip_mode *= num_samples\n", + "\n", + "sum_ms = non_zero_shares.mode_share_obs.sum()\n", + "if sum_ms != 1.0:\n", + " if np.abs(1.0 - sum_ms) > 1e-1:\n", + " print(f\"NOT GOOD {sum_ms} should be one\")\n", + " #print(f\"sum ms is not one but {sum_ms}, re-normalising\")\n", + " non_zero_shares.mode_share_obs /= sum_ms\n", + "\n", + "powerdiscrepancy(non_zero_shares.trip_mode.values, non_zero_shares.mode_share_obs.values, lambd=0, ddof=ddof)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 99, + "outputs": [ + { + "data": { + "text/plain": "array([[4.17345080e-03, 5.98926219e-03, 5.00000000e-03],\n [1.45406892e-04, 6.18851051e-04, 3.00000000e-04],\n [4.13473933e-02, 4.66025328e-02, 4.39000000e-02],\n [5.64412252e-02, 6.25036058e-02, 5.94000000e-02],\n [8.87346329e-01, 8.95325012e-01, 8.91400000e-01]])" + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 106, + "outputs": [ + { + "data": { + "text/plain": "array([ True, True, True, True, True])" + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 230, + "outputs": [], + "source": [ + "# test_trip_res = merged_results.loc[merged_results.trip_id == 1045822217]\n", + "# ci = multinomial_proportions_confint(num_samples * test_trip_res.value_sim.values, alpha=0.999, method='goodman')\n", + "# np.hstack((test_trip_res[['value_obs']].to_numpy(), ci))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 275, + "outputs": [], + "source": [ + "@numba.jit\n", + "def within(lower, upper, x):\n", + " return (x >= lower) & (x <= upper)\n", + "\n", + "@numba.jit\n", + "def numpy_within(x):\n", + " lower = x[0]\n", + " upper = x[1]\n", + " val = x[2]\n", + " return within(lower, upper, val)\n", + "\n", + "def all_within_ci(trip_df, num_samples):\n", + " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=0.99999, method='goodman')\n", + " vals_with_ci = np.hstack((ci, trip_df[['value_obs']].to_numpy()))\n", + " return np.apply_along_axis(numpy_within, 1, vals_with_ci).all()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 276, + "outputs": [], + "source": [ + "vals_within_ci = merged_results.groupby(\"trip_id\").apply(lambda group: all_within_ci(group, num_samples))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 277, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Have 57 with contribution outside 0.99 CI\n" + ] + } + ], + "source": [ + "trips_not_within = vals_within_ci.loc[~vals_within_ci]\n", + "print(f\"Have {trips_not_within.shape[0]} with contribution outside 0.99 CI\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 291, + "outputs": [], + "source": [ + "bad_res = []\n", + "\n", + "for bad_trip_id in trips_not_within.index.values:\n", + " trip_df = merged_results.loc[merged_results.trip_id == bad_trip_id]\n", + " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=0.999, method='goodman')\n", + " vals_with_ci = np.hstack((ci, trip_df[['value_obs']].to_numpy()))\n", + " x_ = pd.DataFrame(vals_with_ci)\n", + " x_['in_range'] = np.apply_along_axis(numpy_within, 1, vals_with_ci)[:np.newaxis]\n", + " bad_res.append(x_.loc[~x_.in_range])\n", + "bad_res = pd.concat(bad_res)\n", + "bad_res.columns = [\"lb\", \"ub\", \"value\", \"within\"]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 292, + "outputs": [ + { + "data": { + "text/plain": " lb ub value within\n8 0.001687 0.003703 0.001667 False\n8 0.005178 0.008409 0.004877 False\n8 0.005531 0.008856 0.004911 False\n3 0.000054 0.000738 0.000051 False\n8 0.004739 0.007848 0.004560 False\n3 0.000154 0.001038 0.000112 False\n3 0.000154 0.001038 0.000125 False\n3 0.000403 0.001589 0.000400 False\n3 0.100737 0.112972 0.100614 False\n3 0.000540 0.001852 0.000525 False\n3 0.000470 0.001721 0.000423 False\n7 0.000273 0.001319 0.001341 False\n8 0.000017 0.000575 0.000011 False\n8 0.003184 0.005805 0.002994 False\n8 0.001930 0.004060 0.001838 False\n8 0.003099 0.005690 0.003060 False\n3 0.000154 0.001038 0.000117 False\n8 0.002676 0.005113 0.002415 False\n3 0.001133 0.002858 0.001023 False\n8 0.005266 0.008521 0.004850 False\n8 0.002509 0.004881 0.002161 False\n8 0.005531 0.008856 0.005396 False\n8 0.003269 0.005920 0.003160 False\n3 0.000054 0.000738 0.000030 False\n3 0.000054 0.000738 0.000025 False\n3 0.076921 0.087807 0.076860 False\n8 0.004477 0.007510 0.004245 False\n2 0.001133 0.002858 0.001085 False\n8 0.004477 0.007510 0.004245 False\n8 0.003526 0.006263 0.003404 False\n8 0.003440 0.006149 0.003351 False\n8 0.004739 0.007848 0.004276 False\n8 0.003784 0.006605 0.003404 False\n8 0.004303 0.007285 0.003717 False\n8 0.000017 0.000575 0.000008 False\n1 0.000101 0.000891 0.000084 False\n8 0.001930 0.004060 0.001920 False\n3 0.032773 0.040191 0.032192 False\n3 0.000337 0.001455 0.000256 False\n3 0.000756 0.002236 0.000730 False\n8 0.000017 0.000575 0.000017 False\n1 0.000054 0.000738 0.000033 False\n1 0.000017 0.000575 0.000006 False\n1 0.000017 0.000575 0.000008 False\n8 0.000017 0.000575 0.000014 False\n3 0.000054 0.000738 0.000041 False\n3 0.000101 0.000891 0.000058 False\n8 0.000017 0.000575 0.000016 False\n8 0.018715 0.024460 0.018662 False\n0 0.998545 0.999663 0.999677 False\n1 0.000054 0.000738 0.000053 False\n3 0.103271 0.115636 0.102986 False\n3 0.103271 0.115636 0.102967 False\n8 0.000017 0.000575 0.000012 False\n8 0.000017 0.000575 0.000013 False\n8 0.002676 0.005113 0.002658 False\n3 0.001289 0.003102 0.001196 False\n7 0.009844 0.014139 0.009778 False\n3 0.000470 0.001721 0.000384 False", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
lbubvaluewithin
80.0016870.0037030.001667False
80.0051780.0084090.004877False
80.0055310.0088560.004911False
30.0000540.0007380.000051False
80.0047390.0078480.004560False
30.0001540.0010380.000112False
30.0001540.0010380.000125False
30.0004030.0015890.000400False
30.1007370.1129720.100614False
30.0005400.0018520.000525False
30.0004700.0017210.000423False
70.0002730.0013190.001341False
80.0000170.0005750.000011False
80.0031840.0058050.002994False
80.0019300.0040600.001838False
80.0030990.0056900.003060False
30.0001540.0010380.000117False
80.0026760.0051130.002415False
30.0011330.0028580.001023False
80.0052660.0085210.004850False
80.0025090.0048810.002161False
80.0055310.0088560.005396False
80.0032690.0059200.003160False
30.0000540.0007380.000030False
30.0000540.0007380.000025False
30.0769210.0878070.076860False
80.0044770.0075100.004245False
20.0011330.0028580.001085False
80.0044770.0075100.004245False
80.0035260.0062630.003404False
80.0034400.0061490.003351False
80.0047390.0078480.004276False
80.0037840.0066050.003404False
80.0043030.0072850.003717False
80.0000170.0005750.000008False
10.0001010.0008910.000084False
80.0019300.0040600.001920False
30.0327730.0401910.032192False
30.0003370.0014550.000256False
30.0007560.0022360.000730False
80.0000170.0005750.000017False
10.0000540.0007380.000033False
10.0000170.0005750.000006False
10.0000170.0005750.000008False
80.0000170.0005750.000014False
30.0000540.0007380.000041False
30.0001010.0008910.000058False
80.0000170.0005750.000016False
80.0187150.0244600.018662False
00.9985450.9996630.999677False
10.0000540.0007380.000053False
30.1032710.1156360.102986False
30.1032710.1156360.102967False
80.0000170.0005750.000012False
80.0000170.0005750.000013False
80.0026760.0051130.002658False
30.0012890.0031020.001196False
70.0098440.0141390.009778False
30.0004700.0017210.000384False
\n
" + }, + "execution_count": 292, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bad_res" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 8668383a489973a91af2c1c5e5043891f05b1409 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 17 May 2022 00:26:03 +1000 Subject: [PATCH 032/135] nicer comparison --- notebooks/validate_frozen_impl.ipynb | 228 +++++++++------------------ 1 file changed, 72 insertions(+), 156 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index f62f539bcb..6567e6ba41 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -643,20 +643,35 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 302, + "outputs": [], + "source": [ + "# should use only those without zero probability - they are really not available so not a valid choice\n", + "merged_results_wo_zeros = merged_results.loc[(merged_results.value_sim != 0) & (merged_results.value_obs != 0)]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 303, "outputs": [ { "data": { "text/plain": "" }, - "execution_count": 157, + "execution_count": 303, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA14ElEQVR4nO3deXzU9Z348dd7kkkm950AISEEgkC4jXhUaZXapS6CWmutW3uslke3a6W1v23toVbs9tzaymq30mpb3W1F62rRVdsKtGgrlqCIgEhCICGQ+5hcM5nj+/n9MYdJCDBKJte8n4+HD+f7ne9kPl8C857P9X6LMQallFKxyzbWDVBKKTW2NBAopVSM00CglFIxTgOBUkrFOA0ESikV4+LHugHvRW5urikpKRnrZiil1ISye/fuVmNM3tDzEzIQlJSUUFlZOdbNUEqpCUVEaoc7r0NDSikV4zQQKKVUjNNAoJRSMU4DgVJKxTgNBEopFeOiumpIRB4GVgPNxpgFwzwvwH3AFUAf8GljzGvRbJNSSk00nS43hxp7aerqpyA9kYykOAoyHGQmOUbk50d7+eivgPuBR07x/IeBsuB/5wP/Ffy/UkopAkFgx9ttVLf0YBmobu5mVl4qnX0ezpnKiASDqAYCY8wOESk5zSVrgUdMIBf2ThHJFJGpxpiGaLZLKaUmiiPNfRzvdLFpRw1ur4XDbmP9yjKKspI41NjL8plnHwjGeo6gEDg24Lg+eO4kIrJORCpFpLKlpWVUGqeUUmPFsgw1LT109/u4b2sVbq8FgNtrcd/WKrr7fTR19Y/Ie02YncXGmE3AJoCKigqtpqOUmnQsy1DX3ktzdz8NTjc1LT3Mm5oeDgIhbq9Fn8dPQXriiLzvWAeC40DRgOPpwXNKKRVT3G4f+xu7qGnt5Y7f7wsPA/3m5vNx2G2DgoHDbiMvNZFZBckj8t5jPTS0BfikBFwAOHV+QCkVS3w+iwMnOvnb0TYanO5wEIDAN/9ut4tvX7UAhz3wce2w2/j2VQuYVZA8MVYNichvgQ8AuSJSD9wF2AGMMT8DniOwdLSawPLRz0SzPUopNZ74fBbP72+gy+3jnmcPcPMlpScNA33ql3t4fv0FPPKZ5TR1B5aPzpmSMmJBAKK/aujjZ3jeAP8azTYopdR45Hb72NvgBISirCQunJkNMOww0PP7Wpk7JZ1/XDgVm01GvC1jPUeglFIxw7IMR9t6aeh00e+zuPvZ/dS2uXDYbdy9ppxXD7dy62VlbNxWFZ4juGftAs4tzqQkNzUqQQA0ECilVNSFVgO9VtfJ1596M/whf+tlZTy6s5YGp5u7tuznwRvP5fYn32TdilLmTkljWoaD+VMySEiIi2r7xnqyWCmlJjXLMrywv5H/ff14OAhAYCJ447Yqrlk2PXzc2eelo8/DzJwULivLZ0lxdtSDAGggUEqpqPH5LF6r6+C2x/dgGYbdDyDB0R6H3caUdAfP3nIxa5cU4nCM3oCNDg0ppdQIsyzDkZYejjvdOF1ebr6klNTEuGEngo0J/H/DmgUsnpYxqgEgRAILdyaWiooKozWLlVLjjc9nsf+Ek8bufrpc3kEbw770wTk44m1894WDgyaCM5LtZCXZWTg1+kFARHYbYyqGntcegVJKjYDQnoCq5h7K8tNO2hj24xcPcculs1m3opTirGRaevpx2G1cNief+PixHaXXQKCUUmfJsgz7Tjip7whkCR1uY5jba1GUnUxWsh2fZVGWn0r5tIwxDwKggUAppd4zn89i3wknxzr6yEtLHJQldLj5gOzkBDpdHv5h3pRRWQ0UqbEPRUopNQH5fBZP7TnO9T/fyRd+u4e/HW4Lf/A/ubueWy8rG5Qf6J61CyjIsPPh8qnjKgiA9giUUupdCe0Obu3pHzQPYJl3egENTjeP7qxl3YpS5k9NJzPZzpJpmWOyIigS2iNQSqkIWZbhpepmDpzooqmrf9DQz9BeQGhj2NwpaSwvyRm3QQC0R6CUUhHx+SzePNFJr9sPAplJ9kHzAA1ON5sr69h047k0d/czMzeFpUVZUcsPNJI0ECil1Bm4XF6ee6uJbwzIE7RhzXzuXlPOXVv2h899/gOzqWvrJSM5gcWFmRMiCIBuKFNKqVMKbRDrcHn53H/vPmkV0DevmEtRTgqdfd7wstCclETKp46PZaFD6YYypZSKQGgy2Gf56Oj109TlJiHeNuy+gMZuD809Hs6fmc3UjCRKclImTC9gIA0ESikVFMoU+ueDDVSU5HHnlsCqoPUrZw+7L8AmsLQok/NLcsZlDyBSE7flSik1QizLUNvWQ+XRdrrdXj5y7oxwEAB4vLKe9SsH7wtYv7KMuQVpvK80d0IHAdAegVIqxlmWYU99Gy6Pob7DRXJCPJ0uz6Bv/w1ON4+8UsuDN55Ll8tHuiOe6ZlJzMyLXtWw0aSBQCkVk0JVwxqcbpq7+6lp6eHxyno6+jx85+qFzMhJorbNFb6+o8+DIz6OhFRhaWHWuN4X8G5NnjtRSqkIeTx+Xj/eQV2ba1Cq6FDpyK8/9SY/vWEZn//NawOWiy5genYiU9Mn5oTw6WggUErFFJ/P4s/VLSTFx52UKnrjtipuuriUB7ZX09vv45F/Xk5TVz9T0hNHpV7AWJmcd6WUUsOwLMPe4524PH46ez2nLB3psNvITk1gWVHWhJ8IjoQGAqVUTAgtDb3t8T1nXBK6Ye0ClhVmxkQQAF0+qpSKEUfbesNBAIZfEvrvVy/ksnPyuWrRNJKS7GPZ3FGlPQKl1KQT2h3c1OWmIN1BSU4KTV3uYZeE/vDaxVQ1d3NhaQ7nzciOmV7AQBoIlFKThmUZjrT28lZDF/UdfRjA5fVz3oxspmclnTQU1NHnoaq5m5k5KTEbBECHhpRSk4TH42dHVQu/f+M4B5u6+e2uOgCeqKzns49W8uZxJ/ffsHTQUNAdq+dzYWkOVy6aFrNBADT7qFJqEvB4/Dy7r4GvD0gTfetlZWyurGP1okIe2F6Nw27j/75wCQB17b0kJ8RTkJ5Icfbk2xdwKpp9VCk1Kfl8Fn870hYOAjB4T4AEP+PdXouWHjcXlOYyKz91DFs8/mggUEpNSKEUEfWdrpMmgiHwwR9nA3/wtMNuIz/NMQYtHf+iPigmIqtE5G0RqRaR24d5vlhEtovI6yKyV0SuiHablFITm89n8eqRNuo7XLR095OSGB8e+w9x2G3Mm5LO/75Wj8Nu497rllCSkzJGLR7fotojEJE44AHgcqAe2CUiW4wxBwZc9k3gcWPMf4nIfOA5oCSa7VJKTUyWZTjW0cuxdhe17X3c8+wB3F6LGTlJ3LW6nLuffads5HeuXsi8qWn8+GOLyU9zTNiiMaMh2kNDy4FqY0wNgIg8BqwFBgYCA6QHH2cAJ6LcJqXUBOLzWexvcNLR109vv0Wny0tTl5tNO2rCw0G1bS5+tqOaH1y7mENN3Zw3I4sLZ+aQkBBHSa7OB5xJtIeGCoFjA47rg+cG+hbwCRGpJ9Ab+MJwP0hE1olIpYhUtrS0RKOtSqlxxuPx89Se43xs0052HXXydlM39zx7AMtw0pxAbZuLQ03d/OKlGgozk0hIiBujVk8842Hh7MeBXxljpgNXAI+KyEntMsZsMsZUGGMq8vLyRr2RSqnRZVmG3cc6whlCRRgUAIabE7AJ3HvdEkrztBfwbkR7aOg4UDTgeHrw3EA3AasAjDGviIgDyAWao9w2pdQ4VtfeS5fbF14Cek5BGg2dfTjsNp7cXc+tl5WxcVvVoDmBZcWZMbUvYKREOxDsAspEZCaBAHA9cMOQa+qAlcCvRGQe4AB07EepGObx+Gnq7qfL5eWhl2vCH/bfurKcr394Lt95/iCP7qxl3YpSyvJTmTclndJJUjZyLEQ1EBhjfCJyC/AHIA542BizX0Q2AJXGmC3Al4Gfi8iXCEwcf9pMxO3OSqn3bGCSuMIsB9XNvbR093Oi00VWcgINzsA+gW89s59ffKqCez+6GATK8tOYpQHgrEV9Q5kx5jkCk8ADz9054PEB4H3RbodSanyyLMO2t5uoae5hRm4qx4908M2nTy4fGQoGXS4vhVlJlE/NiOn8QCNJ/xSVUmOqrr2XEx0u/Ab2n3CGgwC8kyrimmXTgcCE8Nwp6SyOkcpho0VTTCilRs3QOgHFWck0d/WTaI+jrddDvM122vKRujs4OjQQKKVGxdBSkQ67jQ1rF/DA9ipq21w47DZ+fN2SYctHVszIYu3ii3U+IEq0b6WUGhVDS0W6vRZ3/n4fqxcVho+/98Jb3LF6/qCaAd+9eiEXzcyhrCBNg0CUaI9AKRVVoeGgQ03d3HxJKU/urqfB6QbeGfYJqW1z0e32sm5FKXMK0pg3JZ2ZubovINo0ECilRtTAeYBpmQ72HHPy1Sf3DrsKyGG3MXCxuMNuY2FhBlPSkzQAjCIdGlJKjZjQPMBnfvV3dtd2UNvWFw4CMHgVkMNu467V5Ty7N5BsIDQZfGGwcIwGgdGjPQKl1Ig52tbLwy8f5taVZbR199PW6xl2FdCM7CR+cO1iHvlbDbevmoeFYW6BDgONFQ0ESqmzMnAoCIHbLp/DkTYXPR4/va29w64Cqm13UT4tjX/7h3kxVzd4PNKhIaXUexYaCrpi40t8/OevcvuTe3H7TDhV9OOV9SetArr1sjKe3XucuVPSOb80h5JcHQYaa9ojUEq9Z0OXhK5eVMieY53h444+D5v/Xse91y3hYGMXfgs2V9bx1VXzdGPYOCITMb9bRUWFqaysHOtmKBXTLMvwel0HJ5xu+vp9tPb2E2cTevr9/OKlGrKSE7jxghls3FZFVnICH62YrplCx5iI7DbGVAw9rz0CpVREPB4/e084aexyMzXdgR8/u486uW/rOzUB7lo9nxcPNIZrBQxMFT1/aoZOBo9TGgiUUmfkdvvY8mYDd255JyvohjXlvF7XNmhp6N3PHuCBG5ax4dn93HRxKXE2qJiRzUWlOZokbhzTQKCUOi3LMrxxwhkOAhBMD7FlPw/eeC5/PNAavtbttbAsw8brl9Ln8VOQ7qAkR3sB450GAqXUsELLQlu6+8O1AAZyey06+7yDzjnsNmblp2rN4AlG+2pKqUEsy3C0tYen9xznio0v8VJ1K9kp9mGLxeenJQ5aGqppoicm7REopcJC+wIONnaxaUdNuBfw3zuPcPeacu7asn/AHMECpmc7+PVnltPn8VGcnaKTwROUBgKlVFhoX8DNl5SGg8CTu+u58YIZbH2rgQdvPJfOPi/5aYlkJMVTlJVKUZYOA010OjSklMKyDNVN3VQ1dfPTG5bxvlnZVMzIAKDB6ebRnbXMnZpJY6ebQ0093P6/e3HY9XvkZKG/SaVinGUZnt/XwJefeCM87LN+ZRk3XlgCHKWy1klHn4ckexw/2VpFR59H5wImGQ0ESsUoyzLUtPTQ2OUOBwEIrAa6b2sV61aUcuvKOTR39ZOZbCclMZ4lRZm6JHQS0qEhpWKQ2+1jyxsnWH3/y+w80j7s0lDLgNPl4z+3V5GXlsj5M3O4cFaupoeYhDQQKBVj+lwe/l7Xzu3/+07BmOGWhtoEMpLi+eqqeSwszNQP/0lMA4FSMcKyDFWN3fy5qhWnyzdoVdD/+9A5g/YDrF9Zxuy8VPLSElhVPkWDwCSncwRKTXKhDWKvHevkm08H0kTcf8PScMGYBqebh14+wlf+4RymZSbhsMeRlGBDxHBOQYYGgRiggUCpScqyDLVtveyu66SuvXfQBrH6jj7WrywLZw7t6PPQ77NISYwjxR5PTloCRVk6IRwrNBAoNcmEAkBdRx9dLh+t3W7ibYPLRf76b7V8bkUp61aUYhmwCczISeb8GTkkJMSNYevVWNBAoNQk4vNZ7DzaRqPTzbH2Ph6vrKejz8NPPrZkUO3gBqebh/92hHvWLqCn38fsvBRm56drDyBGaSBQaoILZQlt6+2nvt3F1556M7wx7NbLynh0Zy3fff4t7lg9n3uePRB+7pZLy0hJiGd5cTYOh34UxLKo//ZFZBVwHxAH/MIY871hrrkO+BZggDeMMTdEu11KTQahJHG3Pb6Hmy4u5aGXawZtDNu4rYqbLi7lge3VdLu9rFtRSnF2MnlpiczITqY4W+cBVISBQERSAJcxxhKROcBc4HljjPcMr4sDHgAuB+qBXSKyxRhzYMA1ZcDXgPcZYzpEJP893otSMSPUCzja2svbjV1kJScgwrAbw0QCS0LnTkknPk6ItwnLpmfpXIAKi7RHsAO4RESygD8Cu4CPAf90htctB6qNMTUAIvIYsBY4MOCazwIPGGM6AIwxzZE3X6nYM7AXMHAIyGZj0DwAvLMx7K4ry+n3WZTmpTI9U3sBarBIN5SJMaYPuAb4qTHmo0B5BK8rBI4NOK4PnhtoDjBHRP4qIjuDQ0knN0BknYhUikhlS0tLhM1WavII7QfYdbQ9HATgnSEgn99w62VlgzaGffuqBVwwM5vZ+SlcWpZHcbamh1Ani7RHICJyIYEewE3BcyPVr4wHyoAPANOBHSKy0BjTOfAiY8wmYBNARUWFGaH3VmpC8PksXjzYxJHWXlxe/7BDQG6fxTNvHOfe65ZQ09JDSU4KUzMcZKUkaJI4dVqR9gi+SGAc/yljzH4RKQW2R/C640DRgOPpwXMD1QNbjDFeY8wR4BCBwKCUIhAEdtW243R5cXv9JCfEDZsbaG5BGqsXFfL9F94iziY4EmwsLc7SJHHqjMSYyL9ci0g6YIwx3RFeH0/gg30lgQCwC7jBGLN/wDWrgI8bYz4lIrnA68ASY0zbqX5uRUWFqaysjLjdSk1ElmWoa+/ltbpOvj5gSeiXPjgHR7yN775wMHzunrULKM5JosvlI91hpyA9UVcEqZOIyG5jTMXQ85GuGqoAfgmkBQ6lE/hnY8zu073OGOMTkVuAPxAYSno42KPYAFQaY7YEn/uQiBwA/MC/nS4IKDXZhQLAkdY+evt94SAAgSGgH794iFsunc1NF5cSZ4M5+WkkxNvIS3Vw/kwtG6nevUjnCB4GPm+MeQlARC4mEBgWnemFxpjngOeGnLtzwGMD3Bb8T6mYFUoNcbzTRVuvh5qWHuKGpIaAd+YDHnq5htsun4PXskizx2nFMPWeRRoI/KEgAGCMeVlEfFFqk1Ixx7IM295uoqqpJ5wIzmG38ePrlgy7JHR5SRYXlVbgiI/TBHHqrJ02EIjIsuDDv4jIg8BvCez+/Rjw5+g2TanYcaS1l731zkEZQt1ei++9cHJqiDtWz8dguKA0Tz/81Yg4U4/gR0OO7xrwWJdwKjUCfD6LI609WObkncG1ba5waoi5U9KwIUzPTmL+VK0ToEbOaQOBMebS0WqIUrHE4/HzZoOTlu5+khPiyEi2k54YN+wwUJ/HT3F2Mk+/foyrlhZrEFAjLtJVQxkEegMrgqf+Amwwxjij1TClJqs+l4dX6zp5va4Dy8Azbxzn+vOKmZrh4Osfnst3nh+8LHRWXgpev5/bP1yuG8NUVLybVUP7gOuCxzcSWDV0TTQapdRk5XJ5ee5AM998enCq6Md21bF2SSHzpqTzw2sX09fvo73PQ06qHZ+xOK8kVwOAippIA8EsY8xHBhzfLSJ7otAepSYlyzIc6+ilwekOBwEYnCraMvBWYxcbt1aH8wQJsGx6tgYBFVWRBgKXiFxsjHkZQETeB7ii1yylJgfLMhxp7eWthi4yk+209XqG3RcQF8wYMTs/jVtXzmZpcRYzs5MoztH0ECr6Ig0EnwMeCc4VAHQAn4pOk5Sa+CzLUNPSw4HGLg4394RLRv7y0+cNOyG8pCiT3n4/aY441iyapvmB1KiKKBAYY94AFgdzDWGM6Rr4vIh8yhjz6yi0T6kJJZQeYnddJ98YpmTkL/96mA1ryrlzy/7wc9++aiG9/R6SEuysKMvXAKBG3btKOnfKHyLymjFm2ZmvHBmadE6NR6GCMQcbuwZtDIPAt/5Qycj/uHYBeelJdPZ5KUhPJCMpnsT4eF0RpKLuVEnnIk1DfcafP0I/R6kJJzQMFCoYM9zGsIElI/PTk2h0uokTwenyUpaXrkNBakyNVCDQXcYqJoV6AVdsfImXqlvDAWC4egE2gW9dWY7ba5GSGE9RVhIrzykgPn6k/hkq9d5oj0Cps3C0rXdQ2UiH3caTu+tPKhl5x+r5LJ+Zze921+Hy+lk1fwqLi7M0CKhxIdJVQ2fy1xH6OUpNGB6PnxOdLm6+pBSAHW83c+tlZWzcVsWjO2tZt6KUmbkpJMXHkZIYR3aynR9cu0TnAtS4E2mKiQLgO8A0Y8yHRWQ+cKEx5iEAY8wtUWyjUuOGz2ex/4STE043Dnsc33pmH7VtrvDKoBf2NbBuRSlLizJJTYyn0+UlKzmBRdMySEgYqTLfSo2sSHsEvyKQUuIbweNDwGbgoSi0SalxJxQAjrb3UdPyzr6A0LLQBqebjduqgllC03n/HF0GqiaOSAcoc40xjwMWBEpQEigrqdSk53b7ePVoGweburGJ8NfqFm68YAZZyQls3FbFNcumB67zWsybms6q8ikaBNSEEmmPoFdEcgiuDhKRCwDNPKomPY/HzzP7Grjj9/vCG8DuWl3Ok6/Vcc2y6TywvRoJfuY77DbyUhM1CKgJJ9JAcBuwBZglIn8F8oBro9YqpcZQaHdwY5ebzj4v9R19ZCUn0OB04/Za3P3sfn5w7WIONXXjsNswJhAENqxZwKJpGWd+A6XGmUhTTLwmIu8HziGwVPRtY4w3qi1TapSFisc3drmo73AP6gUMnAtwey3cHh82gXvWLiAn1c6KsuUs1glhNUFFumrok0NOLRMRjDGPRKFNSo06n8/ixYNNHGntxeX1n1Q7OJQq+oHtgRTRhVlJFKQn4vZZzM5PZXqmLglVE1ekQ0PnDXjsAFYCrwEaCNSE5/NZ7Kpt50BDIEfQzZeUnjZFxIa1C0h3xCMizC1I101hasKLdGjoCwOPRSQTeCwaDVJqNHk8fv50sAljoDAjiZsvKSX1FLWDl5dkMbdgMWlJcZRPy9QegJo03uvO4l5g5kg2RKnRFEoU19Ttpq69j/u2VoXnA770wTl8bdVcvvvCO7WDN6wpJ80Rz7SMJE0QpyadSOcInuGdxHI2YD7weLQapVS0WJbhcEsPVc09WMaQkWQPBwEIDAH9+MVD3HLpbNatKGVWXiq5qQl4/Bbz8tNxOEYqK4tS40ekf6v/Y8BjH1BrjKmPQnuUigrLMtR39lJ5tJOvDygY8+2rFgw7H1CSm0K6Ix6fZeGwx3FecbauCFKTVqRzBH+JdkOUihafz+KVIy3YxMbRtl5uvqSUJ3fX0+B0c6y9b9j5gCnpDnJTEyjO1tVAavI7bSAQkW6GrzUggDHGpEelVUqNEI/Hz57jnTQ6PcPuC3i8sp47Vs/nnmcPhJ/70UeXsKw4SwOAihmnDQTGmLTRaohSI8myDIebezjY1E12ij0cBGDwvoCHXq6h1+3loU+dh8vjoyQnRSeDVcx5VzNfIpJPYB8BAMaYuhFvkVJnybIMz+9r5MtPBArG3Lpy9rDzAHE2uGP1fNId8aQ74njf7NwxarFSYyuinTAiskZEqoAjwF+Ao8DzEb52lYi8LSLVInL7aa77iIgYETmpsLJSkbIsw5v1nbzd1MXNl5QyNcOBZYYvHXn+zGyKshz8+m9HSEm0j1GLlRp7kfYI7gEuAF40xiwVkUuBT5zpRSISBzwAXA7UA7tEZIsx5sCQ69KA9cCr76bxSg3k81k8t6+Brzy5d9BcwAv7GsKVwwauFvrFjhpeOdLOvdcFqoYpFasiDQReY0ybiNhExGaM2S4iP4ngdcuBamNMDYCIPAasBQ4Mue4e4PvAv0XYHqWAdxLFnXC66Hb7ONzSMyhTaGguYHNlHT+4djHVzd1cPDuX5IQ4CjNncceV5Vo6UsW8SANBp4ikAi8B/yMizQR2F59JIXBswHE9cP7AC0RkGVBkjPk/ETllIBCRdcA6gOLi4gibrSYzj8dPZV07R9r6Bq36GZopNM4GH6so5kd/PMhX/mEuFTOy9YNfqQEizZa1HcggMHzzAnAYuPJs31xEbMC9wJfPdK0xZpMxpsIYU5GXl3e2b60mMJ/P4o1jHTx/oJH2Pi+bdhw+aUVQqGqYw27jwtIc5k9L46FPnseHF0zVIKDUEJH2COKBPwLtBGoVbzbGtEXwuuNA0YDj6cFzIWnAAuDPEijzNAXYIiJrjDGVEbZNxQiPx8++BifHOlynrBsMgzOFrl9ZBsCKMq0hrNSpRNQjMMbcbYwpB/4VmAr8RURejOClu4AyEZkpIgnA9QQqnYV+rtMYk2uMKTHGlAA7AQ0CahCfz+LN+k5+v/cEN/ziVdY/tocHd9QMWzcYAgHgnII01q0oZWZuCufpUJBSp/VuE6k3A41AG5B/pouDRe5vAf4AvAU8bozZLyIbRGTNu22sij0+n8XTbxznT281Dbsp7Jpl08PzABAIAv9+9UKykuNZu7iQD82fovUClDqDSLOPfh64jkCt4ieAzw5dAnoqxpjngOeGnLvzFNd+IJKfqSY/yzIcbeultbufbz6974zFYipmZLHx+iWU5CQzf2qGfvgr9S5EOkdQBHzRGLMnim1RCgC328dzBxr5+lNvcveV5eEAMFxyOJvA+pVlpCXGc1FprgYApd6DSOcIvqZBQI0Gj8fP3462hVNFJyfG47DbeHJ3PbdeVhbeIeyw27hj9XxSE+IozEpiYWGmBgGl3iOtsqHGhVDFsMOtPTj7vOFv/j/fcZi7Vpdz97P7eXRn7aBiMT39PhZMS6dch4KUOisaCNSY8nj8vHnCSYPTTXpSPE1dbvr6/eFhoL3Hu+DvtfzHtYvxG0Nmsp1jbb1kJSew8pwCDQBKjQANBGrMuFxentnfyJ0D6gTcdWU50zLiue3yOdz7p0O4vRaHmnvw+C2WFWfS2OXmotl5mhZCqREkxgxXd2Z8q6ioMJWVutVgIgqtBmrodBEfb+ONug66+v3himEOu43/uHYxxzr6mJ6VTFVzN4unZzI7P4UZOalj3XylJjQR2W2MOSnDs/YI1KixLMPWg028edyJZSBOoCQnmT8eqOXGC2aEdwf3eny4vH5SEuK4sDQHj99PUZZmB1UqWnSAVY0KyzIcOOGkurmHTTtquH9bNQ/uqKGxq59/+cBsNlfWcc2y6TjsNlIS4pk3JZ0ej49pmQ4uma3pIZSKJg0EKup8Potn956gubuf+7ZWDdodfN/WKlwei9WLComzwV1XlpNotzEzN4kryqcyI0fLRioVbRoIVFR5PH5213XwlSf30uvxD7s7uLffR5wNLpqVy/wpabx/dh5zp+q+AKVGi84RqBEXKhbT3O2msaufOJuQlZyAPU6G3R3c3udh0fRMRAzl03RPgFKjTQOBGlGWZdj2dhMnOly09nrCk8L/8v5SNv+9jg1ryrlzy/7wctHbLp9DXloiNgznFmVrEFBqDGggUCPCsgx17b00dbtxxMeRlmSnrdfDk7sDNQPWryzjwlk5/PfOWn5y3RIMkOqIJ8luIzc1keJs3Reg1FjRQKDOmsfj540TnTR0ujje6Q5PCA8sG3nf1ip+esMyPjDXT2aKHbtN6HB5ed+sXA0ASo0xDQTqPQnlBmrudlHf2U99Rx8Am3bUnFQz4KaLS3lgezWtPf0UZibh8viYlp/K0mItGKPUeKCBQL1rPp/FiwebONLay+KiDOo7+ijMSCI5MZ6s5IRwyUgYXDOgJDeFJHsc5+SnkZAQN4Z3oJQaSAOBiphlGY609lLV3E28TchJsXO8wx3uBYRqBD/yyjv1g0M1A77/kUVUaMlIpcYlDQQqIpZleH5fI19+Yk/4Q/+etQu4f/vJG8TWrShl49ZqHHYbd66er3WDlRrnNBCoiNS09ISDAAQ+9O/4/b7w+H+I22sxd0o6P7/xXDKT7eSl6YogpcY7DQTqlHw+i30nnLT09GMZM+yu4Lghy/4ddhtz8lOZXZA2ii1VSp0N3b2jhuXzWTy15zjrN79OVVMPB050hctEhjjsNuZOSR9UPvL7H1lEaZ6mi1ZqItEegRrWvhNO7t9exZc/NJev/O4NspITuPWyMjZuG7xH4Gd/ruami0uJs8GFs3I4T5eEKjXhaCBQYaGiMW29/bT3evhYRTHVzd24vRYNTjeP7qzlpotLSYy3saAwnbuf2U9tm4tDzT384COLOK9YU0QoNRFpIFBAYHfw34608VpdB2X5aWQkxbNxWxU3X1IaThTX4HTzwPbAaqBHb1rOvR9dgsdvUZDu0NKRSk1gGghimM9nsb/BSUdfP+29Pu7beojViwrp8/jw+CzcXosnd9efNCR0z9oFLJ2epd/+lZokNBDEIJ/P4mBjF281dnP/9iq+taac3NQE/t+H5mKPE366vZpL5+bjsNsGDQnF2eDiWbksma61ApSaTPRfc4zxePw8tec4f3yric27arlrdTnOPh+7azv4znNv8cXNe/j4+TPYe6yT9SvLwsHgoZdrmJ2XyrLiLE0PodQkoz2CGGJZhv2NTjKS7BRl5VCcncznf/PaSZlC735mPz+4djHffe4tfv7JCnx+i+LsFGbm6jyAUpORBoJJLpQf6ISzj7TEeJq7+jFAa4+Hbz6975SZQl0eHx19Hgozk3RfgFKTnA4NTWKWZXhhfyP//Ou/U9Pci8vrx+ny8sXNezjY1D3sTuFQptCUhHjuvW4JJTkpY9R6pdRoiXogEJFVIvK2iFSLyO3DPH+biBwQkb0islVEZkS7TbHA57N4ra6DbreX712zkH6/xSs17eEykcCwO4VtAhvWLmDh9HRWlU/RoSClYkBUA4GIxAEPAB8G5gMfF5H5Qy57HagwxiwCfgf8IJptmuwsy3C8s4eXDreyo6qFYx0u9hxzcu+fDmEZwkEgtCx0YHqIe9Yu4EPzCrhmSSEzclI1CCgVI6I9R7AcqDbG1ACIyGPAWuBA6AJjzPYB1+8EPhHlNk1aPp/FwSYnBxt7+c9tgT0BcTYoykoe1AsYuFN43YpS5hSkkZls54KSHF0WqlQMiva/+kLg2IDj+uC5U7kJeH64J0RknYhUikhlS0vLCDZxcrAsw3P7GnB5LVq63dxyaRnxNniisp6q5m4cdttJvYCOPg8lOSksLEznotJcDQJKxahxs2pIRD4BVADvH+55Y8wmYBNARUWFGcWmjWuh/EAdvR4anS76fdZJxeNf2NfA+pVl3Le1alAvYN6UdF0SqpSKeiA4DhQNOJ4ePDeIiHwQ+AbwfmNMf5TbNGl4PH7eONGJ2+OnsaufHo+fe18cfknoI6/U8uvPLMdgyE/T3EBKqXdEOxDsAspEZCaBAHA9cMPAC0RkKfAgsMoY0xzl9kwaHo+fPx5s4lh7Hy6vn007arj5ktJTLgnt6POQl5aoewKUUieJ6qCwMcYH3AL8AXgLeNwYs19ENojImuBlPwRSgSdEZI+IbIlmmyY6yzIcbu6h8lg7GLhva9Wg1UCnWhKqewKUUqcS9TkCY8xzwHNDzt054PEHo92GySK0Qez7L7zFuhWzaOpyDwoAw2UK/c7VC1lWnKl1g5VSpzRuJovVqQ2cED7Y2MXn3z+bE04XyQlxJwWA0GTwrLxUyqemU5qn+wGUUqengWCcCuUIqmvvRUS4f9shrl5WxKYdNeFv+1/64By+tmou333hYDgAzM5LpSDdwdLpmZolVCkVETFm4q3ErKioMJWVlWPdjKixLMNL1c10u/z09vtIccSTbI8LZwoNcdht3HLpbFxeK1AzuDSHaZkOirJ0GEgpdTIR2W2MqRh6XnsE40ygJ+CkpdvLN59+M/zt/3vXLBp2RZDbZ/HQyzWBmsEztGawUurd00Awjng8fnYebcVvSTgIQOAD/0hrTzg9RIjDbuOS2blcs7RQ9wUopd4z/fo4DliWoaa5m51H2+jzBFJEDP32/3hlPfesXTAoSdy91y3hvJJsnRBWSp0V7RGMMY/Hz47qFpq6+7nn2QO4vRbrV84+6dt/R5+HktxkNn/2Avq8fgrSdXewUmpkaCAYIx6Pn7cau/AbQ0piPE3Hndx8SSlP7q7n8cr6cG6g0BzBt69ayOJpuhJIKTXyNBCMAbfbx65j7bT2eGh0uk9KEvfozloeeaWWH167mKrmbpYWZbF8RoYGAaVUVOgcwSiyLENtWw+76zto6/GSk5LAY7vqTkoSd82y6XT0eUhPiufi2blcMCOLlKTEMW69Umqy0h7BKLAsQ117L1XNPbT3ergrWC5yYA+gwekGAsEgzgbfvXohF5bkaC9AKRV12iOIIssyHGnp4bW6dhq73KQmxoeDAAzuAYQ47DbeNyuXKxdN0yCglBoV2iOIEo/Hz8HmLvp9Fr39fnr6fcTbhKzkhPC3f3inBwAEJ4UXsHR6pm4MU0qNGg0EUeDx+PnrkRYEG609/Rxr7+Pxyno6+jzcdvkcfvnXo+FgEOoBzJuSzrQMB+XTMjQIKKVGlQaCEWRZhpqWHtr6XHT0+vj6U2+eNBdw758OsW5FKRu3VuOw29iwZgHZKXGcW1ygAUApNSY0EIwQl8vL68eddPR5SE6MDwcBGFwy8oHt1RRmJHHrytlUzMgiM8lOWUHm2DZeKRXT9CvoCOhzeXhmfyP//OtdHGjo5vW6jlOWjHTYbRx3ukhNjKezz8ucvLQxarVSSgVoj+AsuFxe3mzswuX1c+fv94U/7C3DsAnibAL/ftVCCrMcJMbZmFeQjsOhvwKl1NjST6H3qNvl5vl9Ldy5Zd9JReOfeeP4SSUjv33VApYWZTIzVxPEKaXGFw0E71Kfy8O+xm76fRZ3btk3qGaw22vx5O56brxgBpsr67jp4lLibLBoeiazcpKZma/DQEqp8UfnCN6FTpeb1487OdLai9dnkZWcABCuGeyw22hwutlcWcdXV80jyW5j3pR07DaYkZs6xq1XSqnhaY8gAj6fxeHWLt6o7w7PBTjsNtavLOORVwLpIUI1g+dNSScnJYHWnn4WTc+gKCuZGZouWik1jmmP4DQCOYK6eaWmjZoWF8kJcXzxg2VMzXDg9lrct7WKj1YE0kN09HkozEzCkSBY+CkrSOPi2XnM1KIxSqlxTnsEw7AsQ0NXLyc6+mlwujnc0hPeGbx+ZRmfW1HKz3bU0OB0U5afxn/90zKyUhJIsgtdLi8Vxbm6OUwpNWHop9UQlmU43OpkZ00nNz78d259bA8P7qjh0xeVkJWcwH1bq2jr83DNsuk47DYyk+wUpCfi81ukJdp53+x8DQJKqQlFP7GGONHVS2u3j28M2Rn84xcP8U/nF+P2WlgG4mywYc0CfMZPbloCF83KpTQ/TYeBlFITjg4NEZgMrm3vostlcbi1F79lht0ZnJeaGN4YdtGsXNIdcczOTdN00UqpCS3mA4HL5eVoZy/tvT5auvtx2OOIt8mwO4NTEuNZv7KMouxkFk5N1aphSqlJIWaHhjweP0dbO6lq7aHB2c+rR9qobunhh384SHJCHOtXBvYFAOGlogUZiVxYmsOH5hZoEFBKTRox1yNwu3283RIoGOPyWvS4fRxr7+OJ4KqgWy8rY+PWQ3zyopmsW1GKZcAmMDMnhcXTMnUYSCk16cRUj6DT5aaqrYumrn76vQZHvI3URBu/3VUXXhW0cVsV55fmYYwBYE5+KpfMzuGyc/I1CCilJqWoBwIRWSUib4tItYjcPszziSKyOfj8qyJSEo12dLrcHGro5Wiri8zkBMry7dS09FDb5uK2y+fwm7/Xcs2y6eHSkXmpDs4ryWZpcTrnztAi8kqpySuqQ0MiEgc8AFwO1AO7RGSLMebAgMtuAjqMMbNF5Hrg+8DHRrIdnS43fwxmCg2lh9iwppxLz8nkz293crill9WLCsP1ApYUZZKTGk9RRjLJSQkj2RSllBp3ot0jWA5UG2NqjDEe4DFg7ZBr1gK/Dj7+HbBSREZ0Mf6hxt5BmULdXos7t+ynrt3P1Kzk8L4Am8B3rl7IBTOyOGdKpgYBpVRMiPZkcSFwbMBxPXD+qa4xxvhExAnkAK0DLxKRdcA6gOLi4nfViKau/mH3BTR1u4mzCTaBpcVZ5KfaKctL12EgpVRMmTCrhowxm4BNABUVFebdvLYgPXHYfQEFaQ4QKJ8qLC1OJzPJMbKNVkqpCSDaQ0PHgaIBx9OD54a9RkTigQygbSQbMWdKChvWLBi0L2DDmnKKs+PIT43jklk5GgSUUjEr2j2CXUCZiMwk8IF/PXDDkGu2AJ8CXgGuBbaZ0NrNEZKZ5OBDC/IoyV1OU1c/BWmJFGfH4Uh0MCVTA4BSKrZFNRAEx/xvAf4AxAEPG2P2i8gGoNIYswV4CHhURKqBdgLBYsRlJjlYPlM/9JVSaqiozxEYY54Dnhty7s4Bj93AR6PdDqWUUsOLqZ3FSimlTqaBQCmlYpwGAqWUinEaCJRSKsbJCK/UHBUi0gLUvseX5zJk13IM0HuODXrPseFs7nmGMSZv6MkJGQjOhohUGmMqxrodo0nvOTboPceGaNyzDg0ppVSM00CglFIxLhYDwaaxbsAY0HuODXrPsWHE7znm5giUUkoNFos9AqWUUgNoIFBKqRg3aQOBiKwSkbdFpFpEbh/m+UQR2Rx8/lURKRmDZo6oCO75NhE5ICJ7RWSriMwYi3aOpDPd84DrPiIiRkQm/FLDSO5ZRK4L/q73i8hvRruNIymCv9fFIrJdRF4P/t2+YizaOZJE5GERaRaRfad4XkRkY/DPZK+ILDurNzTGTLr/CKS8PgyUAgnAG8D8Idd8HvhZ8PH1wOaxbvco3POlQHLw8b/Ewj0Hr0sDdgA7gYqxbvco/J7LgNeBrOBx/li3O8r3uwn4l+Dj+cDRsW73CNz3CmAZsO8Uz18BPA8IcAHw6tm832TtESwHqo0xNcYYD/AYsHbINWuBXwcf/w5YKSIyim0caWe8Z2PMdmNMX/BwJ4GKcRNZJL9ngHuA7wPu0WxclERyz58FHjDGdAAYY5pHuY0jKZL7NUB68HEGcGIU2xcVxpgdBOqznMpa4BETsBPIFJGp7/X9JmsgKASODTiuD54b9hpjjA9wAjmj0rroiOSeB7qJwDeKieyM9xzsMhcZY/5vNBsWRZH8nucAc0TkryKyU0RWjVrrRl4k9/st4BMiUk+g9skXRqdpY+rd/ns/rQlTvF6NHBH5BFABvH+s2xJNImID7gU+PcZNGW3xBIaHPkCg17dDRBYaYzrHslFR9HHgV8aYH4nIhQQqHi4wxlhj3bCJYrL2CI4DRQOOpwfPDXuNiMQT6FK2jUrroiOSe0ZEPgh8A1hjjOkfpbZFy5nuOQ1YAPxZRI4SGEvdMsEnjCP5PdcDW4wxXmPMEeAQgcAwEUVyvzcBjwMYY14BHAQSs01mEf17j9RkDQS7gDIRmSkiCQQmg7cMuWYL8Kng42uBbSY4CzNBnfGeRWQp8CCBIDCRx41DTnvPxhinMSbXGFNijCkhMC+yxhhTOTbNHRGR/N1+mkBvABHJJTBUVDOKbRxJkdxvHbASQETmEQgELaPaytG3BfhkcPXQBYDTGNPwXn/YpBwaMsb4ROQW4A8EVh08bIzZLyIbgEpjzBbgIQJdyGoCkzLXj12Lz16E9/xDIBV4IjgvXmeMWTNmjT5LEd7zpBLhPf8B+JCIHAD8wL8ZYyZkbzfC+/0y8HMR+RKBieNPT/AvdYjIbwkE89zg3MddgB3AGPMzAnMhVwDVQB/wmbN6vwn+56WUUuosTdahIaWUUhHSQKCUUjFOA4FSSsU4DQRKKRXjNBAopVSM00CglFIxTgOBUqchIj2j8B5rTpdCW6lo030ESp2GiPQYY1LHuh1KRZP2CFRMEZHvici/Djj+loh8M1io5zUReVNETkplLSIfEJFnBxzfLyKfDj4+V0T+IiK7ReQPp0sHLCK3DigO9Fjw3KdF5P7g41+JyH8Fs4bWBN/3YRF5S0R+NXJ/Ekq9QwOBijWbgesGHF9HoC7F1caYZQSK9/wo0toUImIH/hO41hhzLvAw8O+necntwFJjzCLgc6e4Jgu4EPgSgZwyPwbKgYUisiSSdin1bkzKXENKnYox5nURyReRaUAe0AE0Aj8WkRWARSCve0Hw/JmcQyDD6Z+CsSMOOF3yr73A/4jI0wSSww3nGWOMEZE3gSZjzJsAIrIfKAH2RNAupSKmgUDFoicIZJydQqCH8E8EgsK5xhhvMGW1Y8hrfAzuQYeeF2C/MebCCN/7HwmUIbwS+IaILBzmmlB6cGvA49Cx/ptVI06HhlQs2kwg2+y1BIJCBtAcDAKXAjOGeU0tMF9EEkUkk2DaY+BtIC9YEAURsYtI+XBvGiyUU2SM2Q58Nfi+OhGtxpx+u1AxJ5jGOA04boxpEJH/AZ4JDsVUAgeHec0xEXkc2AccIVAcHmOMR0SuBTaKSAaBf1M/AfYP89ZxwH8HrxNgozGmc2KXylaTgS4fVUqpGKdDQ0opFeN0aEipKBCRB4D3DTl9nzHml2PRHqVOR4eGlFIqxunQkFJKxTgNBEopFeM0ECilVIzTQKCUUjHu/wPOP/NCYCbGFQAAAABJRU5ErkJggg==\n" + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA19UlEQVR4nO3de3iU9Znw8e89ySST8zkcQiAEgkA4G89Kq1TX+iKotda6tbWvlmu3a6W173ZdW7Vit8etVla7lVbb6u5WtK4WWbVdgRZtxRIUOYkkBBIScj5MDjOTOTy/94/JjAkJMEgmp7k/1+XlPM88k/k9QOae3+m+xRiDUkqp2GUb7QYopZQaXRoIlFIqxmkgUEqpGKeBQCmlYpwGAqWUinHxo92AjyI3N9cUFRWNdjOUUmpc2bVrV4sxJu/E8+MyEBQVFVFeXj7azVBKqXFFRKqHOq9DQ0opFeM0ECilVIzTQKCUUjFOA4FSSsU4DQRKKRXjorpqSESeAlYCTcaYBUM8L8CjwDWAC7jNGPNONNuklFLjTYfbw6GGHho7e5mUnkhGUhyTMhxkJjmG5edHe/nor4DHgKdP8vwngZK+/y4A/r3v/0oppQgGge0ftFLZ3I1loLKpi1l5qXS4vJwzhWEJBlENBMaY7SJSdIpLVgNPm2Au7B0ikikiU4wx9dFsl1JKjRdHmlzUdbjZsL0Kj8/CYbexdkUJhVlJHGro4fyZZx8IRnuOoAA41u+4tu/cICKyRkTKRaS8ubl5RBqnlFKjxbIMVc3ddPX6eXRLBR6fBYDHZ/Holgq6ev00dvYOy3uNm53FxpgNwAaAsrIyraajlJpwLMtQ09ZDU1cv9U4PVc3dzJuSHg4CIR6fhcsbYFJ64rC872gHgjqgsN/xtL5zSikVUzweP/sbOqlq6eG+3+0LDwP91x0X4LDbBgQDh91GXmoisyYlD8t7j/bQ0Cbg8xJ0IeDU+QGlVCzx+y0OHO/gL0dbqXd6wkEAgt/8uzxuvnPdAhz24Me1w27jO9ctYNak5PGxakhEfgN8HMgVkVrgAcAOYIz5GfAKwaWjlQSXj34xmu1RSqmxxO+3eHV/PZ0ePw9tPsAdlxUPGgb6wi938+raC3n6i+fT2BVcPjpncsqwBQGI/qqhz57meQP8QzTboJRSY5HH42dPvRMQCrOSuGhmNsCQw0Cv7mth7uR0/s/CKdhsMuxtGe05AqWUihmWZTja2kN9h5tev8WDm/dT3erGYbfx4KpS3j7cwl1XlLB+a0V4juCh1Qs4d3omRbmpUQkCoIFAKaWiLrQa6J2aDu59cW/4Q/6uK0p4Zkc19U4PD2zazxO3nss9L+xlzfJi5k5OY2qGg/mTM0hIiItq+0Z7slgppSY0yzK8tr+B/363LhwEIDgRvH5rBTcsmxY+7nD5aHd5mZmTwhUl+SyZnh31IAAaCJRSKmr8fot3atq5+7ndWIYh9wNI32iPw25jcrqDzXdeyuolBTgcIzdgo0NDSik1zCzLcKS5mzqnB6fbxx2XFZOaGDfkRLAxwf+vW7WAxVMzRjQAhEhw4c74UlZWZrRmsVJqrPH7LfYfd9LQ1Uun2zdgY9jXPjEHR7yN7712cMBEcEaynawkOwunRD8IiMguY0zZiee1R6CUUsMgtCegoqmbkvy0QRvDHnn9EHdePps1y4uZnpVMc3cvDruNK+bkEx8/uqP0GgiUUuosWZZh33Ente3BLKFDbQzz+CwKs5PJSrbjtyxK8lMpnZox6kEANBAopdRH5vdb7Dvu5Fi7i7y0xAFZQoeaD8hOTqDD7eVv5k0ekdVAkRr9UKSUUuOQ32/x4u46bv75Dr7ym9385XBr+IP/hV213HVFyYD8QA+tXsCkDDufLJ0ypoIAaI9AKaXOSGh3cEt374B5AMt82Auod3p4Zkc1a5YXM39KOpnJdpZMzRyVFUGR0B6BUkpFyLIMb1Q2ceB4J42dvQOGfk7sBYQ2hs2dnMb5RTljNgiA9giUUioifr/F3uMd9HgCIJCZZB8wD1Dv9LCxvIYNt55LU1cvM3NTWFqYFbX8QMNJA4FSSp2G2+3jlfcb+Wa/PEHrVs3nwVWlPLBpf/jclz8+m5rWHjKSE1hckDkuggDohjKllDqp0AaxdrePv/uPXYNWAX3rmrkU5qTQ4fKFl4XmpCRSOmVsLAs9kW4oU0qpCIQmg/2Wn/aeAI2dHhLibUPuC2jo8tLU7eWCmdlMyUiiKCdl3PQC+tNAoJRSfUKZQv94sJ6yojzu3xRcFbR2xewh9wXYBJYWZnJBUc6Y7AFEavy2XCmlhollGapbuyk/2kaXx8enzp0RDgIAz5XXsnbFwH0Ba1eUMHdSGpcU547rIADaI1BKxTjLMuyubcXtNdS2u0lOiKfD7R3w7b/e6eHpt6p54tZz6XT7SXfEMy0ziZl50asaNpI0ECilYlKoali900NTVy9Vzd08V15Lu8vLd69fyIycJKpb3eHr211eHPFxJKQKSwuyxvS+gDM1ce5EKaUi5PUGeLeunZpW94BU0aHSkfe+uJef3rKML//XO/2Wiy5gWnYiU9LH54TwqWggUErFFL/f4o+VzSTFxw1KFb1+awW3X1rM49sq6en18/T/PZ/Gzl4mpyeOSL2A0TIx70oppYZgWYY9dR24vQE6erwnLR3psNvITk1gWWHWuJ8IjoQGAqVUTAgtDb37ud2nXRK6bvUClhVkxkQQAF0+qpSKEUdbe8JBAIZeEvov1y/kinPyuW7RVJKS7KPZ3BGlPQKl1IQT2h3c2OlhUrqDopwUGjs9Qy4J/dGNi6lo6uKi4hzOm5EdM72A/jQQKKUmDMsyHGnp4f36TmrbXRjA7Qtw3oxspmUlDRoKand5qWjqYmZOSswGAdChIaXUBOH1Bthe0czv3qvjYGMXv9lZA8Dz5bV86Zly9tY5eeyWpQOGgu5bOZ+LinO4dtHUmA0CoNlHlVITgNcbYPO+eu7tlyb6ritK2Fhew8pFBTy+rRKH3cb/fOUyAGraekhOiGdSeiLTsyfevoCT0eyjSqkJye+3+MuR1nAQgIF7AqTvM97js2ju9nBhcS6z8lNHscVjjwYCpdS4FEoRUdvhHjQRDMEP/jgbBPpOO+w28tMco9DSsS/qg2IicrWIfCAilSJyzxDPTxeRbSLyrojsEZFrot0mpdT45vdbvH2kldp2N81dvaQkxofH/kMcdhvzJqfz3+/U4rDbePimJRTlpIxSi8e2qPYIRCQOeBy4EqgFdorIJmPMgX6XfQt4zhjz7yIyH3gFKIpmu5RS45NlGY6193CszU11m4uHNh/A47OYkZPEAytLeXDzh2Ujv3v9QuZNSeORzywmP80xbovGjIRoDw2dD1QaY6oARORZYDXQPxAYIL3vcQZwPMptUkqNI36/xf56J+2uXnp6LTrcPho7PWzYXhUeDqpudfOz7ZX88MbFHGrs4rwZWVw0M4eEhDiKcnU+4HSiPTRUABzrd1zbd66/bwOfE5Fagr2Brwz1g0RkjYiUi0h5c3NzNNqqlBpjvN4AL+6u4zMbdrDzqJMPGrt4aPMBLMOgOYHqVjeHGrv4xRtVFGQmkZAQN0qtHn/GwsLZzwK/MsZMA64BnhGRQe0yxmwwxpQZY8ry8vJGvJFKqZFlWYZdx9rDGUJFGBAAhpoTsAk8fNMSivO0F3Amoj00VAcU9jue1neuv9uBqwGMMW+JiAPIBZqi3Dal1BhW09ZDp8cfXgJ6zqQ06jtcOOw2XthVy11XlLB+a8WAOYFl0zNjal/AcIl2INgJlIjITIIB4GbglhOuqQFWAL8SkXmAA9CxH6VimNcboLGrl063jyffrAp/2H/72lLu/eRcvvvqQZ7ZUc2a5cWU5Kcyb3I6xROkbORoiGogMMb4ReRO4PdAHPCUMWa/iKwDyo0xm4CvAz8Xka8RnDi+zYzH7c5KqY+sf5K4giwHlU09NHf1crzDTVZyAvXO4D6Bb7+8n198oYyHP70YBEry05ilAeCsRX1DmTHmFYKTwP3P3d/v8QHgkmi3Qyk1NlmWYesHjVQ1dTMjN5W6I+1866XB5SNDwaDT7aMgK4nSKRkxnR9oOOmfolJqVNW09XC83U3AwP7jznAQgA9TRdywbBoQnBCeOzmdxTFSOWyk6J+kUmrEWJahqrmbtw63cLipm6Mt3Rxs6CI7NZFnd9YMuSy0f/lI3R0cHZprSCk1Ik4sFemw21i7ooSn36qm3eXlritKMJghy0eWzchi9eJLdT4gSrRHoJQaESeWivT4LB7dEhz2CQ0B2US464qB5SO/d/1CLp6ZQ8mkNA0CUaI9AqVUVIVWBB1q7OKOy4p5YVct9U4P8OGwT+ix2xfg+fJa1iwvZs6kNOZNTmdmru4LiDYNBEqpYdV/KejUTAe7jzn5pxf2DLkKyGG3EVos7rDbuHRWLksLg5vCNACMHB0aUkoNm9A8wBd/9Vd2VbdT3eoKBwEYuAooNEcQShP9g08toqwom8vnTmJWvs4FjCTtESilhs3R1h6eevMwd60oobWrl9Ye75CrgGZkJ/HITUs41tbD16+aw+QMBxcW5eiH/yjRQKCUOiv9h4IQuPvKORxpddPtDdDT0jPkKqDqNjelU+NZXJgVc3WDxyIdGlJKfWShoaBr1r/BZ3/+Nve8sAeP34RTRT9XXst9K+cPWAV01xUlbN5Tx9zJ6VxQnENRrg4DjTbtESilPrITl4SuXFTA7mMd4eN2l5eNf63h4ZuWcLChk4AFG8tr+Ker5+nGsDFExmN+t7KyMlNeXj7azVAqplmW4d2ado47Pbh6/bT09BJnE7p7A/zijSqykhO49cIZrN9aQVZyAp8um6aZQkeZiOwyxpSdeF57BEqpiHi9AfYcd9LQ6WFKuoMAAXYddfLolg9rAjywcj6vH2gI1wronyp6/pQMXRI6RmkgUEqdlsfjZ9Peeu7f9GFW0HWrSnm3pnXA0tAHNx/g8VuWsW7zfm6/tJg4G5TNyObi4hxNEjeGaSBQSp2SZRneO+4MBwEIfujfv2k/T9x6Ln840BK+1uOzsCzD+puX4vIGmJTuoChHewFjnQYCpdSQQstCm7t6w7UA+vP4LDpcvgHnHHYbs/JTtWbwOKN9NaXUAJZlONrSzUu767hm/Ru8UdlCdop9yGLx+WmJA5aGapro8Ul7BEqpsNC+gIMNnWzYXhXuBfzHjiM8uKqUBzbt7zdHsIBp2Q5+/cXzcXn9mh9oHNNAoJQKC+0LuOOy4nAQeGFXLbdeOIMt79fzxK3n0uHykZ+WSEZSPIVZqRRm6TDQeKdDQ0opLMtQ2dhFRWMXP71lGZfMyqZsRgYA9U4Pz+yoZu6UTBo6PBxq7Oae/96Dw67fIycK/ZtUKsZZluHVffV8/fn3BlQOu/WiIuAo5dVO2l1ekuxx/GRLBe0ur84FTDAaCJSKUaH6wQ2dnnAQgA8rh61ZXsxdK+bQ1NlLZrKdlMR4lhRm6pLQCUiHhpSKQR6Pn03vHWflY2+y40jbkEtDLQNOt59/21ZBXloiF8zM4aJZuZoeYgLSHoFSMcSyDMfauqluc3PPf39YMGaoVNE2gYykeP7p6nksLMjUD/8JTHsESsUIv99ix5EW9h3vwun2D1gV9P+uOmfAfoC1K0qYnZdKXloCV5dO1iAwwWmPQKkJLrRB7J1jHXzrpWCaiMduWRruBdQ7PTz55hG+8TfnMDUzCYc9jqQEGyKGcyZlaBCIARoIlJqgLMtQ3drDrpoOatp6BmwQq213sXZFSThzaLvLS6/fIiUxjhR7PDlpCRRm6YRwrNBAoNQEEwoANe0uOt1+Wro8xNsGzgH8+i/V/N3yYtYsL8YyYBOYkZPMBTNySEiIG8XWq9GggUCpCcTvt9hxtJUGp4djbS6eK6+l3eXlJ59ZMmBCuN7p4am/HOGh1Qvo7vUzOy+V2flp2gOIURoIlBrnQllCW3t6qW1z888v7g1vDLvrihKe2VHN9159n/tWzuehzQfCz915eQlJCXFcWKS9gFgX9UAgIlcDjwJxwC+MMd8f4pqbgG8DBnjPGHNLtNul1EQQShJ393O7uf3SYp58s2rAxrD1Wyu4/dJiHt9WSZfHx5rlxUzPTiYvLZEZ2clMz9Z5ABVhIBCRFMBtjLFEZA4wF3jVGOM7zevigMeBK4FaYKeIbDLGHOh3TQnwz8Alxph2Ecn/iPeiVMwI9QKOtvTwQUMnWckJiDDkxjCR4JLQuZPTiY8T4m3CsmlZ2gtQYZH2CLYDl4lIFvAHYCfwGeBvT/O684FKY0wVgIg8C6wGDvS75kvA48aYdgBjTFPkzVcq9vTvBfQfArLZTr4x7IFrS+n1WxTnpTItU3sBaqBIN5SJMcYF3AD81BjzaaA0gtcVAMf6Hdf2netvDjBHRP4sIjv6hpIGN0BkjYiUi0h5c3NzhM1WauII7QfYebQtHATgwyEgf8Bw1xUlAzaGfee6BVw4M5vZ+SlcXpLH9GxND6EGi7RHICJyEcEewO1954arXxkPlAAfB6YB20VkoTGmo/9FxpgNwAaAsrIyM0zvrdS44PdbvH6wkSMtPbh9gSGHgDx+i5ffq+Phm5ZQ1dxNUU4KUzIcZKUkaJI4dUqR9gi+SnAc/0VjzH4RKQa2RfC6OqCw3/G0vnP91QKbjDE+Y8wR4BDBwKCUIhgEdla34XT78PgCJCfEDVk2cu6kNFYuKuAHr71PnE1wJNhYOj1Lk8Sp0xJjIv9yLSLpgDHGdEV4fTzBD/YVBAPATuAWY8z+ftdcDXzWGPMFEckF3gWWGGNaT/Zzy8rKTHl5ecTtVmo8sixDTVsP79R0cG+/JaFf+8QcHPE2vvfawfC5h1YvYHpOEp1uP+kOO5PSE3VFkBpERHYZY8pOPB/pqqEy4JdAWvBQOoD/a4zZdarXGWP8InIn8HuCQ0lP9fUo1gHlxphNfc9dJSIHgADwj6cKAkpNdKEAcKTFRU+vPxwEIDgE9Mjrh7jz8tncfmkxcTaYk59GQryNvFQHF8zUspHqzEU6R/AU8GVjzBsAInIpwcCw6HQvNMa8Arxywrn7+z02wN19/ykVs0KpIeo63LT2eKlq7ibuhNQQ8OF8wJNvVnH3lXPwWRZp9jitGKY+skgDQSAUBACMMW+KiD9KbVIq5liWYesHjVQ0docTwTnsNh65acmQS0LPL8ri4uIyHPFxmiBOnbVTBgIRWdb38E8i8gTwG4K7fz8D/DG6TVMqdhxp6WFPrXNAhlCPz+L7rw1ODXHfyvkYDBcW5+mHvxoWp+sR/PiE4wf6PdYlnEoNA7/f4khLN5YZvDO4utUdTg0xd3IaNoRp2UnMn6J1AtTwOWUgMMZcPlINUSqWeL0B9tY7ae7qJTkhjoxkO+mJcUMOA7m8AaZnJ/PSu8e4bul0DQJq2EW6aiiDYG9ged+pPwHrjDHOaDVMqYnK5fbydk0H79a0Yxl4+b06bj5vOlMyHNz7ybl899WBy0Jn5aXgCwS455OlujFMRcWZrBraB9zUd3wrwVVDN0SjUUpNVG63j1cONPGtlwamin52Zw2rlxQwb3I6P7pxMa5eP20uLzmpdvzG4ryiXA0AKmoiDQSzjDGf6nf8oIjsjkJ7lJqQLMtwrL2HeqcnHARgYKpoy8D7DZ2s31IZzhMkwLJp2RoEVFRFGgjcInKpMeZNABG5BHBHr1lKTQyWZTjS0sP79Z1kJttp7fEOuS8gri9jxOz8NO5aMZul07OYmZ3E9BxND6GiL9JA8HfA031zBQDtwBei0ySlxj/LMlQ1d3OgoZPDTd3hkpG/vO28ISeElxRm0tMbIM0Rx6pFUzU/kBpREQUCY8x7wOK+XEMYYzr7Py8iXzDG/DoK7VNqXAmlh9hV08E3hygZ+cs/H2bdqlLu37Q//Nx3rltIT6+XpAQ7y0vyNQCoEXdGSedO+kNE3jHGLDv9lcNDk86psShUMOZgQ+eAjWEQ/NYfKhn5rzcuIC89iQ6Xj0npiWQkxZMYH68rglTUnSzpXKRpqE/784fp5yg17oSGgUIFY4baGNa/ZGR+ehINTg9xIjjdPkry0nUoSI2q4QoEustYxaRQL+Ca9W/wRmVLOAAMVS/AJvDta0vx+CxSEuMpzEpixTmTiI8frl9DpT4a7REodRaOtvYMKBvpsNt4YVftoJKR962cz/kzs/ntrhrcvgBXz5/M4ulZGgTUmBDpqqHT+fMw/Rylxg2vN8DxDjd3XFYMwPYPmrjrihLWb63gmR3VrFlezMzcFJLi40hJjCM72c4Pb1yicwFqzIk0xcQk4LvAVGPMJ0VkPnCRMeZJAGPMnVFso1JjRmhfQE1bD8bAg5v3U93qDq8Mem1fPWuWF7O0MJPUxHg63D6ykhNYNDWDhIThKvOt1PCKtEfwK4IpJb7Zd3wI2Ag8GYU2KTXm+P0W++udHG11UdX84b6A0LLQeqeH9Vsr+rKEpvOxOboMVI0fkQ5Q5hpjngMsCJagJFhWUqkJz+Px8/bRVg42dGET4c+Vzdx64QyykhNYv7WCG5ZNC17ns5g3JZ2rSydrEFDjSqQ9gh4RyaFvdZCIXAho5lE14Xm9AV7eV899v9sX3gD2wMpSXninhhuWTePxbZVI32e+w24jLzVRg4AadyINBHcDm4BZIvJnIA+4MWqtUmoU+f0WBxs6aXN5cXsD1La7yEpOoN7pweOzeHDzfn5442IONXbhsNswJhgE1q1awKKpGad/A6XGmEhTTLwjIh8DziG4VPQDY4wvqi1TaoRZluFoSzf1nR7q2t0D0kD0nwvw+Cw8Xj82gYdWLyAn1c7ykvNZrBPCapyKdNXQ5084tUxEMMY8HYU2KTXi/H6L1w82cqSlB7cvMKh2cChV9OPbgimiC7KSmJSeiMdnUTIplYIMXRKqxq9Ih4bO6/fYAawA3gE0EKhxz++32FndxoH6YI6gOy4rPmWKiHWrF5DuiCdgwUUz07UXoMa9SIeGvtL/WEQygWej0SClRpLXG+B/DzZiDBRkJHHHZcWknqR28HlFWcydtJi0pDhKp2ZqD0BNGB91Z3EPMHM4G6LUSAolimvs8lDT5uLRLRXh+YCvfWIO/3z1XL732oe1g9etKiU5IY6CjCRNEKcmnEjnCF7mw8RyNmA+8Fy0GqVUtFiW4XBzNxVN3VjGkJFkDwcBCA4BPfL6Ie68fDZrlhczOy+VNEc83oBh4eQMHI7hysqi1NgR6b/qf+332A9UG2Nqo9AepaLCsgy1HT2UH+3g3n4FY75z3YIh5wOKclNId8TjtyzSk+wsnKIrgtTEFekcwZ+i3RClosXvt3jrSDM2sXG0tYc7LivmhV211Ds9HGtzDTkfMDndQW5qAtOzdTWQmvhOGQhEpIuhaw0IYIwx6VFplVLDxOsNsLuugwand8Du4NC+gOfKa7lv5Xwe2nwg/NyPP72EZdOzNAComHHKQGCMSRuphig1nCzLcLipm4ONXWSn2MNBAAbuC3jyzSp6PD6e/MJ5uL1+inJSdDJYxZwzmvkSkXyC+wgAMMbUDHuLlDpLlmV4dV8DX38+WDDmrhWzh5wHiLPBfSvnk+6IJ90RxyWzc0epxUqNroiyj4rIKhGpAI4AfwKOAq9G+NqrReQDEakUkXtOcd2nRMSIyKDCykpFyrIMe2s7+KCxkzsuK2ZKhgPLDF068oKZ2RRmOfj1X46QkmgfpRYrNfoi7RE8BFwIvG6MWSoilwOfO92LRCQOeBy4EqgFdorIJmPMgROuSwPWAm+fSeOV6s/vt3hlXz3feGHPgLmA1/bVhyuH9V8t9IvtVbx1pI2HbwpWDVMqVkUaCHzGmFYRsYmIzRizTUR+EsHrzgcqjTFVACLyLLAaOHDCdQ8BPwD+McL2KAUEewDVrT0cd7rp8vg53Nw9IFNoaC5gY3kNP7xxMZVNXVw6Oze4OSxzFvddW6qlI1XMizQQdIhIKvAG8J8i0kRwd/HpFADH+h3XAhf0v0BElgGFxpj/EZGTBgIRWQOsAZg+fXqEzVYTmdcboLymjSOtrgGrfk7MFBpng8+UTefHfzjIN/5mLmUzsvWDX6l+Iq1Qtg3IIDh88xpwGLj2bN9cRGzAw8DXT3etMWaDMabMGFOWl5d3tm+txjG/3+K9Y+28eqCBNpePDdsPD1oRFKoa5rDbuKg4h/lT03jy8+fxyQVTNAgodYJIewTxwB+ANoK1ijcaY1ojeF0dUNjveFrfuZA0YAHwRwmWeZoMbBKRVcaY8gjbpmKE1xtgX72TY+3uk9YNhoGZQteuKAFgeYnWEFbqZCLqERhjHjTGlAL/AEwB/iQir0fw0p1AiYjMFJEE4GaClc5CP9dpjMk1xhQZY4qAHYAGATWA1xtgd007v9tznFt+8TZrn93NE9urhqwbDMEAcM6kNNYsL2Zmbgrn6VCQUqcU6dBQSBPQALQC+ae7uK/I/Z3A74H3geeMMftFZJ2IrDrTxqrY4/UGeGnPcbZ+0DTkprAblk0LzwNAMAj8y/ULyUqOZ/XiAq6aP5n4+DP9Z65UbIk0++iXgZsI1ip+HvjSiUtAT8YY8wrwygnn7j/JtR+P5Geqic+yDEdbe2jq8nD/7/adtlhM2Yws1t+8hKKcZOZPydAPf6XOQKRzBIXAV40xu6PYFqUA8Hj8vHKggXtf3MuD15aGA8BQyeFsAmtXlJCWGM/FxbkaAJT6CCKdI/hnDQJqJHg8fv5ytDWcKjo5MR6H3cYLu2q564qS8A5hh93GfSvnk5oQR0FWEgsLMjUIKPURaZUNNSaECsYcbu6m0+0Lf/P/+fbDPLCylAc37+eZHdWsWV7MrLxUclMT6O71s2BqOqU6FKTUWdFAoEaV1xtg73En9U4P6UnxNHV5cPUGwsNAe+o64a/V/OuNiwkYQ2aynWOtPWQlJ7DinEkaAJQaBhoI1Khxu328vL+B+/vVCXjg2lKmZsRz95VzePh/D+HxWRxq6sYXsFg6PZOGTg8Xz87TtBBKDSMxZqi6M2NbWVmZKS/XrQbjUWg1UH2Hm/h4G+/VtNPZGwhXDHPYbfzrjYs51u5iWlYyFU1dLCnM5OKiHK0XrNRZEpFdxphBGZ71N0uNGMsybDnYyN46J5aBOIGinGT+cKCaWy+cEd4d3OP14/YFSEmI45JZuSyeqkXjlYomHWBVI8KyDAeOO6ls6mbD9ioe21rJE9uraOjs5e8/PpuN5TXcsGwaDruNlIR45k1Op9vrZ+m0TA0CSkWZBgIVdX6/xeY9x2nq6uXRLRUDdgc/uqUCt9di5aIC4mzwwLWlJNptzMxN4prSKSQkxI1y65Wa+DQQqKjyegPsqmnnGy/soccbGHJ3cE+vnzgbXDwrl/mT0/jY7DzmTtF9AUqNFO1zq2EXKhbT1OWhobOXOJuQlZyAPU6G3B3c5vKyaFomIobSqbonQKmRpoFADSvLMmz9oJHj7W5aerzhSeG//1gxG/9aw7pVpdy/aX94uejdV84hLy0RG4ZzC7M1CCg1CjQQqGFhWYaath4auzw44uNIS7LT2uPlhV3BmgFrV5Rw0awc/mNHNT+5aQkGSHXEk2S3kZuayPRs3Reg1GjRQKDOmtcb4L3jHdR3uKnr8IQnhPuXjXx0SwU/vWUZH58bIDPFjt0mtLt9XDIrVwOAUqNMA4H6SCzLUNXcTVOXm9qOXmrbXQBs2F41qGbA7ZcW8/i2Slq6eynITMLt9TM1P5Wl07VgjFJjgQYCdcb8fovXDzZypKWHxYUZ1La7KMhIIjkxnqzkhHDJSBhYM6AoN4Ukexzn5KfpslClxhANBCpilmU40tJDRVMX8TYhJ8VOXbsn3AsI1Qh++q0P6weHagb84FOLKNOSkUqNSRoIVEQsy/Dqvga+/vzu8If+Q6sX8Ni2wRvE1iwvZv2WShx2G/evnK91g5Ua4zQQqIhUNXeHgwAEP/Tv+92+8Ph/iMdnMXdyOj+/9Vwyk+3kpemKIKXGOg0E6qT8fot9x500d/diGTPkruC4E5b9O+w25uSnMntS2gi2VCl1NnT3jhqS32/x4u461m58l4rGbg4c7wyXiQxx2G3MnZw+oHzkDz61iOK81NFoslLqI9IegRrSvuNOHttWwdevmss3fvseWckJ3HVFCeu3Dtwj8LM/VnL7pcXE2eCiWTmcp0tClRp3NBCosFDRmNaeXtp6vHymbDqVTV14fBb1Tg/P7Kjm9kuLSYy3saAgnQdf3k91q5tDTd388FOLOG+6pohQajzSQKCA4O7gvxxp5Z2adkry08hIimf91gruuKw4nCiu3unh8W3B1UDP3H4+D396Cd6AxaR0h5aOVGoc00AQw/x+i/31TtpdvbT1+Hl0yyFWLirA5fXj9Vt4fBYv7KodNCT00OoFLJ2Wpd/+lZogNBDEIL/f4mBDJ+83dPHYtgq+vaqU3NQE/t9Vc7HHCT/dVsnlc/Nx2G0DhoTibHDprFyWTNNaAUpNJPrbHGO83gAv7q7jD+83snFnNQ+sLMXp8rOrup3vvvI+X924m89eMIM9xzpYu6IkHAyefLOK2XmpLJuepekhlJpgtEcQQyzLsL/BSUaSncKsHKZnJ/Pl/3pnUKbQB1/ezw9vXMz3Xnmfn3++DH/AYnp2CjNzdR5AqYlIA8EEF6oWdtzpxjIWLq+FMdDS7eVbL+07aaZQt9dPu8tLQWaS7gtQaoLToaEJLFQt7LX9Ddz74l7ibDacLh9f3bibg41dQ+4UDmUKTUmI5+GbllCUkzJKrVdKjZSoBwIRuVpEPhCRShG5Z4jn7xaRAyKyR0S2iMiMaLcpFvj9Fu/UtOP2WsQJ3HL+DHZUtYXLRAJD7hS2CaxbvYCF09K5unSyDgUpFQOiGghEJA54HPgkMB/4rIjMP+Gyd4EyY8wi4LfAD6PZponOsgx1Hd28cbiF7RXNfNDYRcDAI68fwjKEg0BoWWj/9BAPrV7AVfMmccOSAmbkpGoQUCpGRHuO4Hyg0hhTBSAizwKrgQOhC4wx2/pdvwP4XJTbNGH5/RYHG50cbOjh37YG9wTE2aAwK3lAL6D/TuE1y4uZMymNzGQ7Fxbl6LJQpWJQtH/rC4Bj/Y5r+86dzO3Aq0M9ISJrRKRcRMqbm5uHsYkTg2UZXtlXj9tn0dzl4c7LS4i3wfPltVQ0deGw2wb1AtpdXopyUlhYkM7FxbkaBJSKUWNm1ZCIfA4oAz421PPGmA3ABoCysjIzgk0b00L5gdp7vDQ43fT6rUHF41/bV8/aFSU8uqViQC9g3uR0XRKqlIp6IKgDCvsdT+s7N4CIfAL4JvAxY0xvlNs0YXi9Ad473oHHG6Chs5dub4CHXx96SejTb1Xz6y+ej8GQn6a5gZRSH4p2INgJlIjITIIB4Gbglv4XiMhS4AngamNMU5TbM2F4vQH+cLCRY20u3L4AG7ZXccdlxSddEtru8pKXlqh7ApRSg0R1UNgY4wfuBH4PvA88Z4zZLyLrRGRV32U/AlKB50Vkt4hsimabxjvLMhxu6qb8WBsYeHRLxYDVQCdbEqp7ApRSJxP1OQJjzCvAKyecu7/f409Euw0ThWUZXtvfwA9ee581y2fR2OkZEACGyhT63esXsmx6ptYNVkqd1JiZLFYn139C+GBDJ1/+2GyOO90kJ8QNCgChyeBZeamUTkmnOE/3AyilTk0DwRhlWYYjLT3UtPUgIjy29RDXLytkw/aq8Lf9r31iDv989Vy+99rBcACYnZfKpHQHS6dlapZQpVRExJjxtxKzrKzMlJeXj3YzosayDG9UNtHlDtDT6yfFEU+yPS6cKTTEYbdx5+WzcfusYM3g4hymZjoozNJhIKXUYCKyyxhTduJ57RGMMX6/xZHWTpq7fHzrpb3hb//fv2HRkCuCPH6LJ9+sCtYMnqE1g5VSZ04DwRji8fh5q7oFjC0cBCD4gX+kpTucHiLEYbdx2excblhaoPsClFIfmX59HAMsy1DV1MXOY+30+gzNXZ5B3/6fK6/lodULBiSJe/imJZxXlK0Twkqps6I9glHm9QbYXtlMY1cvD20+gMdnsXbF7EHf/ttdXopyk9n4pQtx+QJMStfdwUqp4aGBYJR4vQHeb+gkYAwpifE01jm547JiXthVy3PlteHcQKE5gu9ct5DFU3UlkFJq+GkgGAUej5+dx9po6fbS4PQMShL3zI5qnn6rmh/duJiKpi6WFmZxwQwNAkqp6NA5ghEUrB/cza7adlq7feSkJPDszppBSeJuWDaNdpeX9KR4Lpudy0VF2SQnJYxy65VSE5X2CEaAZRlq2nqoaOqmrcfLA33lIvv3AOqdHiAYDOJs8L3rF3JRUY72ApRSUac9giiyLMOR5m7eqWmjodNDamJ8OAjAwB5AiMNu45JZuVy7aKoGAaXUiNAeQZR4vQEONnXS67fo6Q3Q3esn3iZkJSeEv/3Dhz0AoG9SeAFLp2XqxjCl1IjRQBAFXm+APx9pRrDR0t3LsTYXz5XX0u7ycveVc/jln4+Gg0GoBzBvcjpTMxyUTs3QIKCUGlEaCIaRZRmqmrtpdblp7/Fz74t7B80FPPy/h1izvJj1Wypx2G2sW7WA7JQ4zp0+SQOAUmpUaCAYJm63j/fqnXS6/djjbeEgAANLRj6+rZKCjCTuWjGbshlZZCbZKZmUObqNV0rFNP0KOgxcbi+b9zdw2y938l6tk3dr2k9aMtJht1HndJOaGE+Hy8ecvLRRarVSSgVpj+AsuN0+9jZ04vYFuO93+8If9pZhyARxNoF/uW4hBVkOEuNszJuUjsOhfwVKqdGln0IfUZfbw6v7mrl/075BReNffq9uUMnI71y3gKWFmczM1QRxSqmxRQPBGXK5vexr6KLXb3H/pn0DagZ7fBYv7Krl1gtnsLG8htsvLSbOBvMmpzN3Uioz83UYSCk19ugcwRnocHt4t87JkZYefH6LrORg2odQzWCH3Ua908PG8hruuXoes/NSSLLHER8nzMhNHeXWK6XU0LRHEAG/3+JwSyfv1XZxf99cgMNuY+2KEp5+K5geIlQzeN7kdLJTEvD4/MTZhKvmT2ZmrqaLVkqNXdojOAXLMhxr7+KvR9s41OgiOSGOOfnBb/Yen8WjWyr4dFkwPUS7y0tBZhKOBMEQYGpmMp+YN4lZ+TonoJQa27RHMATLMtR39lDf4aWuw01Vc3d4Z/ADK0vhr9XsqevE47MoyU/j3/92GVkpCSTZhU63j7Lpubo5TCk1buin1Qksy3C4xcmOqg4+9+TbrH12N09sr+K2i4vISk7gwc37uWP5LCA4QZyZZGdSeiL+gEVaop1LZudrEFBKjSv6iXWC4509tHT5+eYJO4Mfef0Qf3vBdDw+C7fXH04P4TcBctMSuHhWLsX5aToMpJQad3RoiGCSuOr2Lro9FodbeghYZsidwXmpiTjsNgqzknnqtvPITIpndm6apotWSo1rMR8IXG4v1R0u2nr8NHf14rDHEW+TIXcGpyTGs27VAhLtwrz8NK0appSaEGI2EHg8fo45u3G6AtQ5PQMmhDfceu6g4vFrV5QwJcNBUoKNOfnpOg+glJowYi4QuNxePmjqJmAM3b0Bmjo9NHf38tLuOm69cAbP7Kjmvt/t4+tXncOa5cVYBmwCM3NSWDg1Q4eBlFITTkx9re1wezjU3E1zdy9ur0WS3UZuWgLPlR/jlvODaSFuWDaN6lY3xhgA5uSnctnsHK44J1+DgFJqQop6j0BErgYeBeKAXxhjvn/C84nA08C5QCvwGWPM0eFuR4fbw58r2vigsQvLQJzA5PREfAHDVz9Rwk9er2DlooJwqui8VAfZKYkU5TgoyNRNYUqpiSuqgUBE4oDHgSuBWmCniGwyxhzod9ntQLsxZraI3Az8APjMcLajw+1h77FOatpcbNheNWDcPyspniMtLlYuKgjXDl63egE5qfEUZiTrhLBSasKL9tDQ+UClMabKGOMFngVWn3DNauDXfY9/C6wQkWH9+n2ooQen2x+e/IUPU0RMyUoO9hBssKQwk4+X5HFt6WTOmZypQUApFROiHQgKgGP9jmv7zg15jTHGDziBnBN/kIisEZFyESlvbm4+o0Y0dvbS0+sfcm9AT68fm8DS6VnMzk9k0bRMkpLsZ/TzlVJqPBs3k8XGmA3GmDJjTFleXt4ZvXZSeiLtLi8O+8Dbddht5KYmUjolnaXT05menanLQpVSMSfan3p1QGG/42l954a8RkTigQyCk8bDZs7kFKZmJnH3lXPCwcBht/G96xeSnxrHx2bnkZnkGM63VEqpcSPaq4Z2AiUiMpPgB/7NwC0nXLMJ+ALwFnAjsNWE1m4Ok8wkB8vPyaGqycUTt55Lr9ciPz2BorxkDQBKqZgX1UBgjPGLyJ3A7wkuH33KGLNfRNYB5caYTcCTwDMiUgm0EQwWwy4zycGyGfqhr5RSJ4r6PgJjzCvAKyecu7/fYw/w6Wi3Qyml1NB0ZlQppWKcBgKllIpxGgiUUirGaSBQSqkYJ8O8UnNEiEgzUP0RX54LtAxjc8YDvefYoPccG87mnmcYYwbtyB2XgeBsiEi5MaZstNsxkvSeY4Pec2yIxj3r0JBSSsU4DQRKKRXjYjEQbBjtBowCvefYoPccG4b9nmNujkAppdRAsdgjUEop1Y8GAqWUinETNhCIyNUi8oGIVIrIPUM8nygiG/uef1tEikahmcMqgnu+W0QOiMgeEdkiIjNGo53D6XT33O+6T4mIEZFxvdQwkvsVkZv6/p73i8h/jXQbh1sE/66ni8g2EXm379/2NaPRzuEkIk+JSJOI7DvJ8yIi6/v+TPaIyLKzekNjzIT7j2DK68NAMZAAvAfMP+GaLwM/63t8M7BxtNs9Avd8OZDc9/jvY+Ge+65LA7YDO4Cy0W53lP+OS4B3gay+4/zRbvcI3PMG4O/7Hs8Hjo52u4fhvpcDy4B9J3n+GuBVQIALgbfP5v0mao/gfKDSGFNljPECzwKrT7hmNfDrvse/BVaIiIxgG4fbae/ZGLPNGOPqO9xBsGLceBbJ3zPAQ8APAM9INi4KIrnfLwGPG2PaAYwxTSPcxuEWyT0bIL3vcQZwfATbFxXGmO0E67OczGrgaRO0A8gUkSkf9f0maiAoAI71O67tOzfkNcYYP+AEckakddERyT33dzvBbxTj2Wnvua/LXGiM+Z+RbFiURPJ3PAeYIyJ/FpEdInL1iLUuOiK5528DnxORWoK1T74yMk0bVWf6+35KUS9Mo8YeEfkcUAZ8bLTbEk0iYgMeBm4b5aaMpHiCw0MfJ9jj2y4iC40xHaPZqCj7LPArY8yPReQighUPFxhjrNFu2HgxUXsEdUBhv+NpfeeGvEZE4gl2KVtHpHXREck9IyKfAL4JrDLG9I5Q26LldPecBiwA/igiRwmOpW4axxPGkfwd1wKbjDE+Y8wR4BDBwDBeRXLPtwPPARhj3gIcBBOzTWQR/b5HaqIGgp1AiYjMFJEEgpPBm064ZhPwhb7HNwJbTd8szDh12nsWkaXAEwSDwHgfO4bT3LMxxmmMyTXGFBljigjOi6wyxpSPTnPPWiT/rl8i2BtARHIJDhVVjWAbh1sk91wDrAAQkXkEA0HziLZy5G0CPt+3euhCwGmMqf+oP2xCDg0ZY/wicifwe4KrDp4yxuwXkXVAuTFmE/AkwS5kJcFJmZtHr8VnL8J7/hGQCjzfNy9eY4xZNWqNPksR3vOEEeH9/h64SkQOAAHgH40x47anG+E9fx34uYh8jeDE8W3j/EsdIvIbggE9t2/u4wHADmCM+RnBuZBrgErABXzxrN5vnP95KaWUOksTdWhIKaVUhDQQKKVUjNNAoJRSMU4DgVJKxTgNBEopFeM0ECilVIzTQKDUKYhI9wi8x6pTpdBWKtp0H4FSpyAi3caY1NFuh1LRpD0CFVNE5Psi8g/9jr8tIt/qK9TzjojsFZFBqaxF5OMisrnf8WMiclvf43NF5E8isktEfn+qdMAicle/4kDP9p27TUQe63v8KxH5977MoVV97/uUiLwvIr8avj8JpT6kgUDFmo3ATf2ObyJYl+J6Y8wygsV7fhxpbQoRsQP/BtxojDkXeAr4l1O85B5gqTFmEfB3J7kmC7gI+BrBnDKPAKXAQhFZEkm7lDoTEzLXkFInY4x5V0TyRWQqkAe0Aw3AIyKyHLAI5nWf1Hf+dM4hmOH0f/tiRxxwquRfe4D/FJGXCCaIG8rLxhgjInuBRmPMXgAR2Q8UAbsjaJdSEdNAoGLR8wQzzk4m2EP4W4JB4VxjjK8vZbXjhNf4GdiDDj0vwH5jzEURvvf/IViG8FrgmyKycIhrQunBrX6PQ8f6O6uGnQ4NqVi0kWC22RsJBoUMoKkvCFwOzBjiNdXAfBFJFJFM+tIeAx8AeX0FURARu4iUDvWmfYVyCo0x24B/6ntfnYhWo06/XaiY05fGOA2oM8bUi8h/Ai/3DcWUAweHeM0xEXkO2AccIVggHmOMV0RuBNaLSAbB36mfAPuHeOs44D/6rhNgvTGmY3yXylYTgS4fVUqpGKdDQ0opFeN0aEipKBCRx4FLTjj9qDHml6PRHqVORYeGlFIqxunQkFJKxTgNBEopFeM0ECilVIzTQKCUUjHu/wMEkxgTX3YboAAAAABJRU5ErkJggg==\n" }, "metadata": { "needs_background": "light" @@ -667,7 +682,8 @@ "source": [ "import seaborn as sns\n", "\n", - "sns.scatterplot(data=merged_results, x=\"value_sim\", y=\"value_obs\")" + "sns.scatterplot(data=merged_results_wo_zeros, x=\"value_sim\", y=\"value_obs\")\n", + "#sns.regplot(data=merged_results, x=\"value_sim\", y=\"value_obs\")" ], "metadata": { "collapsed": false, @@ -678,13 +694,13 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 304, "outputs": [ { "data": { - "text/plain": "LinregressResult(slope=0.999889299809943, intercept=5.271437621759412e-06, rvalue=0.9999938263765986, pvalue=0.0, stderr=3.49259911845371e-05, intercept_stderr=6.896656958760079e-06)" + "text/plain": "LinregressResult(slope=0.9998410722635966, intercept=4.4762082968718975e-05, rvalue=0.9999904091368037, pvalue=0.0, stderr=0.00011336879523644994, intercept_stderr=5.826949896067866e-05)" }, - "execution_count": 159, + "execution_count": 304, "metadata": {}, "output_type": "execute_result" } @@ -692,7 +708,7 @@ "source": [ "#from scipy.special import rel_entr\n", "from scipy.stats import linregress\n", - "linregress(merged_results.value_sim.values, merged_results.value_obs.values)" + "linregress(merged_results_wo_zeros.value_sim.values, merged_results_wo_zeros.value_obs.values)" ], "metadata": { "collapsed": false, @@ -701,85 +717,31 @@ } } }, - { - "cell_type": "markdown", - "source": [ - "#### generic power discrepancy implementation in statsmodels\n", - "\n", - "lambd{float, str}\n", - "float : exponent a for power discrepancy\n", - "\n", - "‘loglikeratio’: a = 0\n", - "\n", - "‘freeman_tukey’: a = -0.5\n", - "\n", - "‘pearson’: a = 1 (standard chisquare test statistic)\n", - "\n", - "‘modified_loglikeratio’: a = -1\n", - "\n", - "‘cressie_read’: a = 2/3\n", - "\n", - "‘neyman’ : a = -2 (Neyman-modified chisquare, reference from a book?)\n", - "\n", - "note - they are all based on continuous approximations (x^2) so technically they might be inadequate for very small\n", - "occurences of counts" - ], - "metadata": { - "collapsed": false - } - }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 305, "outputs": [ { "data": { - "text/plain": "(0.3439655414093617, nan)" + "text/plain": "" }, - "execution_count": 64, + "execution_count": 305, "metadata": {}, "output_type": "execute_result" - } - ], - "source": [ - "# Note power tests will be inf when expected count is zero (any non-zero count will be impossible)\n", - "eps = 0 #1e-14\n", - "non_zero_shares = ms_comp.loc[(ms_comp.mode_share_obs > eps) & (ms_comp.trip_mode > eps)][[\"mode_share_obs\", \"trip_mode\"]]\n", - "ddof = non_zero_shares.shape[0] - 1 # what to use here? num alternatives - 1? What about zero expected count?\n", - "\n", - "non_zero_shares.trip_mode *= num_samples\n", - "\n", - "sum_ms = non_zero_shares.mode_share_obs.sum()\n", - "if sum_ms != 1.0:\n", - " if np.abs(1.0 - sum_ms) > 1e-1:\n", - " print(f\"NOT GOOD {sum_ms} should be one\")\n", - " #print(f\"sum ms is not one but {sum_ms}, re-normalising\")\n", - " non_zero_shares.mode_share_obs /= sum_ms\n", - "\n", - "powerdiscrepancy(non_zero_shares.trip_mode.values, non_zero_shares.mode_share_obs.values, lambd=0, ddof=ddof)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 99, - "outputs": [ + }, { "data": { - "text/plain": "array([[4.17345080e-03, 5.98926219e-03, 5.00000000e-03],\n [1.45406892e-04, 6.18851051e-04, 3.00000000e-04],\n [4.13473933e-02, 4.66025328e-02, 4.39000000e-02],\n [5.64412252e-02, 6.25036058e-02, 5.94000000e-02],\n [8.87346329e-01, 8.95325012e-01, 8.91400000e-01]])" + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEKCAYAAADenhiQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAB0GklEQVR4nO29eYBcV3ng+/vuraX3RepuydYSS1hG2I4NtobgGT+PAg6xw3t2Mk8h9gwDmWennUBiIANjM2GLCfPsAWJwQoj9zE5iA3ohiBcvYBwhmFhg2cbBxrIlWsJaLPWi3qprv/d7f9x7q6urq7qrqqu6qlvnB21V3bq36py7nO+cbxVVxWAwGAyGWmI1ugEGg8FgWH0Y4WIwGAyGmmOEi8FgMBhqjhEuBoPBYKg5RrgYDAaDoeYY4WIwGAyGmtNw4SIi14jIiyJyWERuL/J5VES+7n/+YxE5L++zD/jbXxSR38zb/l4ReV5EnhORB0SkZZm6YzAYDAYaLFxExAY+C1wLXAjcKCIXFux2EzCuqucDdwN3+cdeCNwAXARcA/yNiNgisgG4FdihqhcDtr+fwWAwGJaJUIN///XAYVUdAhCRB4HrgZ/n7XM98FH/9W7gr0VE/O0PqmoKOCIih/3vexmvX60ikgHagJOLNaSvr0/PO++8WvTJYDAYzhqeeuqpUVXtL9zeaOGyATiW9/448Gul9lHVrIhMAmv97fsLjt2gqk+IyCfxhEwC+K6qfrfYj4vIIDAIsHnzZg4cOLD0HhkMBsNZhIj8stj2httcao2I9OKtarYA5wLtIvK2Yvuq6n2qukNVd/T3zxO8BoPBYKiSRguXE8CmvPcb/W1F9xGRENANjC1w7NXAEVUdUdUM8A/Av61L6w0Gg8FQlEYLlyeBbSKyRUQieIb3PQX77AHe4b/eBTyuXrbNPcANvjfZFmAb8BM8ddgbRKTNt828CXhhGfpiMBgMBp+G2lx8G8ofA4/ieXV9QVWfF5E7gAOqugf4PPBV32B/Bt/zy9/vG3jG/yzwLlV1gB+LyG7gaX/7M8B9y903g8FgOJsRk3LfY8eOHWoM+gaDwVAZIvKUqu4o3N5obzHDKmHvwWHu3TfEsfE4m3rbuOWqrezcPtDoZhkMhgbRaJuLYRWw9+AwH97zPMPTSXpawwxPJ/nwnufZe3C40U0zGAwNwggXw5K5d98QYVtoi4QQ8f4N28K9+4Ya3TSDwdAgjHAxLJlj43Faw/acba1hm+Pj8Qa1yGAwNBpjczFURb6NZSqRwXFd+jpm84MmMg4be9sa2EKDwdBIjHAxVExgYwnbQk9rmKzjMjydBmBte5RExiHjKLdctbXBLTUYDI3CCBdDxeTbWAD6O70Vy0zKIWRl2Gi8xQw1xngjrjyMcDFUzLHxOD2t4Tnb+jqiTCYy/PC2NzaoVYbVSuFKOfBGvAOMgGlijEHfUDGbettIZJw524yNxVAvjDfiysQIF0PF3HLVVjKOEk9nUfX+NTYWQ70w3ogrEyNcDBWzc/sAd1x3EQOdLUwmMgx0tnDHdRcZFYWhLpiV8srE2FwMVbFz+4ARJoZl4ZartvLhPc8TT2dpDdvGG3GFYISLoSYYbx5Dvdi5fYA78Gwvx8fjxhtxhWCEi2HJGG8eQ70xK+WVhxEuhgUpZ0VSGPfSFgkRT2e5d9+QGRAMhrMUY9A3lKTcbMfGm8dgMBRihIuhJOXGFxhvHoPBUIhRixlKEkTiTyUyjMZSpB2XiG0xPJXkxvv251RlV2xdw+6nTxhvHoPBkKPhKxcRuUZEXhSRwyJye5HPoyLydf/zH4vIeXmffcDf/qKI/Gbe9h4R2S0iB0XkBRG5Ypm6s6rY1NvGaCzFyckEWUexRUhmHaZTDkdGYzlV2Vf3/xLXcTg+nuDQcIyIbZm4F4PhLKehwkVEbOCzwLXAhcCNInJhwW43AeOqej5wN3CXf+yFwA3ARcA1wN/43wfwGeARVd0OXAq8UO++rEZuuWor4/EMAGJB1nVxXO+z4ekU08ksjquMxzOMxjNsG+igty3M0OgM79v9LDfet99UozQYzlIavXJ5PXBYVYdUNQ08CFxfsM/1wJf917uBN4mI+NsfVNWUqh4BDgOvF5Fu4Crg8wCqmlbVifp3ZfWxc/sAnS0hLCCZccm6s5+5CicnE7wykcAScFxlOpllbCaNq0o8lTXljg2Gs5hGC5cNwLG898f9bUX3UdUsMAmsXeDYLcAI8EUReUZE7heR9mI/LiKDInJARA6MjIzUoj+rjv6OKI6CyNztAlgIaUcBiNgWo7EUFoJtCRlXTYLBJmfvwWFuvG8/V971uFllGmpOo4VLPQgBlwGfU9XXATPAPFsOgKrep6o7VHVHf3//crZxxaCq/r9FPhRF8VYxfR1R0o6LiLdvxPZuLeOS3JyU62ZuMFRLo73FTgCb8t5v9LcV2+e4iISAbmBsgWOPA8dV9cf+9t2UEC6GxYmlHXrbQozEMrlttngCJesqbRGbtohNyBYituUJGIT+zigw3yW5HmliTOqZyjGBr4Z60+iVy5PANhHZIiIRPAP9noJ99gDv8F/vAh5Xbzq9B7jB9ybbAmwDfqKqp4BjIvJq/5g3AT+vd0dWK53REGN5ggXAUbAs2NDTxt/8x8v45K5LGehsoTVsYYmwtiNMRzQ0LxV/PWbLZgZeHSbw1VBvGrpyUdWsiPwx8ChgA19Q1edF5A7ggKruwTPMf1VEDgNn8AQQ/n7fwBMcWeBdqhpE8v0J8He+wBoC/suydmwVMZ1I4xbZ7riw67INuVlu8G+wijh0eoq0o0RCVs7mUo/ZspmBV8em3jaGp5O58wYm8NVQWxqtFkNVHwIeKtj24bzXSeB3Sxz7ceDjRbb/FNhR04aepYzMZEp+9sTQGW4t2BYM6B/e8zzdttAatnOriXg6y/quljn7L3W2XKzkspmBL45JY2+oNw0XLoaVgeT+4xnsBUoO4KVWE+msSyLj1HS2bGbgi1PKJnUHcOfDL3BoOAbA1r6iTpUGQ1U02uZiqDNLdTedN+D4XmNhW0oO4KX0+RFbal4e2ZRcXpjFbFLxjMvG3la2DXSQdlxjrzLUDCNcVjG1MHbfds12ultDKLPuyLZAV2u45ABeKpHltnVdNS+PbEouL8xCyUfLTUxqMFSDUYutYmph7N65fYCb/t0W/vYHQ8QzDgic093CX/z2rxb9jr0Hh5mIpzk6FidsC+s6o4Rsi4yjXLF1DXc9cpCh0RkA2iP2vOOrwRSSKs1CNikFY68y1A2zclnFFKqnppMZXplI8JOjZ8pWke09OMzup09wTk8LF5/bxda+diyr+G0TrJTSjsvGnhZQOD6RIGwJuy7bwFf3/5JDwzFUFVXl8IiXg8yoYerHQuUQTKkEQz0xwmUVkz94TCcznJxIknGVlpBVtoqsEtVJ/r5drRG2revkvLXt9LZHeWLoDNPJLLYl2Jbl/YkQS2VrqoYxKU3mspBNytirDPXECJdVTP7gMTyVRH1rfF9HtGz9eiXBdsX2zTouT788zk+OniGZdXPpZMDLV+a4WjM1jAmonM9CNiljrzLUE2NzWcUE7qb37hvi6FiclpBFX0eULl/PXo5+Pd/VdzqZYWQ6RTLr0B4Jsffg8JyBqNAteDqZ4cREkpAttIQsZtKOF1iJYluCKthWaa+zSjEBlcVZyCZl7FWGemFWLqucndsHeGDwDbz+vDWs727JCRYoT78erH5GY0lOjCdIOy6WCG0Re96qoFDNcmoyCcC6zhb6OqLYlhcok3ZcHNfFUaUjGqqZGsakNKkdRr1oWCpGuJwlVKtfD1QnMykHxct2fG53K/2dLfPUaoVqFgU29HgCras1zMaeVqIh75YTEc7vb+eTuy6dkzpmKQOaMVDXBqNeNNQCoxY7S8hXkR0fj7OxguzBO7cP0NUaZvOaNiSvsEuxVUG+muXG+/YzPJ3MfdbVGiZkCwOdLTww+IY5xwUDWtiWOQPaHVC22sakNKkNRr1oqAVGuJxFLEW/Xk2alUoG+1rF5FQrQFc7lZQlWM58be998Gn2/OspHNezw113yXruvuGymv+OYfkxwsVQFtWsCioZ7IsNaIGn2ZV3Pb7ogJjLxjw8zXQiQ8pRjo0n+PGRMdoiFr+6ofesFDR7Dw57+cNGYoQti3Vd0UVXhcuVr+29Dz7Nt376Su6946r//mkjYFYBokVLDJ597NixQw8cONDoZtSMehblqseqIFChBQPaVCLDiYkEIUs4f6AjJ8yKucoGKrWM43BqMkWxO7qn1Yu9OZtcbfNtJ66riHgeeuf2tGBbxdWT+ceF/azWC537pbD1A/+Em3exBC91nW0Jv/gfv1Wz3zHUFxF5SlXnZaE3K5dVSC3sF8Wop9tqsDI6MR5nPJ7JCYiO6GzwZik1WaBSG4tliwoWgKlklrUd0bPKbhCcF8dVbBFEBBdlZDrFlr72kmqu5VAv7j04PEewQC4nKk7hB4YViREuK4xyViQrySB7z2Mvcf+PjjCTdhC80sn5TCQy6NgMm9e2Lxi82dMaJu0UK2vm4erZ55YcnJeIbZF1FBEvcDXtuIuqueod/7JQ8G7gsl4upsx1c2JckVcQ5bqIrpR4j3see4nPPH6YRMYhZM0XLAGTySxTiUzJATFwQY7YpW9nS84+t+TgvPR1RHFRXNf7s0Ua7kV3bDxOV7T49brukvVlf49xm25ejHBZQZSb56szGuLwcIyDp6YYGokxnSw9MDeS+390BEsgZFmoLjxbPT2dLDkgBjE8Xa0hSn1LV0uo4QNqtVQb/xOcl5AtnNvdgljgKGzpa2+47WlTbxv9Xa10RedOgta2hbj7hsvK7rMpG9C8NFy4iMg1IvKiiBwWkduLfB4Vka/7n/9YRM7L++wD/vYXReQ3C46zReQZEfn/lqEby0I5K5K9B4c5Ph4nmXXJOMpM2uHYmThTiUzTDawzaQfLzy+WWUClBV4tmVIDYhC8ed7aDtZ2RIjasyLGEuiMWrzmnO6GD6jVsJSZeX5Qq6vwuk29fP7tO3j4PVc1/DwEgq+/y8u2/ar+djavaeNTb31dRX1eKav0s5GG2lxExAY+C/wGcBx4UkT2qOrP83a7CRhX1fNF5AbgLuD3RORC4AbgIuBc4DERuUBVgxDtdwMvAF3L1J26U46L6J0Pv0Ai4xIScPFsDa5C1JaGDyiFtEc8TyTHXViwrGkL8+r1XQu2fzlyZDVCt79U+1mz5Q7LP4cdERsRYTKRmeM0cON9+8vusylz3bw0euXyeuCwqg6pahp4ELi+YJ/rgS/7r3cDbxIvTPx64EFVTanqEeCw/32IyEbgLcD9y9CHZaOcFC5HxuJYAuGQTTRk0xq2CdvCyEymgS0vzs1XbskJv1L+QWvawnS0lK56WS5LTS1T6QqiVrm5VtPMvPAcZlxvZf2x6y/mgcE35ARHJX02ZQOal0YLlw3Asbz3x/1tRfdR1SwwCaxd5NhPA/8Nb/JeEhEZFJEDInJgZGSkyi4sH6stRfqtV1/Au994fs47yBLo74jwK2vaaAlbREMWr17fVbKP5Q7gtTD6VqLbr6WReTXlSyv3HFbS59X2TKwmVp0rsoj878Cwqj4lIjsX2ldV7wPuAy+Isv6tWzqLqTm29rVzaDiGqOd6qv7KYFt/+zK2snxuvfoCLtnYMy9ob7FBopJYnmBQyzrKkckZ0o6LbQl3PvxC2YNQJSlRaukKvprypZV7Divtc7Op/gwejV65nAA25b3f6G8ruo+IhIBuYGyBY/8dcJ2IHMVTs71RRL5Wj8Y3I7dds53etrAXM+K4CNDbFua2a7aXdXwlq4FapWSvZvZZyUri2HicqUSal8/EmUk7ZB0l67gcGomV3e5KZtO1VGWtppl5uedwNfX5bKbRK5cngW0isgVPMNwA/MeCffYA7wCeAHYBj6uqisge4O9F5C/xDPrbgJ+o6hPABwD8lcv7VPVty9CXpmDn9gE+sevSqqKry10N1CMDQKWzz0pWEgKMxGZtTgpkXYj4wqic361kNl1rI/NqmJnvPTjM+EyKo2MzuRxnIdsqeQ5XQ5/Pdhq6cvFtKH8MPIrn2fUNVX1eRO4Qkev83T4PrBWRw8CfArf7xz4PfAP4OfAI8K48T7GzkmA18cFvPwcwz1C6GOWuBpohtqDcWfDeg8O8Mpko+h0CZa8mKplNGyPzXHK531xlY08rCByfSBKxLbMiWcU0euWCqj4EPFSw7cN5r5PA75Y49uPAxxf47r3A3lq0s9mpxWqi3NXAcqZkL0W5K4l79w2hzCZFzCfInHzJRx/l5iu3cOvVFyz4m+XOpnduH2DX8YlcWpv2iM3NV245awfRQhtUV2uEeDpLT1vkrD0nZwMNFy6GuVQbS7GYEbmc7y1XndMMsQXlJlc8Nh4nalukcckW8R0MW17bP/P4YYBFBUw57D04zO6nT9DfGWWzL/h2P32CSzb2nJWDaTNMRgox+cjqjxEuTUTh6uPIaIxbvvYUnS0htg10Vl3gqdxVTbmrgWbxYCpcSQRqwfwBY1NvG47rMhbLELLcXFwNQMgSQn4+sqzr8rf7hnhi6MySB5x8QT+VyDAaS5HKutz64DPcc8PrzrpBrBkmI/nUK2u4YS6mnotPM9Rzya9pMpXIcNK3FYQt4Zye1gVrahTWQwGIp7MMdLYAlPysWLnhcpwBytlvOWeHpWqQ7LpsA7ufPkHGcZiMZ0g5Lo4LtkAkZJN1XBzVnMBZ3xWlryO6pBomV971OD2tYaaTWU5OJrAQECXrKht62nLfWc35WYkz7uWqD1Psd4udq4WelWL1bZqVZrkXTD2XFUD+6mM0lsJCEAsyri4aK7HQauKD336ubLVEJXaFhfZb7tlhKbXgE0NnuOO6izxBaHmC8NnjE2Qcl6zjkinIxDw2kyYasulqDVcdmxLM1INraFmCq9ASsuY4PlR6flbqjLtW9WEqGUwXOlfNqKYrh/z+d0ZDjMRSdLeGm/ZeMMKlichXH6QdF1uErOOpcg6emiJiW0zG00WPXegB3rRv+dUSy11TZqEBo1AQBqn+XV+wBMb+kCVYCKOxFF2t4aoHnEDQp7IutuWnt1Ho74zmvrOa87OS6vQUslTX4koF60LnqtnUdOVQ2P/DwzGyrtIeCSGRhYvpNQojXJqI/NVHxLZIZBxc9QY9W4S045J1lb0Hh0tmBy62/Yqta/js3l/guEo0ZNHZEiISsnM2kvz68+msi6pXEjdsCxes66pqlrncs8NKBozAaH/3Y4cAr4BWyBIsEQRyRceqHXACQX/rg88wk87SErLo74zS2eKthjb2tlV8fvYeHObpl8dxVYnYFn0d0SUJwJVGpYJ1ofP7sesvrshm2Azqp8L+O6pY4mk4wPs37bgcH0+UHB+Wm0ZH6BvyyI+laA1bfj1xCNniu9MKa9rDFcWTBJ5La9rDRGwhmXUYj2fYddmGnN7/w3ue5+hYjMl4hpl0lumUQyyVZSqZ5chorKq8WMudE6vS2JJbr76AN2xdy6v627no3G429LSi6j20EdtaUmxKMBiFbSFkWXS1huiIhuZ852LnJz8DwjV3/4D3734WAT/zgnJyMrFgAbXVRqVZDxY6v5XELDVLMbLC/geF8ZJZl5OTCa/SKN790SzF0oxwaTJ2bh/ggcE3cOBDb2Zte4SobeG4SsgSetvCTMYz7B8a45KPPsqOv/jeoqlX7nrkIMNTSUZjaUSETb1tbOxt5YmhM8DsjGgqkfVsA767rqveIDYez1QVILncgYTVpAzJb2NHNMTajjCWCK1hq+qUI/mD0TndrfS2hTkzk+HlM3FGplPMpDLcu2+IK7auKXl+Cge0o2fijMczdLeGAQn+v2ABtdVGpZOVxe6/4Dn74W1vXDDQuBkChmF+//s7o7661RMqwX/Wd7c0TbE0oxZrYrYNdOZUPdPJDCcnkmTdWZVNIu1wdCxWUve89+AwLw3HsMWrS551lZMTSc7pjuZmfPn15wsDDRVIZT3Dd6Wql1oZcSv9zUq/vy1scWQsjusq4ZBFR9RmW5WqwL0Hh7n1wWeIpx2iIU911e97643HM2zsbaU1bDM8nWT30yfYddkGnhg6M+/8FNYzcVxPBRJLZTm3p4WRac+1WVSW5Hm2kqgmmWUt7r9mMf4X9t/2J5vj8TSuKlF7VvWqqk2hKjXCpYnJv6GGp5IoiqonKEKWhavKVCLL+u5QUd1zMOtS11OpiYCLcnoqxes29wKztgpPFTQ/e443Q07xuk29Fbe/mfND5RtI13dFOTGRJJ116e+IVOR5k2+vmk5mSWddIiHJqa4AppNZsq5b1JOtmOtr4YAWsS0yjkvacelsCedsNwOdLXNUmyvNi6wSqhEWtbj/msX4X6z/H3rLhdy7b6gp2lcMI1yalL0Hh7nrkYOcnIjj+N5G0ZCFZXnGZ/AM0WnHLTmTOjYeZ11nlJOTSXC9/R3Xi1R/6fQUN963nyu2rmH30yfoag0xU0S4WEJTGDtr/Tv56o6hkRi2eLqm0Viarf0dZXne5A/q8VQW148Zc1wlbFvgkgugjNpzNdCL2QvyB4y+jignJhKELEFV583ayzF2r4aVTSMmK80SMJxPvnahGdsXYIRLE7L34DDv3/0s4/EMlkhugM86LmHbQpVcrZbAq2xjb9u8AaQjYjORyGDhCaHgpgxbcE536zz1zOh0Gkc19/0AYdtiS1/7gsbOes+Yc4kPHYex6RTHxxM8MTTGpt5WPnb9xVX9Vv7qIHD7RmY9xcpRfQT2LEeVrKOEbcG2BMdVbEtBlFRWsS2hJWwzNBIj7bhEbM/If97ajqLfWzhghGyhpy1Mf0d0Xkngwr4E5Lc//zrZAs8cG+emrxzggoEObrtm+6oSQLWmEerdYpR81q67aDaOq4HtK4YRLk3IvfuGmE5msX33WADXdcgqWK63BHH92sBd7WEyjnLF1jXzbr7RWIqpRJaQLURCQirriZfetkjOOJmvnikVSV2qFsxyxV3cu2+IjOMwPJXCyZu2HRtP8P7dz/KJXZdWHejYFgkRsS2y/hcHXjiLqRaK2bPSjhK2vKDJkCUksy7tkRBv2t7Pnn89haqiQMZxmEk7vGHLmqLfXUoFUqqPi6lu8oulnZxMYuEJmSOjM7nJAFQe1Hm20Azq3YWetUoyny8nRrg0IcfG42RdN5f3CiBkW7j+6kMUsq4SsoXuljC3X/uaojdfKpMgbAth28oZ7G0LJhMZZtJObhYdBGZWOktbLmPnsfE4k/FMrma1+J4HimfPWGqFx76OCCcmkqBe+pdyPNsK7VlhyzvHGVdpj9is727JpTi5d98QnVGbyUQ2l6HZEnjoudNseewlHvrZKxwZ887Z1r723Gpioejz/BVGoNospRoJrtORyZlcxgAlUN/NehatlADNs3GF1SyOBZVghEsTsqm3jdHp1Bz1lKqXD8uyrJzXUSLjEM94Q26xmy/ruogIW/s99cvQSIxkxiHlKqBFAzMrmaVVa+zMHxw6IjYiwnQqu2C25lcmExSmwbPE62PVFR6ZFaTn97cjIsRSnqF8sQGr0J5lCYQsrwiZJTAyncoN3IeGp8k4XvyM5dvLFCWTdfmrfz6MqncMwKHh2IKrsWLqkYU8z4LzNzydnFX/MatSDQYohRUxeC2380KzCLJmcSyoBCNcGsBiN+wtV23N2VxUvBHVVRDxgijLTWkRsizf/92jvzPKL/0ZsljeAJMfmLmU2X+5xsRC/f/hkRkANvS0zBkogDl5lIIAMYWcRdMST/20mPqq1LleirojON/ndrcyGkuRzLqekADSjtLbFqKvI8rwdNLzFvPtZQHqZ2d2HM+NNBA6orrgamyhHGqlki4G18m2PHWqMJuOJn+AWgmD13KmwGkmL7xmNtyXwgRRLjOFAXJBWv38gMid271SxdsGOhARRITz+9tpDdtMxjMcPDXF0EiM6WQmN7ssFjTW2TIbGa7qGZZFIGpLLjDz3J4W1rZHq5/9X3cRYUs4NBzj+HiC9oi94DH5g8NoLI0tnhF8NJbOBajd+fALc85R2nFp9wVMQDDT72wJccXWNblo9uAc7j04zLWf3sdNXznAM8fGsYWaRlcH5zuVdbzMyq6XWdmywFVlbCbNdDJLWyREb1sYFy/6X1HcPKcJYXZ1iv96odVYpZHqMHudzlvThuMH3Z3THcW2JDdArZTqmdX0v1qaJYASqgsSbjRm5bLMFNb6GJvx7B3xVHbezCj/xtl7cJg/+OoBMr7hOes4HB9P0N8Z4by1HSWNwBRsi/j2l8J040uZocYz7hxV3UKzu3K8tA4Nx9jY2zpndtrXAed2t6Kqc+wT1168nt1Pn8jNLo+OxfiDrx7IGegtAXWFk5PeKiMYHJb6UO7c7lWb/OzeX5DK8zLIumCLErKsXALMvo4oM6kMaQcyjueW3N3uRe67rjtP/RmyrJLXo1r1SHA/5ZdKKFT/NYNX1GLUQj209+Awdz78QlE7Vz7NZudoBseCSmi4cBGRa4DPADZwv6reWfB5FPgKcDkwBvyeqh71P/sAcBPgALeq6qMissnffx2eAuU+Vf3MMnVnQQqTD2ZdLTut/l2PHMTJG8QUz6j/ymSKVyZTbPvvD3F+gWtpQKGQquXyulI1RTleWkDR2elkIsMPb3vjnO350ezTyQxjsQwZP8+S4mckhly24y197UseHIIB+umXx3Hd+eUtHQVLXdKOJzFGYylELDqiQsbx+jmTcrxcbypkHJdgXeYq9LaFS16PpapHFhqgVsLgtdT+7z04zPt2P8tEPLOonWsl2jmaiYaqxUTEBj4LXAtcCNwoIhcW7HYTMK6q5wN3A3f5x14I3ABcBFwD/I3/fVngv6rqhcAbgHcV+c5l557HXuKWrz1FKuv66hSXVNbNqUiCwbXYzGjvwWEOnpqe9ZYq8v0ZV3nx1DR/8NUDXP6x75bMOVbr5XWlaop89UtfRwRHFcdV+joiOVXM1r72svNI5f/+yHRqdgXArOrMc2zwVkdLHRzy1ZquatHSyQAZ17umI9NJRmJp2qM253S30h61mUk72JbnreW6nnty1vEyUW8b6FjQtXolqkdqyVL7f+++IWKprK+Otfw/ydm58mmkqjA/cWmpZ7mcfRpJo1curwcOq+oQgIg8CFwP/Dxvn+uBj/qvdwN/LSLib39QVVPAERE5DLxeVZ8AXgFQ1WkReQHYUPCdy8reg8N8du8v/BWLkHa8qoSCJxQiIvR3RoH5g2gwmAVrlsL8XzAbUOkCrqOcmcnwzMvjJb2OajlDrXR2V46XFlD27HReDRxLcsI3ZHlpU7zSxjrHxlAt+Ss1Ly3L/KwGAZYo8bTDQGeEvg4vx9hUIgsoE/EskZBFOGT5ucOEe992eVnXZSWsMOrJUvp/bDzuBbnmGbpEKJo/r1EBlIWOBM8eG+f3v/Qk4MVUXXfJeq5/7camcTYoRaOFywbgWN7748CvldpHVbMiMgms9bfvLzh2Q/6BInIe8Drgx8V+XEQGgUGAzZs3V9uHRfGCAN15rrTB28CAG7K8WiuBrSQ4NmwLYcubDZdTlFr97xyPZ7jrkYN1vdmqUVOUMziU+1AHvz8yncRxNWeTAnJxPY7rGdO3rmnn9mtfs6TzEejhp5MZsk6JZQsQsYWLN/TO09unHRfXv46q5IQf6JKuVbO4zDY7m3rbGI2lvPikMuxcjRDk+ROYX47GcuEG4K12v/XTV9j30gg97dGmjktqtHCpGyLSAfy/wHtUdarYPqp6H3AfwI4dO8oZt6vipdNTqJYWDG7esqRwn2PjcWwBLbpm8SmyOTAsD43OVNfoMql0dlfuIFjsoS51bGBYD+aitu+37KqLImxfX9wWVQ2bets4OhZjLJZBRLBQ8kVMyPfIO6e7heN+O/NXdsFqR/AES4AALw3Hqir01Ewus8tFtcL0lqu25mwu6gbOMYptC4eGp7nxvv0NF8z5E5Kp1NyVcaClGItnOadnrjBstrikRrsinwA25b3f6G8ruo+IhIBuPMN+yWNFJIwnWP5OVf+hLi2vgIyjc9xNS7FtoJPu1rnFwDb1tnF6KoVtiReEV+R7FhVadWbn9vJqYyyl8NJCxz4xdIaNva1ceG43561to8W3wbRGQnz+7Tt4+D1X1WywuOWqrZyZyaC+uM+PXRG85KLndrcSsq2coM3X23e1ekJG8/7AE4jVurk2k8vscrCU+2jn9gE+uetSzu9vR9WLSXLxrl17xK5bMbBK7CNB7ZapRGbB71zOYnzV0Gjh8iSwTUS2iEgEz0C/p2CfPcA7/Ne7gMdVVf3tN4hIVES2ANuAn/j2mM8DL6jqXy5LL1j45omELGwRQgucbVdhKpGZN/u45aqtZFyv9LAlELYsb3ZcRpsU2LK2eW62pQyCCx2bb9TvbAnT3xklalvMpB3u3TdU04Fi5/YBOqI2EdvCUS8FT39HhLAtWBZs6WsnZM/adgoN0Oet7eB3XntO7vssgZCAWMK6zurijY6Nx8k6LkMjMX7+yhTPn5zkyMgMT7883nRG3lqwVGG6c/sAt1/7Gs7paaUlbBGxvViyVyZT81LiVEqxcaBSYXjLVVuZTGQ4MZGY/2EugJimj0tqqFrMt6H8MfAonivyF1T1eRG5AzigqnvwBMVXfYP9GTwBhL/fN/AM9VngXarqiMiVwH8GfiYiP/V/6r+r6kP16kd+1t7JeIaTE17WXku8ma1tCR0tNvGUQ3YBq8nJyQRrsxG29M1my925fYBt/R0cPeMZIiO2BY4ituQ8zOJpp+i3hm3h9mtfU+vuVs1S4gYWOjZf9RQUVVOUlpDF8HSS9+9+lrXtEWJppyb2iAvWdc1zYrAtIZ52imYtLqbie+GVH8y5pn0dUUK2MOAXF6uEjoidy3Tg5C1XXdddleqxWsSfBAIqMO6LCC7KyHT17uql1JNtYasid/1/PT7BmZl0Uc1DsOn6S8/h+tdubOq4pLKEi4i0AwlVdUXkAmA78LCqLrxuKwN/0H+oYNuH814ngd8tcezHgY8XbPsRxb1160aQtXcslsFVzd0Urvr1UxyIp31bQAkEb2A4PZ0i62qu1soTQ2cYnUmjCmvbI/R1RHnh1BSWCO0Rm5FYeo5gsWRWHfYnv35+U6VTX0rcwELHBkb90ViSU5OpXHLInlYbx/VKNU+nspzf31ETe0QxJ4ZIyObO/3BJ2d95+7WvKZqBupqZp/g6V6dwNBKpWdBoM1GL+JNAQAWxViIs2V29VMzXkbE42wbmllcoJQzveewlPvP44QVV2r/z2nO4+4bLgPLu4UY9++WqxfYBLSKyAfgu3srgS/Vq1Erj2HicM7E0GddLAplP1p1N8+FqaamXC/hTmIinOXB0jE9//xBHx2Ks72phTXuY8XiGU5MJ2iMhOqI2E4nsvC+0RGgJWbxmfSe3Xn0BsDQddS1ZStzAYseqegGl+TaMiUSGVyYSWOINvMVUKNXECtQi1qSW8SrTqSxr2sLzVq9BDaBmMvJWSrHrU4v4k8Cu0dcRxcWLN3L9VUy1Qv6l01Ocmkzm0jMFam4o3z5y/4+OYIk3ScxPDWQJvKq/nSu2rs0JlnIIakM98/I4pyYTuRCF5Xj2y1WLiarGReQm4G9U9X/mqZzOejqjIY77UeHFCALtQpY3ywyyEQeuySHfzThAgKwvaMZnMvR1tNDX0UJbJJSLBbnla0+hKCGBjP89tnjeSgNdLXNqsCxnsr+FWErcQKljAd8VOUXEt3Uo+J5c3nkO56kQwU8xc3qKaz+9j5eGY4Rtz95RyaqmFi6qtXJz3dTbxjMvj8/efxIkJa2+RHUzUM8CWcHqM2wL53a3eBoDxxvAq/Es3HtwmFjK8eKpZLbM9dpshK197cyknbLc9WfSjjdO+DFa5GlBqhF6dz1ykPF4xiuN7hcaXI4QBahAuIjIFcB/wouYB89GYsCbNcPiMSghPxo463i5p4J8WpmCcIn896m8EPBgKR0YlZMZr6561MYrIOYXoyqcATdTjqSlDKjFjr3xvv1kHGfOeQL8FaR3PtKO0t06e7uOxlJMpxxiqRnPzdul4txjzaBmDLjlqq3c9JUnsf2U/4VG36UaeRvV13oWyCqcrLxuU29ODf3Bbz/Hpn2V9fPefUOsaQ8zFst4kxsLcL2B/M7/cElun8WEYXvEEz5eKXOLrJ97zrakqpXt0OiMpyp3NVdlFuDQ8HRF31MN5QqX9wAfAL7lG9K3Av9ct1atMIp6dRQhYgtt0VAuWaUts7mvyiF/KV3MqBxPe1HuhTfgas6RdGh4msn4fNNfLnRIvOJq4/EMbRGbkG0xHs+wpj3sZWW2BEFy9e7LMeY2W1xJvtOHornJzkIlqsulkX2t96Qof7Ky1H4eG4+ztj1KNGQzMp3yVsyW97znO3Ysxs1XbuEzjx8m67pYfpVTUXj3G8+v+nw7rs6p4AreJKSamKpKKMvmoqo/UNXrgM+JSKeqDqnqrXVr1QrinsdeYjpVOgUIeOqJiC1s6G2jpzWMqpf+xRLJVSUsRdiWonrlQr3zyHSS4+OJXCBYvk612dOpB3r1yz/2XS756KNzyg8sRjrrgsyNNwkI28Lm3jY2r20jZAmnplIMdLbQ2RJibXuUiK8mgMqMuc0YV3L7ta9hoLOF89a2ceE5XWzpa5+nHq3GvtTIvgZ2kXzqNSlaaj+Dtna2hNna38H29V2c09PKtoHOitpx69UX8O43nk9r2CbresL03W88P2c/rZQta9tyzgFBTSTwxqN6X8OyhIuI7BCRnwH/CjwnIs+KyOV1bdkK4XM/+EVZ+/W0hhmJpRibSXvqMX9W4lLa0N8ZtXlVf0dRg2++QfjUZCI3G1/f1TLPYN/MyQ6DGeOR0RhTySyJjOfOfXQsVpbTQdh3wRPwVQmznNvdSldrmM6WMOcPdDDQGeWBwTewbaCTRMahvzPqF+3SinKPLWdNkXJZ7BpX69TRyL4u56Roqf1calvzBf8TQ2e454bX8Yv/8Vv860d/syrBEnzf6Mxcb1J8u2yQQaKelKsW+wLwTlX9IYAfS/JF4JJ6NWwlsPfgMIlCg0kRFBiOpf0gSpljXMsnP8GLAH9142VFBUChDry/s4XuNndBg32zJjsMZoxjsWyuvrurylQiy/ru0KL2jwvWdXFkNMZ0MkvaUdoiNumsg21bdOWpVPJnvPnG3HO6o5yeSpGtIPdYs6oZF7rG1Tp1NLKvy5k4cqn9LNbWK7au4d59Q54NZ4G211r1GJQViKWynpck+E4unsDs7/QKxVUTU1UJ5QoXJxAs4MWSiEi2Tm1aMdz1yMGK9g/yfVniLRjzU8IUZj1WyC1b82+wwLVwOpkl67qM+vrdTb2tc36r0TPpcgn06vn13QMVVTl9CATF+u5QzhNnKuEZVUt558wz5m7urWjQWoklZ6u1XzS6r8s1KapFP6u14SzFm7OYs8WdD7/ARDzjlRUQQS3PwcUWL4PEcl3DBYWLiAQO1T8QkXuBB/DGvd8D9ta1ZSuAapJC5ofBzAt6Y1bIRCyK3pDFXAtV4ZXJJF2tkdz3NMNMuhyCGWN+MFtQ36bcCovlVOAsFB5L9VpbCVUb81lKBcuV1tdqqHU/KxEY1Qr+UgLs5EQcSzwtQMaZjb3LuPDLsRgXb/AmU/f+4HAulT/AFVt6eeCWf1tVf4ux2MrlUwXvP5L3epnSIjYvmVKVogoQfzkSrEhc1xtEC7Pi5p/Qc3vbit6QgWuhJUHlwll320Onp1nXFSVkW00/kw4IZoydLZ4Xneu7z3W1h8vuQylBUc8BsFnVjKVYysw86GcwQy62ol4N1PKaViIwqhX8pQSYo4Eb+vyg7nhGPXXdDw7zxJHxOZ89cWScG+/9l5oJmAWFi6r+ek1+ZRWy9+Aw5YkWcoLFtjyhIH4VwqBa4tr2CPG0w0x61jNmZDoFQEc0VHIG49UvmRt9eXwiyQUDHXzoLbVJMV9v8meMWWeKtKNEQhbnre1Y8gy52viMZophqRVLmZk3m+v1SqASgVGt4C8lwEIiuMo8wRJ4pt7/oyNMJYtbNQoFzlIoN7dYN96q5Sp/0w+AO1R1smYtWWFUYm8JLnF72Obm/20rTwyd4fh4nMmEF3sRDdnzLvZM2iE+Fqe3Lcyr13fltm9Z28bhkRnEFyzBd0dtYdtAJ/F0lp62yIp66At11eXMkBcTANUOiKt5IC12nhczNkPzZHhYSQR1Y05MJLzkmJbQEZ1bCDCgWsFfSoCdP9DBaCzFSCw9Z/+QJVgC0yUES62pxFvsOeCt/vv/jOct9h/q0aiVQKX2logtRMM2l2zsybkW3vPYS9zz+KGSddgVOBPPcMXWNbltt1/7mpwniDqz+YfWd3sG/ZViyC9GuQN7OftVMiDmC6qpRIb2qE13a8uix61Uip2/d3/9GdJZl7SjtEdsbr5yS+4+baYMD81M/n3UEbFz6VtUFXThMhnVqORKrXg+9BYvtummrxzwS2jjlfywvYh/y5Ki9t5aU65weZWq/p957//8bM8tVq69pb8jkhv4g0EKvJXPwVPTZRmuHvrZK7kHfed2r9jRvfuGePrlcURgXWdLzu12pRjyi1GuQChnv3IHxMKB9tRkkkTaIRqy6WwJFz2uWdVm9zz2Evf/6AgzaWeegMin8PzFklkmE1kv2DckJDIOn3n8MOAF9TXa9bpZz3c+hffR4ZEYWUfZ2Nuau49qPUlZbMXz7jeez2ceP5xLhJl1vZLa7ZH5mpJ6UG5W5IQf2wKAiPw7oLycJ6uUcu0tI7E0pye9U9Uatjk0PM2H9zzP4eHyBAvAL0bmrpJ2bvcqP977tssZ6GwhVCKKf6VRLJAt67g8/fL4nKjy/P2mEhmGRmK8fCY+pzhWudHdhZHZ0ZAFMmvzKjyuWTJMFxKkavfyUpETEPc89tK8fQvP80hstq+WWIQsK6ebh8ZmeGjW811I4X0UrBjy76N6rPaCsaBYFdhS0f4Xndtd0zaUotyVyx8CX/FtLwDjzFaHNCzCSCxNWzREKuswHs8wHk/jlO0NMN8wF7DS3EQL1QYiwnQqm5uNFs6QpxIZTkwkCVlz1V8dfnK/IPOs5WUHQyCnHivXSFq4wunriHJyMkEy66Cq845rVvtDkKo9ZHnzxWCmev+PjsxbvRSe51x6kDy9jSXkHEwaeZ816/kupPA+itheVuN03oNeOLlZjhXZrVdfMO/6X3JwmCeGxmr6O8UoS7io6rPApSLS5b+fyv9cRN6hql+uQ/tWBYp38wX3mSykfC2CZUnJG3GluMTmqw1sIVc5cUPPbLqaXZdtYPfTJ3IC4fR0EoD13S25fE/xdBYRz39/eDrp6bEFUGF9dwu2JbmsueUMiIUDbVdrmFTWKVlVslntD0Gq9nzyBUQ+hYI3cIO3827MQH0S0Kj7rFnPdyGF91FfR5QTEwkEZWgkRirrYlvC9ZeeCzTWcWS5rmO5ajHAEyqFgsXn3TVqz6rFcWcTxxWmfVkMC10RqoGFyJ+BjsbSXvSwJYzG0rkkgU8MnZmTH0vVEz6Bzhq8gSWWynLHdRfl8oKFLOFcf7/8gWchlUFAMZVPJGRzzw2vK3rcciZTrIT2iD2vemGhgAgozEO2sbc1J6RddXO6+Zuv3LIsbV+IZj3fhRTeRyFbaA1bIEIy6xCxhTXtYXY/fSI3UWy25Ke1ply12GJUXVZYRK4BPoNXH+Z+Vb2z4PMo8BXgcmAM+D1VPep/9gG8+jIOcKuqPlrOdzaKoPxuucxWopMVoRpYiPwZaC7Vi5/mBebWqgn6dON9+xn2Vy8BwcCyc/sAl23uXbKhuVKVz1KCEeupBilM1e4qCwqIwpVIuc4Ay02j08/ks9D1K3YfRWyvZlNhWYzgO6qNym9254aAWgmXqvzaRMQGPgv8BnAceFJE9qjqz/N2uwkYV9XzReQG4C7g90TkQuAG4CLgXOAxEQmehsW+s2EUnqggkW8kZKGuknG9gj6WJYSEXPT9KxMJBrpaSnowNTv5aoMg1QuQqxBZTCgsNrDUauCpROVTrf2h3mqQQBBUKyCK6eabgWaxK5Zz/QrvoyvverykAKnGA2+lxWBVpBZbgGpXLq8HDvv1YdLAg8D1BftcDwT2nN3Am0RE/O0PqmpKVY8Ah/3vK+c75/HLX/6S73znOwBks1kGBwd56KGHAEgmkwwODvLd734XgFgsRveBLxIZ9uSVpGe89yMveu9T03Qf+CLh0UMAWMlJ7/2Yl57fip+h+8AXCY0f5T1v2sadv3EO/T/9Mp3xE4Rti3WMs/aZL8HECVyF1vhpIvs/z6mXjzCdzJAefZmRR/6Kfp0A4Nlnn2VwcJCjR48C8NRTTzE4OMjx48cB+PGPf8zg4CCnTp0C4F/+5V8YHBxkdHQUgH379jE4OMg/PXmIaz+9j4tu/iSv/Y1dvPmuR9h7cJjvfve7DA4Okkx6q4iHHnqIwcFBslnPnfE73/kOg4ODuXP5rW99i3e+852599/85jex9n8xpzboPvkTWp7+Go6r9HVEGH/2MWL7vpQTCl/60pf4wAc+kFPf2If+mZG9X8mlkT+49x/48z//89zn1guPMvLDB3Kf//SRv+euu+7K/f6nPvUpPvWp2UxGd911F5/+9Kdz7z/+8Y/z13/917n3f/7nf87tf/GpXAr0X7/hj/iz/3t2/0e/fDe/GTmYU5v90+c/wVe/+tXc5+9973t54IEHcu9vvfVW7vrcl3KrzzPf+xwcfTKnBhkcHKzo3hscHOTxxx8HYGJigsHBQfbt28etV1/A4398OW8a/Uf+9s2d3Hr1BZw6dYrBwUF+/OMfA3D8+HEGBwd56qmnADh69CiDg4M8++yzABw+fJjBwUGef/55AF588UUGBwd58UXv3n7++ecZHBzk8OHDNb33JiYmAHj88ccZHBwkFosB8N3vfpe//8sP8sW3v5Yf3vZG/vPGM/z9X36wonvv1ltnS0898MADvPe97829/+pXv8r73//+3Pvg3gu4//77+dCHPpRTYznPf5fJ//VATo318U/ezcc//vHc/p/+9Kdz996m3jbO7P8HJn/yD7nPzzzxTaIvPpJTo43+6AGmnvpOzgOv+4V/5G//9m9z+3/oQx/i/vvvBzwBm9n/d7gv7s2p0tL/8hXu/Kv7cvsXu/e++c1v5t6/853v5Fvf+lbuffeBLxI9+Yz3xnXoPvDFiu+9UtRKuPyvKo/bABzLe3/c31Z0H1XNApPA2gWOLec7ARCRQRE5ICIHMpn51QzrTVfU5uHnTvH1A8dIZl029rbzrp2v4tyeNtJZF9uCjb2trG33ElIqyvBUkmQmiyr87uUba9aWiXiGj+x5nkPDsdzqamg0zvt2P8vPTy49EcPajmhOz694toBtAx246qW4ufDcrpL5wd66YxNvvmh9UbvJzu0DXP/aDVz32g1LLn0b8MpEgr0vjeRsXMmMw+MHh5dk4xqPp5uuBoyhfErVe1ko2v2Wq7biqpJ1NGfPc13l0o09uYlRa9gmlXVzE6NzelpLft+x8Th2Qc0i2xImErNj11gszZf/5WjOdX8sz818uREtw7osIuuA/wGcq6rX+iqpK1T180v6cZFdwDWqerP//j8Dv6aqf5y3z3P+Psf9978Afg34KLBfVb/mb/888LB/2ILfWYwdO3bogQMHym77ebf/U9n7FsPLKRbmTDzLeWvb5qh17rjuIj747efoaQ0jvgfPVCLDaCxFMuvy+vPW1Fw1cON9+3nm5XE/35mfFNNVxILXberlgcE31Oy3StEs+uTA1lOshHS156Ee31krmuW8NzPVXr/g3NZCpbdYG/LVZoXjCTDnGi/kinz0zrdU1C4ReUpVdxRuL9fm8iW8dC9/5r9/Cfg6sCThApwANuW93+hvK7bPcREJAd14hv2Fjl3sOxuOACMxb8ZxajJJX0eUrtZwzuBXzEU2ZMuSBqOFBpFj43Gyrksor1ywiJccsx6z68K2XLF1DbufPtEwfXJ+e0amU6zvis75PFhlFDN8X7KxZ9HBuZkM0/ksJQfb2SSQqr1+tXThDtowMp1kOpmd595cKiborkcOMpN25lzj5aBctVifqn4DPzDdV08tXDi+PJ4EtonIFhGJ4Bno9xTss4fZgM1dwOPqLbf2ADeISFREtgDbgJ+U+Z0NxQKcvLrWQTDgVCKTi0h/6fQUL5+J8+KpKV54ZZJDp6eZSmSqHowWi3Te1NtGyJpbHVPVW3bX2u2zWFs+u/cXZBynIa6Zhe0RgRMTSaaTs+qGwB22MAr+7scO8e6vP7Oom3ih+2+zlJquxiV2pUTN15J6Xb/88sZBBoqF2rDrsg2MxzNF3ZtLqe6GRmfmXePloNxfmRGRtfjOTiLyBjzbx5JQ1ayI/DHwKJ7b8BdU9XkRuQM4oKp78FZHXxWRw8AZPGGBv983gJ8DWeBdqur47Zv3nUttay3JD85XvFgNS4RTU0kc14vb6IiGGJ/xKipalue2W6jArGT2uFik8y1XbeX9u59lPJ5BxfslV6EnGq757LpYWxxXmYxn6OuYLb26XDaJwvas62zhxESCU5NJOqKh3Cx1NJaaHwXvOEwlsmxe057rSyk38WYMeK3GJXalRM1XQjnPUq2vXzWrxieGzrCxt7Woe3MpDzRgntBZDspdufwp3uz/VSLyv/DiTv6kFg1Q1YdU9QJVfZWqftzf9mFfsKCqSVX9XVU9X1Vfr6pDecd+3D/u1ar68ELf2cxkXMVRl5SfDHN9dwujsTThkEXEtmgJWWwb6KS7NZybUVY6eyw1q8kPOPzErkvZNtCBiCAinN/fzid3XVrzAaNYW6Ihi1RBTpzlCpbLb8900rNtua6SclxOTSVzs9S0oxTYU3MF4PJZSYb6aoIUF7uXFqKSmfpy0aiVWDWrxoXOfX4g51QizaHT0xwd84oLjs3MNeyXcu9d2xYu8UnllJv+5WkR+ffAq/12vaiqy+9etYrJup6NI4hIPzGRyHmGFAYaQuWzx85oiJdOT5Px9XHRkEVve5jz1nbk9lmumXWxGVZnS4hsXBtikwja47jKyYkkIhCyvTKxbZFQbhbb7uc0yxcwxR7SZowgL0UpW8IVW9dw4337i87ky43RaDa7WikatRKrZtW40LkPYoLueuQgR8fihG1hY08rqazL8LRX22Vte9RT69riebHlfbcAA10t1IqyVi4i8nbgP+JFyV8G3OhvM9SQNa1h0o7LkJ+uO5X1ypQWCzQsZ/YYzBJ3/MX3ePH0NGn/ZlIgmXU5PZWaUytmuSiVcuVdO19VVKdd79lu0J5Tk0mCsqGKpx7Ln0nefOUWv8Kfm0uTAtDVGmpIxuByuOexl7jko4/yqv/+EJd89NF5WZKL2RKCHG+lZvKlsiQHAunKux7nmrt/wPt3PzvPrpbONsauthBLWYkthWpWjYtlqN65fYCetgjnrW1j20AnXa0R+jtb6O+IMJNycte4uzXM5jWttEdswrbQHrHZvKaVWKp2qfjLtbn8m7zXLcCbgKfx1GOGCgjbkls9FDKVzJKJ51vUIatKb8SadxMtNnvM1+fGU9nZzLd5v2eLp8O9leVloajrwrYsR1Ry0J5bvvYUineN+jq8Gjmqs95yRaPgf33WW6zZMlMHafg9O9FsGn5gTjR+4Yr1xvv2LziTL3b9ClclQT2T9uisIMm6LtPJLP2ds21sBhVio+rVVOOBVk7GgvwV0XQyw8h0ilTWwbYsPnb9xezcPpBza97aP6u5CNyaa0W5arE59hUR6cGLfDdUSKmlooVnewlULq7O1ryeSjqcP9CSu+luvG8/h4anmU5m6W0L09cRnaPOuPbT+zh4ehpVaAlZc1L2i0A0ZKOqOFqem/Heg8Pc9cjBXPXNLWvbuP3a1yxpAC1XBbdcKotyc5WVSpPSDMKkkErS8OdTSl1z6PRUSVVZoUDKr2cSpCyK2lbOrhjQDCrERrmJV5vaZqFn557HXuKVySTHxxNY4nl8RkIWlggilZekWArV+qTNAI1PmdpA/vTqbfzlY4cqOkbw8oQVI3jkcuVIHZesam7FEVz0YBa/vquFsJ3izIznvrxtXRdXbF3DV/f/0vP48o9LZt3ZbMzMZmQu181478HhnBdZIPgOj8zwvt3P1sXYX0i9U67n2wU6oyEm/WjnWj5wjYoJqSQNfz7FZvKjsRTTKWeeqixYQZZTz6S7LcyZmUzTxfrUM3/ZYte+Gjtnqe8MVqqBi0kwdmSyLrYtrOtsIe043PrgM3S1hnM1lYqVlqgFZQkXEfkOs04xFnAh8I2atWIFcuvVF1QsXMrJ7plxFVcd8mVQUAQr0I/O1oxoySWCBPxYETdX1jRY/RTi+CnVy3EzvnffENPJLLYluch9cZVYanlcT+upsihUuSUyDgKErdo9cI1MNhg4IKgqWddFfVtS2yJuqcVmtePxDGvawyVXkKXqmYQsyRVdC9s279q5mSeGzjSdCrEeziz1uPZ7Dw7zvt3PEktlcVzPPT6Y6M2uVG0c0Zxgd4FN3a2IwOh0GgU2r2nzhbubU5XVmnJdkT8JfMr/+7+Bq1T19pq3ZpUSsSvL65kfYCl4rslhWxganSlaBvil4Zjv7eT6Bmdykiww4Fvi2RJsi4rcjIPI/fwCZ/WM3C+kniV2i7mCdrWG6W2P8rHrLwbgg99+bklOBI2s23HzlVtwXG+QcXV2chOyWDRYr9DI39kSYm178awFULyeSU9bmC197XMcNG69+oJFa+ysFupx7e98+AUm4hnU9bQc6np5Ae98+AVm0rOejN5kcPa4rtawV3JZPE/R5bgXy7W5/KAuv36WUGzFIgt8ls9AZ5TOFs+wDN6svS0Syhnqghsq6yjRkI2TdvxIVwiLkHU9D7Fo2GZzbyuxtFO2ambvwWEva4ALjrqELQvbkrpF7hejniqLkraF4ek5VTOfeXmcm77yJNv6Oyq2NTWykuKtV1/ANw4c4+RkEle9CUZfe4ROP15qoX4UM/IH7toj06lcTZ6+jkjODlOoZvnQWy5c8cJjKSrNelz7I2NxTzPhSw4RUFc5Mhaf5yofsrx6Mp7tRUlmHSwR+jpmJwn1vBcXFC4iMk3psVFVtasurVpFeEZ5IWypX55XfY8ki4zrzjm7gV0keP0ra9tyxtDRWApbhKNjcWxfmNj+ikiAk5MJelvDOfdlV70Zqm0JbWGLSNgm42rZy/NgSd8WsZlJZ3FcL94m5EuuekTulyIY6IIH/YPffo5N+5YuZEqp3NJZl+5WwXGVVyZTiHizxKNnvCzR/R1RplPZsgabRnki5RDhwnO6cglQgTkecOWSn8HBTxhBOutycjKJo5qLn5hMpOnviHJsPJ6bEa9UAbNUtVbhtZ9KZDg9nUTVE9aV3r97Dw7POkQ4ngo3ZElOq1BYMC7QWJzbFWUykaE9EqItYtOVJ/DqeS8uqBZT1U5V7Sry12kES3l0RG22rG0DkVyEt2156fMLT37OqOXfGLavrx6ZTjISS9PdFmJjTwsZR3HxLl7U9hwALISZtMPG3lYitnfDBeqvc3ta6W4NV7Q8D5b0/Z0tbOxpo8W3DDtK3SL3F6IeUdSlVG5BVtmRaU+wWOLZm9JZl4l4hiOjM2W3YV7U9PA0R8fiTMTTyxKdXqsywTu3D7C2PULIltzkKBzyVrJTiSwiwnQiw2gszQunpjkTS3F0LMb7dj/LtZ/e11QR+eWyVLVW4bU/MZEg6yjru6IV37+BrSUfxbPRZhxla187t159Ae9+4/m0hm2yrrcqec+btvGjD1zND297I/fc8DoiIXvZYrIq8hYTkQG8OBcAVPXlmrdolZHMumRc5dXrOhmb8by7IjaIeIP1VJF6EJ4A8jIgHx+PE087tIYtTk+lch4gtkDItujvjHJywgv+SzuKbQnru1vnJNVbqCJeKfKX9F2t4VzMx2QiwyPv/fc1ODOVUQ+X5FIqt3v3DTE8nfRUP76OIfCyswQc1dxgs1gbgt+48+EXODqWIGxZbOxpIe24y2LYr6XLaSztcH5/R24VdPDUFJZfqno6mWEkls7tm8i6xCe9lCNTiQwXrOtsmoj8clmqWiv//nr65XFClrC+2xs+T00mSWYcbvrKAXpaQ1ywrmvR3ICxVBZboNDhVIFrL14PLFxRdLmrepbrLXYdnjH/XGAY+BXgBbwSw4aFUM0NiBHbRiRD2oHLNnczPpMikY6Rmev6T9aF7evac6n1L/noo8RSzhyXYkd9l9LpFK0hYTrtGW1HplPcfOWWOTdMNaqZhqtzCigVGHZ8PMHeg8NLEjDFjv3wnuexRchk5xrDHVdpyXOqKGew2bl9gHv3DTGQzDCVyHJyMknEtuhqDVUkHKvR/9dyQCm8J4Ia8RHbYmQ6NVd/nvcm7XjehZ0t4brEKNWLWjwDwf0VTPBiqSwnJ5Io6gkJVaaSWY6MxhYUvMfG4zjubGhCPlZeMHQ9XJ+rpVxvsY8BbwBeUtUteBH6++vWqlVEcDNMJTKcnEzguorjugxPJzk0EqOnLZzTYcOsh9jt176GvQeHuebuH+RWN8WSJKayDpMpBxR+ZU0r/Z3RXArugEo9rvYeHGZ8JsXRsRk/1X+66DHLmYQwUO9MJzOcnEiSdXVOYFgtfzvwlurriOD4giVsedfF8StnBpQ72BwanmZ0Ok3W9VaXWVcZnfbUZOVQC7VgOa7wC1F4H3W2hHDVS3+Tyjpz3d4LvAtHpr1VTDNE5JdLLT0Vg/t3ZDqFqzonS0fGUU5NpRieTnLnwy+UPN62PJWkp/KeHSuCc9pspRDKFS4ZVR0DLBGxVPWfgXmVxwzzCYTLaCyFhZdtOBqyPf2tZTGd8ry32vwYlmjY4tXrvJQM79v9LIdHZhb8/iBGLRKy6GqNFNULV1KLIrhBM66ysacVBI5PeDPt/GNqeSOXI6TKzf9VK3ZuH2DTmnbO6Y7S7ukxifjeY5OJTMWDTTrrgm+/cRwl43jq0vF4pqxzVq3+v5bXqfA+2tLXwbvfeD49rRGcub4pc+oChS3JxVw0Q0R+udSyhktw/yYyzpyMGflksi6HRmIl7/9gUqNKLmbJsoSw7Xno3fK1pxieTpJ1dNnd3otRrs1lQkQ6gB8Cfyciw3hR+mc1fe1hRmcWTg6teDl70o4XKY8K/Z2eK+C6rijHJ5KEbGFLX3tOH377ta/J07EK2RJzzsA2E9gBAorNDqtNt9LVGiGeztLTFplzfK1sIOV65JSb/6uWHBuPs7Y9SjRk57nfeue60iBLV12yjpJxCqLjVcuyQyyk/19IFVJrW1XhfbT34DC7nz7Buq4oYzPpeXnzAueUsCVNl9SzHGqlRgru39//0pMl93GBqGWVrAX0yV2X8sF//BnHJ7xKktGQRWvYm6AOdHrPqSXCyckE4NlKG7lSLFe4/DNeeeF3A2/zX99Rr0atFD75u69d8GYBLxraM8wncgGRgXtxyLa4YKCDnrbIHH34vx6fWLDGdUB/R5RYKpvTewcsZXZYrhGz2sGukEoGv3Lzf9WKTb1tHB2LMRbLeO7IluC43uqjkqjmvQeHyTh+TELe2GsLtITt3OxyMZfmo2MxphKz1zsaskhmXG76ypOELYt1eV5IpdKyQG1VU8H1625tIRqyOTWZIOUoAmxe46V7H49naIuGGOhsaZqI/Ebwr8cnFvxc1Z9wlrg2O7cP8KPb35R7vo6Px/0VVYS+jhamElmyriLqaUq6/KwTjVoplitcQsB38SpBfh34uq8mO6sp5yEJKiZ0RG1iKYdU1qFDZ6sbfugt2+d8zz2PvcTdZaSVWdMWpr8zimXB8HSazpZQLs3GUmaH5RoxS+WfmkpkuekrBwjbwrrO+YNdIZUOfsuZZPCWq7b6KyXFwouGBuhtWzwIMZ979w2xpj3MWCyTUw8Jnsq0vzM6p7+lBPMVW9fwk6Nncql9klmHmbSD7evfU47LL88kaPHr9JRKywJe4aiZlMOVdz2+5FxnxbwKpxJpTvmejVv6OrizDgKlUfnalsLf/mBh9VRLyCJkW4tmJs5fTeV7ghZ6jjZ6pViWzUVV/1xVLwLeBZwD/EBEHqtry1YBlkAi43JkNMY53a30+on78qsbFj4Q9//oyILf2d8R4U+v3sar13cxmchw3lpP772lr6Mmtb3LNWIW7hfE4mRdT3WkLpyc9PS/C+l9K43DqKUefDF2bh+gI2oTsS0cP6XJud2t9HWUnl0WI1CvndvTkoueDlZCnS2zs8uF7CNPDJ2hvyNCxLY87zUN3KI978L8RKWnJlMcOHqGK+96nPGZFFOJTO46jcaSDE+naYvYNTH6Frt+Idviss29dUvx0myG63KJZxZOGJrMuvxiZIZjZ2bK7kv++e9sCfv3mBeXVc9noxwqzYo8DJwCxoAltVhE1uCtgs4DjgJvVdXxIvu9A/ig//YvVPXL/vbLgS8BrcBDwLtVVUXkE8D/AaSBXwD/RVUnltLWanHVk95eDYsW+jtbaPfVA4GbcSEzQfoW5kbsg2dn+IQfvFhY9yT/fWAgr2ZWV67rav5+h4an/UzMSsb12mlZAq63mtnS117TlchyulNesK5r3sw/ns6WrWoIUuicmkwSDVmsbY8w4WdeLrRDLKQiPDYep68jSr8/qw1iTFLZ+fY4BbJ+NoZExrufIrbFZCLDTMqhvyOS+55KbTClqkuWun71WGE0qnLkUgnUooXPdT4hC05NpcrOOl74/NiWMNDVWKESUG4lyneKyF7g+8Ba4A9U9ZIl/vbtwPdVdZv/vfMSYfoC6CPArwGvBz4iIr3+x58D/gDY5v9d42//HnCx376XgA8ssZ1LwsWbkQC5QeYnR88U9YoqNlvJdznc8StrysoHttRZ3c7tA2UlF9y5fYBbrtrqeTDhCRXEi2twXEX8ALtmWYlUw1LcUYNr0R61/XILLhOJDO0RG0skZ4cI+rtQRcTCFUKwgimF57LqDcDdrWF62iL88LY30tUanpNbKv83yu1P/r21++kT7LpsQ8kKovVYYSylcuRyus8XssEPoCx22TzjvE3Y9vp1ZibNLV97atE2NvPzU+7KZRPwHlX9aQ1/+3pgp//6y8Be4LaCfX4T+J6qngEQke8B1/iCrktV9/vbvwL8NvCwqn437/j9wK4atnkeC81CAlQ1F+cCXsqWQltE8CB2Rm0mErNxLcGXd7eGyhrQlntWd+++IdJZB1eVbHY2vCHjuH4WZqnJSqRROvalBCEWGrtHplMksw6uwr1vu3yOW/eN9+1nZDrF6HRqjtNHIJgLZ6hdraFcXfTFyB94lxIYWOreemLoTNGVeL3uxWr7sJBnYtDeet5ff/Hbv8qtDzxNLO3kEokG2o0gP5jjas5VOZ11efLoGZ5+eZx37XzVgpH3zSBMCik3K3I9Zv/rVPUV//UpYF2RfTYAx/LeH/e3bfBfF24v5P/CU70VRUQGgUGAzZs3l93wfNa0hRmLL+yO7PreGwCCt2wtfNCCB3HTmnbCk4k5qTQ29baW7Z0UGFiDKPa04+bqk1RKOQP6S6enmEpmsfBWaYGg9bIIKFvXtC+5amUja6JA9Q9vvrG7syWcy24dXIvCiqLru6KcmPCqCG7oUUK2lRPMO7cPsOv4xJwSy9ddsp5//OkrRdWo+QGN+Qb8pRREq9T5ol6eatU6dRQKO8dVhqeS3PyVJ7EtizXtYda2L+6EUi07tw9wz42XzZmoeMHKcU9d5lcKzSdkCY4qn937Cy7Z2NOUQqQU1VaiLAvf6L++yEd/lv/Gt5UsNYC48Lf/DMgCf1dqH1W9D7gPYMeOHVX9/kBXC2fimQVXLyLezKolZDHQNTsrzX/Q8h/E9d2trO9uzQ1EP7ztjWW3p5j7bMZVppPZitKklDugB3EN4ZCNOC6OXz3TtoTPv31HVQ9DoVAbn0mtSB17qRl2RzSUO7fxVBZXldFYGlvA9S3zJyeS7DhvTU6wBPEk/Z1RNvsD6lMvT7Kxt5WTEwmvkml+4KLtJT0dm0kxPJ2mvyOy5IJola4Y6pVCqNrVZGEKocCzynFBRBmLZYiG7IrT1FSyqi4WJxRkm1aZm97Fy3jsBe5mHLfp7/dCyo3QrwpVvVpVLy7y923gtIicA+D/W0yxeAJPJRew0d92wn9duB3/+34f+N+B/6SqNRVahUynsmxe0zqnME8+AqzrjNIStnEVTkwkGBqJMZ3MzHnQapW99partnJmJoPixRoUus+WS7kR4ZGQBeoNirYtXrZcS+htC1ctWAr19IdGYmSduTO6lZBGpJS9RlVz5zbjetcp6yppV4mELEKW4PrHFwuGDK5HxnEYjaVy6lNLPHfWrqjNq/o75hnwCwuiVerNVan9qZ6F3sq1C+aT/4wFGa/FX+PZfiqhIE1N1nF5+uXxRW0zS7Ur7dw+wCd2Xcq2AS8haKAeC1ue1x14TgBR22r6+72QugqXRdgDvMN//Q7g20X2eRR4s4j0+ob8NwOP+uq0KRF5g3gpWt8eHC8i1wD/DbhOVet+NTb1thGyLTavaZsTyAieYOnriBANWSTSXtoHwZuFHB9PMJXI5B60Wj2ItXSfLcdoum2gk77OiLd8d5WQJfR1Rtg20FlRuwOKCjXL4rT/0AfUOjisHobeUsbWWNrJnduIr/oCQL3BTpB5grzwegxPJXllMkUi46k9w37Ay5a+du658TIefs9VSzbgL9SfU1NJr1hdKsO9+4aKnq9mMzbnP2OprFf+2cUT6IFaKu24TCUynJhIIrCowCg1CbvrkYNl3087tw/w8Huu4sW/uJYvvuPfEA1ZIIKiuKqoQndbeMWkzQmoq1psEe4EviEiNwG/BN4KICI7gD9U1ZtV9YyIfAwIwuDvCIz7wDuZdUV+2P8D+GsgCnzPTw2+X1X/sF6dCPS/YVs4pzvKy2cSXuVHW+huDTOVzM7aT1SxrNlBf237bEqVpRiOC1mq+yyUr9II+r++O1SToMZievogTU69AifradMpZq/ZtG/23PZ3Rjk65g30IuC63oB3rl9uIXdM3vWYTmYY9oWtt2L2BqL+jgiqOltQrbeNDr86Ya1UU0FfPrznebpbvbo3C52vauxV1ThvlHPM3GcsgYiXmw68Ynu4nrrw9LSXXmV99+xqr5SarNj9mnVcjo7FOW9tW8X3087tA7xr56v47N5fkHFcorZFd3uYsG2vqLQ5AFJnrdGKYceOHXrgwIGqjs1Px9ARDTES82wEw1OpebUXbIFNa9ro8A2rldhTKmlPMFjmD8aVzBor+Y78/i+1RkRQTrdQMEZsa16anFrNgEv95kLxSEuh8Ny+dHo6V0iuNWzT1xElZMuc388/5pWJBHG/TkPY8orFua6XMzurynlr23PXbCrh2QO7/TxT+dcRKveQ2ntwmFsffIZ42iHkF7Nz8dRKW/raefg9V83Zt5rvr/TercUxo7EU4/EMnS0hppNZ1ndF6WqN5PYvZf8sdu8cOj0NwpzVe6X3096Dw9z1yEGGRr0UjlvWti3ZMaZeiMhTqjovkXEjVy6rhmJGulsffAZH5wdOuXh63XrWoK/FKqiS76ilK2QpT6DCNDm1ZLnr3Bee24HOKKenU7ko+1TWQZk7U80/5mhQR11m9fLiB1RGQ9YcxwegqGAGKl6tBQPyTNrzDkz6AZxhy6uI+MKpaXb8xffYNtCZC66sdDVYjftyNccUXoP8NDWBwJhKZBiNpXJF485bM/95LXq/ui4be1qX7LEZVJYNvnclFVoDI1zqws7tA4Rtz1Sos+p071+FeNobMK/YuqbqSPpy2lAPlU69qaV6sFwaURgtOLfBgN3XEWEyniGZdcjGlXft3FxSxXTjfftzHoGuesGqjnpZ7NZ1zbevTCYyc1YU4M24qx3EW0I28bSTmzDlF7uLp7IMTyf57N5fsKY9THdrZZkAqhH01U4OSt3ft1y1lfftfpaJeCZXaynrKGMz6Xkel8Xu17AlTCYzS/LYXKlZCPIxwqUO7D04TCzlLOiefPnm7qpmdstBo5MCLrdQCwaTExMJHL+YV0c0xIfecmHdf/vefUNkHIepRJaMq7SEvADJoLJgqfZ+eM/zrO2AyXiGlOMSsiw29bbkVjIB+XnL8q/poeFp1nfNTZBYOCAXHvPS6SnO6W6lIxryqqAWELG9QbQtEsJxlcl4hr6OlpLfX4xmqJq6c/uAl3E8mcVRJeKXE7et4tmrC+/Xex57ibu/fyiX/80CxPegLHZ8sedtuVfT9aCR3mKrliALbqiIf7JXRzvKo88PMzyd5OUzcY6Mziya4HG5WKlJAZeKV2vH062jc4MQ60k1FSoDL6zz1nawtiPK689by71vu5yPXX9xUY/DK7aumXdNp5NZxmZKe+AVuw9iKYcTE3HG45l558dzVpOcx2Q0ZJEqcB8vZ8CvxmuyHi7P06ks5w90sH19F1v7O+hs8WxWh05PLegFFsQjBSmbXPWSi/b4XnuFwqHU8xY4YuSzkgqtgVm51IX8IlOBJ1DA2nYvs2084/iVDYWso5ycTHBud0vDZyarYTleKffuG6KrNcz67tbctuXqc36FSvA9xkS97QtQanVXTKVY7JoGGbrbIsW9/Iods6Y9zKmpFCFLCPmqngBHwfZLCAB0toTIxrViD79q1KL1UKWWKikxnXLmCYJ8bcNdjxxkeCrpqcMFIpaF4KnCiwmHUs+biJBx3GUpLVEvjHCpA8GNCbMz4KD29Xg8A3ieQV7QlB845cLJiQStkVBN6mxUS72X441WuRWjkSqIsC0kMp4Lcn4xsYhd3dqpmND54Lefm9e/vo4oWcf1C9nNH5CLnZO17VFOT6WI2BZpx6UlZHmZsB3PY2xtR5iOqDc4RkI279q5mSeGzlQ84FejFq21KrWYoX48nmFNe7jkxGvvwWFeGo5hi3dd0476xnxIZYvn2Ct1700mMnzs+ouX1fZYa4xwqQPBjTk8lczV3AAIiVc8LOMo/R1hJhMOLprLKeQo9EXthtpgaqG/zhcgnVGviFks7dARsRmbSdPVGm4qO1MjDPoBF6zr4shojOlklmTWRf2gubSjFaXrWYhS/du2rquka+xCqWv6O6O0RUI5bypHXaK20NMamZdSppTdqNkpthqaTGRY2146IPXOh19AVUm7viefH1iccaGrxS7qGr3QvdcIh5paYmwudSDQiStgWUI0ZBH1I6gjtkXI8mrTn9vTkrsBvZobXl34hdKt1Jul6q/zdci2wKHhGIdHZrAFjp7x9PVeOv7G9bGQeqYpKee3I74R38LzLArZQlvEXpKtKz/jwEQ8zWRewbBq7RhTiQxdUZujYzMcfGWS4xNx0o7rRbK3hTk5mWA0luInR89w64PPcM9jL1V3UpqEwhQz2wY6S9pB9h4c5tBILKepcBXfjga2Bffc8LqSnmmNuvfqjREudWLndq/e++Y1bVywrpML1nexfX0X67tb2DbQScbxDLhb+trZvKYNEeGc7oW9d5ar3UtJ2ZGvQx6NpbEtz640GkvjuJ46cCQvlUszeMA0Mk1J8NszqdmiXud2t9Lf2VK14C00EgcCIIi1KKd/heckYlte5olIiI09rTgKjusFT27obcV1YTKRJZlxCVneoPuZxw+veAGTz0KC4N59Q4Qtr0xx2JJcrkFXYVt/x8LxNk2UIqeWGLVYHVkoIBDmLrkDPXY+jfIOWcpyPF+HHASfBa+9PFrunH42iwdMrVUQlWbK7WoN5yYZAdUK3mKp5aeTWaaSWS7b3FuV7ePG+/aTdtzcd9qTSUKQu75B7jcFXNcL7sy6Lvf/6EjJOiTFzlHQ/mayyQUs5DjwwW8/x7quKK9MprD8CZXrKo7C7de+ZtHvbZY+1hIjXOrIYl4shVH91dSoaDbydcgR2/IKH6k3I+/riHJiIpFLG7JS+7gY1eQqq6Xdp1RqeYWq7VyFhudgopDKuv73z5JxlYzrqY8yTvHAwWLn6H27n0Wg6Wxy+ZQSBMH1O7enJReVb4vwqv72ovs3o2NLrTFqsTpTqLdd7cvjQHUwMp0k6yqprOvPeC1CttDTFmZLX3vd+tjIMrYB5ZYryKeWuvdSqeUjtlW1nauwJERfR9SL4fBzmpXCEorajoqdo1gqy3QyW9F5K6RR1z+4fvmq7oGuFm67ZnvRNp4NsWRm5dJErIbl8c7tXsXEz+79BVnX8yJS4Ezc87T55K5L69bHRlesDKjGtbmWsRr56thU1sES7xoE0fLVqNsKVbzBRGHST4wZtmW2dEAea9sjOQGR35di58hxvVo3+VTS1kZe/0qu39kSS2aEi6HmPDF0ho29rfOyDPe2R+v68FT60NZLNVGtiiv47aBNwYy9mpiPYqnlu/zBvBp1W7HB80NvuZB79w3l+jo8lZxTd6e/I5KrqFooIIqdI9sS0LnxPZW0tdGDdrmTw9WQ2qUcjFrMUHPKLTTWyN+tp2qiWhXX3oPDvG/3szxzbJzTU0meOTbO+3Y/W1WbAnXsu3a+ClU4PhHnF8PTjMaSVavbiql489WgsVQW8AKHA8ECpesAFZ6jjmiIzpZQ1arBRt13lVKrqrPNjhEuhprTqIenkt+txC5SqR6/WvvZnQ+/wEQ8g/ouvurCRDzDnQ+/UEbv5xPkueptC9MSskk7ypmZDLsu21CzmfzO7QPsumwD435G56jtueGOxzOcmoxzaHiao2NxJuLpOeet2Dn65K5L+cSuS6u2O66UQXs1x7bkY9RihppTygW7ng/P3oPDTMTTHB2LE7aFdZ1RQn4J4WK/W65qolo9fjX2syNBnRZrNs+YusqRsepm3oEA7W5tod+vWxVPZxfMuFwNhWpQz0MtwUgsQ0vIYmNPC2nHnXfeSp2jagVfI+67aqhHLrRmxAiXs5x62B2W++HJFwAbe1o4PZXi+ESCbf0dfOgtFy7oOrqYXaTRevylUKluv9p7ofB3OlvChKykF0C4bm41xnqet5U0aK8G553FaJhwEZE1wNeB84CjwFtVdbzIfu8APui//QtV/bK//XLgS0Ar8BDwbs1zNRGR/wp8EuhX1dG6dWQFs9y14+tFoQDoao0s6kBQ7ix3OY2vW/vaOTQcQ3Q2iaUX4d1e1fdV4liwlHuh2O+kHJeWUO3tH4sJwEYN2mdD3EqlNNLmcjvwfVXdBnzffz8HXwB9BPg14PXAR0Sk1//4c8AfANv8v2vyjtsEvBl4uZ4dWOlUE4+xEI2KMajGkFuuXWQ59fi3XbOd3rawX/nQS9nS2xYuGitRDpXo9pdyLxT7nZBl0dkyd+661PPWrPEhzdquRtNItdj1wE7/9ZeBvcBtBfv8JvA9VT0DICLfA64Rkb1Al6ru97d/Bfht4GH/uLuB/wZ8u26tr5ByZzbLOQOq5ay8kplvrfu4FNffxX53OfX4O7cP8Ildl9ZMrVOJmmgp90Kx37n+0nPZ/fSJmp63whVq1lGGp5Pc8rWnKkprU2tWsuq0njRSuKxT1Vf816eAdUX22QAcy3t/3N+2wX9duB0RuR44oarP5udpKoaIDAKDAJs3b66iC+VR7sC73EFgtUw5Uu4DVus+7j04zPhMiqNjM4Qti3VdCxvyK2W59fi1VuuU+31LvReK/c4lG3tqdt72HhzmwNEzZFxFgJAFiueZ5qo2NFXM2RK3Uil1FS4i8hiwvshHf5b/RlVVRBYqOV/u77UB/x1PJbYoqnofcB/Ajh07lvz7pSh34F3uGVAtZ+XlPmC17ONcQ34rp6dTHJ9IcsFABx96y/aautuu9hloPVZotTpvQfxPNq/yZcYF8EqDR0N2Q1cLjawH1MzUVbio6tWlPhOR0yJyjqq+IiLnAMUUlCeYVZ0BbMRTn53wX+dvPwG8CtgCBKuWjcDTIvJ6VT21hK4siXIH3uWeAdVyVl7uA1bLPpYy5Pe0RVa9MKg19Vqh1UIFeu++IWKpLLZAVudmMss6yoYer4BXo1YLK8UFerlppFpsD/AO4E7/32L2kUeB/5FnxH8z8AFVPSMiUyLyBuDHwNuBv1LVnwG5O1dEjgI7Gu0tVu7A24gZUK1ml+U+YLXq43sffJonhsZy77uiNr/S19HU6ojl9iiq9PdqvUKrlQr02Hgcx1VCtoW4iqNKsIixLaGzpfq0NrVgJblALyeN9Ba7E/gNETkEXO2/R0R2iMj9AL4h/2PAk/7fHYFxH3gncD9wGPgFs8b8pqNcr52VHLlbrvdVLfr43gef5ls/fWXOtqmUwy9HY02rjlhuj6Jm8GCqlTfipt42bEtQ9erEREM2IX/kCso3NPpZKTf7+dmEFGYhPVvZsWOHHjhwoG7fH8wiF5vZlLvfSmapfXzVf3/IK5XM/GTvm9e0LWupgnJXBzfet3/eii2ezjLQ2VKyjv1SWO7fK8aVdz1OT2t4TgE0VWUykeGHt72x7O8JbC4T8cycCo9tYYsNvW3EUtmy7iMTi1IfROQpVd1RuN1E6C8T5aoczgbj8VL76AQ6EQEp0MEvt2ApV+2z3Pa0ZvBgqpUKdOf2AT6561LufPiFXCqcbf3t3HbNXKeNIM6qmPBolnIMZxNGuKwQ7nnsJe7/0RFm0g7tEZubr9xSsnzsase2ZK6AwYtmty1Z1oGiEs+35banNYMHU74dLuu4nJ5KkXFdwpYUrU65EItNSBYTHpW4ypvVTW0wWZFXAPc89hKfefwwiYxDyPIGic88fph7Hnup0U1rCNdd4nm3q87+5W9fLirJDLDc9rRmsN8FdriIbXF8IgkCG3taybhac/vPYvadcq5VM9ipVhNGuKwA7v/RESyBkGVhieX/620/G7n7hsv4ndee4xWXwlux/M5rz+HuGy5b1nZUkhqm2jT81RL8XtgSDg3HOD6eoD1iL35gHdrR0xbhvLVtbBvopKs1suQ0Q8VYTHiUc61qnQ7pbMeoxVYAM2kn5x0TYIm3/Wzl7hsu4+4bGtuGSuMbGmFPi2dcNva25tq3Uso+V8piasByrlUz2KlWE2blsgJoj9i4BW5RrtKQmahhluVejVRKs8zElyP5ZzE14FQiw/hMiivvepx79w2x67INC16rlVJsbKVgVi4rgJuv3MJnHj9M1nX9XEre381Xbml00856mtm7r1lm4ssRwV4YyNgRDaFAxtWc/WT30ycWFP4m0r62mJXLCuDWqy/g3W88n9awTdb1Boh3v/H8s9ZbzFAezTITX64VXn4gY09bhO7WcEWrtmZfia40TBClT72DKA2G5SbfPTd/Jn42DJi1CuA0LI4JolyhGL97Q7WczTmvmiHO52zHCJcmxkQVG5ZKM9uE6klgPxmZTjKdzJLKutiWcP2l5za6aWcNxubSxDSLt4/BsNLYuX2AXZdtYDyeIZl1iNjCmvYwu58+YYIilwmzcmlimsXbx2BYiTwxdIaNva3zkneWU1DMqKOXjlm5NDHN4u1jMKxEKknPk49JA1MbjHBpYpohP5TBsFKpdnJm1NG1wQiXJsb43RsM1VM4ORuNJTk+nuCl01PceN/+kiuRalc8hrkYm0uTU06qcaMbNhjmk++KfWh4mulklt62MH0d0QU9L40bc20wK5cVjNENGwwLE0Tt97VHEIGxmTRHRmfIOlpS1WXU0bWhYcJFRNaIyPdE5JD/b2+J/d7h73NIRN6Rt/1yEfmZiBwWkXskLxRXRP5ERA6KyPMi8j+Xoz+NwOiGDYbF2XtwmEMjMVxXsUXIOsrJyQRZxy2q6jLq6NrQSLXY7cD3VfVOEbndf39b/g4isgb4CLADr5rtUyKyR1XHgc8BfwD8GHgIuAZ4WER+HbgeuFRVUyKyau+IY+NxbIGhkRhpxyViW/R1RIxu2GDI4959Q4QtCwVEBBHAhdPTKV63qeic9qwNPq0ljVSLXQ982X/9ZeC3i+zzm8D3VPWML1C+B1wjIucAXaq6X73kaF/JO/6PgDtVNQWgqqtWR9QRsTkxkSTrzM7ITkwkTSp+gyGPY+Nx1nVFUQVXleB/RtVVXxopXNap6iv+61PAuiL7bACO5b0/7m/b4L8u3A5wAfC/iciPReQHIvJvSjVARAZF5ICIHBgZGam2Hw0jpwmUvL/87QaDgU29bYRsi3N7WghZguMqlggXDHSY1UkdqatwEZHHROS5In/X5+/nrz5qlZ45BKwB3gC8H/iGlBhtVfU+Vd2hqjv6+/tr9PPLx3Qqy5q2MBnHJZlxyTgua9rCxFLZRjfNYGgaAgO9bQlb+trZvKaNga4Wbrtme6Obtqqpq3BR1atV9eIif98GTvvqLfx/i6mvTgCb8t5v9Led8F8XbgdvFfMP6vETwAX6atuz5qAzGmJsJu3pkvGk89hMmo6o8TA3GAIKDfQR26ItbPHBbz+3YLyLYWk0Ui22Bwi8v94BfLvIPo8CbxaRXt+b7M3Ao746bUpE3uCvSt6ed/w/Ar8OICIXABFgtG69aCDTiTSOgqonWFTBUW+7wWCYJXBJ/tj1FzOTduZUqDTu+/WhkVPcO/FUVjcBvwTeCiAiO4A/VNWbVfWMiHwMeNI/5g5VPeO/fifwJaAVeNj/A/gC8AUReQ5IA+/QVVoRbWQmgy2zgkXEW8GMzGQa3bSqMAGhhnqT774P0BYJlZ3M0lAZDRMuqjoGvKnI9gPAzXnvv4AnMIrtd3GR7WngbTVtbBNjW4JtzS5AHddtYGuqx9SuMSwHJtP48mEi9FcwW/vacfPcK11VXPW2rzRMQKhhOTCZxpcPI1xWMLdds53etjACZB0XAXrbwivSC8YkCzQsBya1y/Jh3IpWMDu3D/CJXZeuihrpQbJAx1VGplOkHRdbPNdRQ+NYbXaw/GSWK/2ZaXZkldq6K2bHjh164MCBRjfjrGXvwWHev/tZxuMZLD8qyVXoaQvzyV2Xmoe/AeTbwVrDNomMQ8ZRk2fLMAcReUpVdxRuN2oxQ1Owc/sAa9sjhGxBgbBtsaGnle7WsLG7NAhjBzMsBaMWMzQNsbTD+f0dc9LXqKqxuzQI41llWApm5WJoGownT3NhrodhKRjhYmgajCdPc2Guh2EpGOFiaBpMkabmwlwPw1Iw3mI+xlvMsBRWm8uuwVAuxlvMYKgTgcvu8HTSJEM0GHyMcDEYlohx2TUY5mOEi8GwREzqGoNhPka4GAxLxLjsGgzzMcLFYFgixmXXYJiPES4GwxIxLrsGw3xM+heDoQbs3D5ghInBkIdZuRgMBoOh5jRMuIjIGhH5nogc8v/tLbHfO/x9DonIO/K2Xy4iPxORwyJyj/jZDkXktSKyX0R+KiIHROT1y9Ung8FgMHg0cuVyO/B9Vd0GfN9/PwcRWQN8BPg14PXAR/KE0OeAPwC2+X/X+Nv/J/Dnqvpa4MP+e4PBYDAsI40ULtcDX/Zffxn47SL7/CbwPVU9o6rjwPeAa0TkHKBLVferl7/mK3nHK9Dlv+4GTtan+QaDwWAoRSMN+utU9RX/9SlgXZF9NgDH8t4f97dt8F8Xbgd4D/CoiHwST3j+21INEJFBYBBg8+bNlffAYDAYDEWpq3ARkceA9UU++rP8N6qqIlKrDJp/BLxXVf9fEXkr8Hng6mI7qup9wH1+W0dE5JdV/mYfMFrlsSsV0+ezA9Pn1c9S+/srxTbWVbioatFBHUBETovIOar6iq/mKpbl7wSwM+/9RmCvv31jwfYT/ut3AO/2X38TuL/MtvaXs18xRORAsaygqxnT57MD0+fVT73620ibyx48QYD/77eL7PMo8GYR6fUN+W8GHvXVaVMi8gbfS+ztecefBP69//qNwKF6dcBgMBgMxWmkzeVO4BsichPwS+CtACKyA/hDVb1ZVc+IyMeAJ/1j7lDVM/7rdwJfAlqBh/0/8DzIPiMiISCJb1MxGAwGw/JhioXVABEZ9O03Zw2mz2cHps+rn3r11wgXg8FgMNQck/7FYDAYDDXHCBeDwWAw1BwjXCpARK4RkRf9fGbF0tVEReTr/uc/FpHzGtDMmlJGn/9URH4uIv8qIt8XkaI+7yuJxfqct9//KSLqO6GsWMrpr4i81b/Oz4vI3y93G2tNGff1ZhH5ZxF5xr+3f6sR7awlIvIFERkWkedKfC5+nsbDfp8vW9IPqqr5K+MPsIFfAFuBCPAscGHBPu8E/tZ/fQPw9Ua3exn6/OtAm//6j86GPvv7dQL7gP3Ajka3u87XeBvwDNDrvx9odLuXoc/3AX/kv74QONrodteg31cBlwHPlfj8t/C8bgV4A/DjpfyeWbmUz+uBw6o6pKpp4EG8/Gj55OdL2w28KcjWvEJZtM+q+s+qGhSL38/c4NaVSDnXGeBjwF147u4rmXL6+wfAZ9XL74eqFgt4XkmU0+dVl6NQVfcBZxbY5XrgK+qxH+jxA9yrwgiX8imV56zoPqqaBSaBtcvSuvpQTp/zuYnZeKOVyqJ99tUFm1T1n5azYXWinGt8AXCBiPwvv5zFNaxsyunzR4G3ichx4CHgT5anaQ2l0ud9QUwlSkNNEJG3ATuYzY6wKhERC/hL4Pcb3JTlJISnGtuJtzLdJyK/qqoTjWxUnbkR+JKqfkpErgC+KiIXq6rb6IatFMzKpXxOAJvy3ufnM5u3j58hoBsYW5bW1Ydy+oyIXI2XjPQ6VU0tU9vqxWJ97gQuBvaKyFE83fSeFWzUL+caHwf2qGpGVY8AL+EJm5VKOX2+CfgGgKo+AbTgJXhczZT1vJeLES7l8ySwTUS2iEgEz2C/p2Cf/Hxpu4DH1beUrVAW7bOIvA64F0+wrHRdPCzSZ1WdVNU+VT1PVc/DszNdp6oHGtPcJVPOff2P+AlkRaQPT002tIxtrDXl9Pll4E0AIvIaPOEysqytXH72AG/3vcbeAEzqbFmUijFqsTJR1ayI/DFeMk0b+IKqPi8idwAHVHUPXnr/r4rIYTzD2Q2Na/HSKbPPnwA6gG/6vgsvq+p1DWv0Eimzz6uGMvsbJJD9OeAA71fVFbsiL7PP/xX4f0TkvXjG/d9f4RNFROQBvElCn29L+ggQBlDVv8WzLf0WcBiIA/9lSb+3ws+XwWAwGJoQoxYzGAwGQ80xwsVgMBgMNccIF4PBYDDUHCNcDAaDwVBzjHAxGAwGQ80xwsVgMBgMNccIF4NhGRGR2DL8xnULlQowGJYDE+diMCwjIhJT1Y5Gt8NgqDdm5WIwLAERuVNE3pX3/qMi8kG/cNrTIvIzEZmXsl9EdorI/5f3/q9F5Pf915eLyA9E5CkReXShtOcicmtesbYH/W2/LyJ/7b/+koh8zs9mPOT/7hdE5AUR+VLtzoTBMBcjXAyGpfF14K1579+KV9Pnd1T1Mrxiap8qt66PiISBvwJ2qerlwBeAjy9wyO3A61T1EuAPS+zTC1wBvBcvf9TdwEXAr4rIa8tpl8FQKSa3mMGwBFT1GREZEJFzgX5gHDgF3C0iVwEuXk2Mdf72xXg1Xtbl7/nyyAYWSh74r8Dficg/4iWYLMZ3VFVF5GfAaVX9GYCIPA+cB/y0jHYZDBVhhIvBsHS+iZcFez3eSuY/4Qmay1U146fmbyk4JstczUHwuQDPq+oVZf72W/DK1/4fwJ+JyK8W2Scog+DmvQ7emzHAUBeMWsxgWDpfx8uAvQtP0HQDw75g+XXgV4oc80vgQhGJikgPfnp34EWg3y9QhYiEReSiYj/qFy7bpKr/DNzm/65xFjA0BWbWYjAsET9deydwQlVfEZG/A77jq6EOAAeLHHNMRL4BPAccAZ7xt6dFZBdwj4h04z2jnwaeL/LTNvA1fz8B7lHViTLNOwZDXTGuyAaDwWCoOUYtZjAYDIaaY9RiBsMKQEQ+C/y7gs2fUdUvNqI9BsNiGLWYwWAwGGqOUYsZDAaDoeYY4WIwGAyGmmOEi8FgMBhqjhEuBoPBYKg5/z/R7FxylAU/AgAAAABJRU5ErkJggg==\n" }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" } ], "source": [ - "\n" + "sns.residplot(data=merged_results_wo_zeros, x=\"value_sim\", y=\"value_obs\")" ], "metadata": { "collapsed": false, @@ -790,33 +752,24 @@ }, { "cell_type": "code", - "execution_count": 106, - "outputs": [ - { - "data": { - "text/plain": "array([ True, True, True, True, True])" - }, - "execution_count": 106, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 230, + "execution_count": 306, "outputs": [], "source": [ - "# test_trip_res = merged_results.loc[merged_results.trip_id == 1045822217]\n", - "# ci = multinomial_proportions_confint(num_samples * test_trip_res.value_sim.values, alpha=0.999, method='goodman')\n", - "# np.hstack((test_trip_res[['value_obs']].to_numpy(), ci))" + "# # Note power tests will be inf when expected count is zero (any non-zero count will be impossible)\n", + "# eps = 0 #1e-14\n", + "# non_zero_shares = ms_comp.loc[(ms_comp.mode_share_obs > eps) & (ms_comp.trip_mode > eps)][[\"mode_share_obs\", \"trip_mode\"]]\n", + "# ddof = non_zero_shares.shape[0] - 1 # what to use here? num alternatives - 1? What about zero expected count?\n", + "#\n", + "# non_zero_shares.trip_mode *= num_samples\n", + "#\n", + "# sum_ms = non_zero_shares.mode_share_obs.sum()\n", + "# if sum_ms != 1.0:\n", + "# if np.abs(1.0 - sum_ms) > 1e-1:\n", + "# print(f\"NOT GOOD {sum_ms} should be one\")\n", + "# #print(f\"sum ms is not one but {sum_ms}, re-normalising\")\n", + "# non_zero_shares.mode_share_obs /= sum_ms\n", + "#\n", + "# powerdiscrepancy(non_zero_shares.trip_mode.values, non_zero_shares.mode_share_obs.values, lambd=0, ddof=ddof)" ], "metadata": { "collapsed": false, @@ -827,7 +780,7 @@ }, { "cell_type": "code", - "execution_count": 275, + "execution_count": 314, "outputs": [], "source": [ "@numba.jit\n", @@ -841,8 +794,8 @@ " val = x[2]\n", " return within(lower, upper, val)\n", "\n", - "def all_within_ci(trip_df, num_samples):\n", - " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=0.99999, method='goodman')\n", + "def all_within_ci(trip_df, num_samples, alpha=0.9):\n", + " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=alpha, method='goodman')\n", " vals_with_ci = np.hstack((ci, trip_df[['value_obs']].to_numpy()))\n", " return np.apply_along_axis(numpy_within, 1, vals_with_ci).all()" ], @@ -855,80 +808,43 @@ }, { "cell_type": "code", - "execution_count": 276, - "outputs": [], - "source": [ - "vals_within_ci = merged_results.groupby(\"trip_id\").apply(lambda group: all_within_ci(group, num_samples))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 277, + "execution_count": 328, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Have 57 with contribution outside 0.99 CI\n" + "Have 17 with contribution outside 95.00% CI, corresponding to 1.14% of all non-zero observations.\n" ] + }, + { + "data": { + "text/plain": " lb ub value within\n3 0.000031 0.001291 0.000030 False\n3 0.000031 0.001291 0.000025 False\n8 0.003744 0.008367 0.003717 False\n8 0.000009 0.001113 0.000008 False\n1 0.000009 0.001113 0.000006 False\n1 0.000009 0.001113 0.000008 False\n3 0.000062 0.001460 0.000058 False", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
lbubvaluewithin
30.0000310.0012910.000030False
30.0000310.0012910.000025False
80.0037440.0083670.003717False
80.0000090.0011130.000008False
10.0000090.0011130.000006False
10.0000090.0011130.000008False
30.0000620.0014600.000058False
\n
" + }, + "execution_count": 328, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ + "alpha = 0.05 # default. this corresponds to a 95% CI, so 95% of all values should lie within our approximation\n", + "vals_within_ci = merged_results_wo_zeros.groupby(\"trip_id\").apply(lambda group: all_within_ci(group, num_samples, alpha))\n", "trips_not_within = vals_within_ci.loc[~vals_within_ci]\n", - "print(f\"Have {trips_not_within.shape[0]} with contribution outside 0.99 CI\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 291, - "outputs": [], - "source": [ + "print(f\"Have {trips_not_within.shape[0]} with contribution outside {1 - alpha:.2%} CI, corresponding to \"\n", + " f\"{trips_not_within.shape[0] / merged_results_wo_zeros.shape[0]:.2%} of all non-zero observations.\")\n", + "\n", "bad_res = []\n", "\n", "for bad_trip_id in trips_not_within.index.values:\n", " trip_df = merged_results.loc[merged_results.trip_id == bad_trip_id]\n", - " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=0.999, method='goodman')\n", + " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=alpha, method='goodman')\n", " vals_with_ci = np.hstack((ci, trip_df[['value_obs']].to_numpy()))\n", " x_ = pd.DataFrame(vals_with_ci)\n", " x_['in_range'] = np.apply_along_axis(numpy_within, 1, vals_with_ci)[:np.newaxis]\n", " bad_res.append(x_.loc[~x_.in_range])\n", "bad_res = pd.concat(bad_res)\n", - "bad_res.columns = [\"lb\", \"ub\", \"value\", \"within\"]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 292, - "outputs": [ - { - "data": { - "text/plain": " lb ub value within\n8 0.001687 0.003703 0.001667 False\n8 0.005178 0.008409 0.004877 False\n8 0.005531 0.008856 0.004911 False\n3 0.000054 0.000738 0.000051 False\n8 0.004739 0.007848 0.004560 False\n3 0.000154 0.001038 0.000112 False\n3 0.000154 0.001038 0.000125 False\n3 0.000403 0.001589 0.000400 False\n3 0.100737 0.112972 0.100614 False\n3 0.000540 0.001852 0.000525 False\n3 0.000470 0.001721 0.000423 False\n7 0.000273 0.001319 0.001341 False\n8 0.000017 0.000575 0.000011 False\n8 0.003184 0.005805 0.002994 False\n8 0.001930 0.004060 0.001838 False\n8 0.003099 0.005690 0.003060 False\n3 0.000154 0.001038 0.000117 False\n8 0.002676 0.005113 0.002415 False\n3 0.001133 0.002858 0.001023 False\n8 0.005266 0.008521 0.004850 False\n8 0.002509 0.004881 0.002161 False\n8 0.005531 0.008856 0.005396 False\n8 0.003269 0.005920 0.003160 False\n3 0.000054 0.000738 0.000030 False\n3 0.000054 0.000738 0.000025 False\n3 0.076921 0.087807 0.076860 False\n8 0.004477 0.007510 0.004245 False\n2 0.001133 0.002858 0.001085 False\n8 0.004477 0.007510 0.004245 False\n8 0.003526 0.006263 0.003404 False\n8 0.003440 0.006149 0.003351 False\n8 0.004739 0.007848 0.004276 False\n8 0.003784 0.006605 0.003404 False\n8 0.004303 0.007285 0.003717 False\n8 0.000017 0.000575 0.000008 False\n1 0.000101 0.000891 0.000084 False\n8 0.001930 0.004060 0.001920 False\n3 0.032773 0.040191 0.032192 False\n3 0.000337 0.001455 0.000256 False\n3 0.000756 0.002236 0.000730 False\n8 0.000017 0.000575 0.000017 False\n1 0.000054 0.000738 0.000033 False\n1 0.000017 0.000575 0.000006 False\n1 0.000017 0.000575 0.000008 False\n8 0.000017 0.000575 0.000014 False\n3 0.000054 0.000738 0.000041 False\n3 0.000101 0.000891 0.000058 False\n8 0.000017 0.000575 0.000016 False\n8 0.018715 0.024460 0.018662 False\n0 0.998545 0.999663 0.999677 False\n1 0.000054 0.000738 0.000053 False\n3 0.103271 0.115636 0.102986 False\n3 0.103271 0.115636 0.102967 False\n8 0.000017 0.000575 0.000012 False\n8 0.000017 0.000575 0.000013 False\n8 0.002676 0.005113 0.002658 False\n3 0.001289 0.003102 0.001196 False\n7 0.009844 0.014139 0.009778 False\n3 0.000470 0.001721 0.000384 False", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
lbubvaluewithin
80.0016870.0037030.001667False
80.0051780.0084090.004877False
80.0055310.0088560.004911False
30.0000540.0007380.000051False
80.0047390.0078480.004560False
30.0001540.0010380.000112False
30.0001540.0010380.000125False
30.0004030.0015890.000400False
30.1007370.1129720.100614False
30.0005400.0018520.000525False
30.0004700.0017210.000423False
70.0002730.0013190.001341False
80.0000170.0005750.000011False
80.0031840.0058050.002994False
80.0019300.0040600.001838False
80.0030990.0056900.003060False
30.0001540.0010380.000117False
80.0026760.0051130.002415False
30.0011330.0028580.001023False
80.0052660.0085210.004850False
80.0025090.0048810.002161False
80.0055310.0088560.005396False
80.0032690.0059200.003160False
30.0000540.0007380.000030False
30.0000540.0007380.000025False
30.0769210.0878070.076860False
80.0044770.0075100.004245False
20.0011330.0028580.001085False
80.0044770.0075100.004245False
80.0035260.0062630.003404False
80.0034400.0061490.003351False
80.0047390.0078480.004276False
80.0037840.0066050.003404False
80.0043030.0072850.003717False
80.0000170.0005750.000008False
10.0001010.0008910.000084False
80.0019300.0040600.001920False
30.0327730.0401910.032192False
30.0003370.0014550.000256False
30.0007560.0022360.000730False
80.0000170.0005750.000017False
10.0000540.0007380.000033False
10.0000170.0005750.000006False
10.0000170.0005750.000008False
80.0000170.0005750.000014False
30.0000540.0007380.000041False
30.0001010.0008910.000058False
80.0000170.0005750.000016False
80.0187150.0244600.018662False
00.9985450.9996630.999677False
10.0000540.0007380.000053False
30.1032710.1156360.102986False
30.1032710.1156360.102967False
80.0000170.0005750.000012False
80.0000170.0005750.000013False
80.0026760.0051130.002658False
30.0012890.0031020.001196False
70.0098440.0141390.009778False
30.0004700.0017210.000384False
\n
" - }, - "execution_count": 292, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ + "bad_res.columns = [\"lb\", \"ub\", \"value\", \"within\"]\n", "bad_res" ], "metadata": { From 921983fb32d8624bffb08615dcd75f70c0ff33c3 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 17 May 2022 10:53:07 +1000 Subject: [PATCH 033/135] adds more convergence analysis --- notebooks/validate_frozen_impl.ipynb | 141 ++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 11 deletions(-) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 6567e6ba41..c267d16757 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -77,6 +77,7 @@ "from statsmodels.stats.gof import powerdiscrepancy\n", "from statsmodels.stats.proportion import multinomial_proportions_confint\n", "import numba\n", + "import matplotlib.pyplot as plt\n", "\n", "from activitysim.cli import run\n", "from activitysim.core import inject\n", @@ -855,22 +856,57 @@ } }, { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "cell_type": "markdown", + "source": [ + "## CI is not great for this\n", + "\n", + "we expect about 5% outliers for 95% CI, and I can see for 1e5 samples the number of outliers actually nearly doubles for the 95CI which makes sense - it is the large sample limit where all this is exact.\n", + "\n", + "Let's plot a histogram of the std deviation, and take a single trip and plot the diff from expected value with number of draws (i.e. prob vs num_draws)" + ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } } }, + { + "cell_type": "markdown", + "source": [ + "### histogram of residuals" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 440, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5.4540809129771874e-05\n" + ] + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEICAYAAAC9E5gJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQB0lEQVR4nO3df6zddX3H8efLVkRx2iI3DWsbbzcbHZo48aawsZhFtlLA2GZRg3HSmGb9Y8zhfkRh/zRDSTBZREkmSWO7lc2IDF1ohMkaKHEm48ctIFgq444ftg3YqwWUGXXF9/44n7ID3tvee8/pPbf3Ph/Jzfl+39/P95z3N4S+7vfH+dxUFZKkhe1Vg25AkjR4hoEkyTCQJBkGkiQMA0kShoEkiSmEQZLtSQ4l+W5X7fQku5I81l6XtnqSXJdkLMlDSc7u2mdjG/9Yko1d9Xcnebjtc12S9PsgJUnHNpUzg38E1r2idgVwR1WtBu5o6wAXAqvbz2bgeuiEB7AFOAdYA2w5GiBtzJ907ffKz5IknWCLjzegqr6VZPgV5fXA77flHcBdwKda/YbqfJPt7iRLkpzZxu6qqsMASXYB65LcBbyhqu5u9RuADcC/Ha+vM844o4aHX9mWJGkye/bs+WFVDU207bhhMIllVfV0W34GWNaWlwP7u8YdaLVj1Q9MUD+u4eFhRkdHp9+5JC1QSZ6abFvPN5DbWcCszGmRZHOS0SSj4+Pjs/GRkrQgzDQMftAu/9BeD7X6QWBl17gVrXas+ooJ6hOqqq1VNVJVI0NDE57pSJJmYKZhsBM4+kTQRuCWrvql7amic4Hn2+Wk24G1SZa2G8drgdvbth8nObc9RXRp13tJkmbJce8ZJPkKnRvAZyQ5QOepoGuAm5JsAp4CPtSG3wZcBIwBPwU+BlBVh5N8Grivjbvq6M1k4E/pPLH0Wjo3jo9781iS1F85WaewHhkZKW8gS9LUJdlTVSMTbfMbyJIkw0CSZBhIkjAMJEnM/BvIJ7XhK259afnJay4eYCeSNDd4ZiBJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJoscwSPIXSfYm+W6SryQ5NcmqJPckGUvy1SSntLGvaetjbftw1/tc2eqPJrmgx2OSJE3TjMMgyXLgz4GRqnoHsAi4BPgscG1VvQV4FtjUdtkEPNvq17ZxJDmr7fd2YB3wxSSLZtqXJGn6er1MtBh4bZLFwOuAp4H3Aje37TuADW15fVunbT8/SVr9xqr6eVU9AYwBa3rsS5I0DTMOg6o6CPwd8H06IfA8sAd4rqqOtGEHgOVteTmwv+17pI1/U3d9gn0kSbOgl8tES+n8Vr8K+HXgNDqXeU6YJJuTjCYZHR8fP5EfJUkLSi+Xif4AeKKqxqvqf4GvA+cBS9plI4AVwMG2fBBYCdC2vxH4UXd9gn1epqq2VtVIVY0MDQ310LokqVsvYfB94Nwkr2vX/s8HHgF2Ax9oYzYCt7TlnW2dtv3OqqpWv6Q9bbQKWA3c20NfkqRpWnz8IROrqnuS3AzcDxwBHgC2ArcCNyb5TKtta7tsA/4pyRhwmM4TRFTV3iQ30QmSI8BlVfXiTPuSJE3fjMMAoKq2AFteUX6cCZ4GqqqfAR+c5H2uBq7upRdJ0sz5DWRJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ9BgGSZYkuTnJ95LsS/I7SU5PsivJY+11aRubJNclGUvyUJKzu95nYxv/WJKNvR6UJGl6ej0z+ALwzap6G/BOYB9wBXBHVa0G7mjrABcCq9vPZuB6gCSnA1uAc4A1wJajASJJmh0zDoMkbwTeA2wDqKpfVNVzwHpgRxu2A9jQltcDN1TH3cCSJGcCFwC7qupwVT0L7ALWzbQvSdL09XJmsAoYB/4hyQNJvpTkNGBZVT3dxjwDLGvLy4H9XfsfaLXJ6r8iyeYko0lGx8fHe2hdktStlzBYDJwNXF9V7wL+h/+/JARAVRVQPXzGy1TV1qoaqaqRoaGhfr2tJC14vYTBAeBAVd3T1m+mEw4/aJd/aK+H2vaDwMqu/Ve02mR1SdIsmXEYVNUzwP4kb22l84FHgJ3A0SeCNgK3tOWdwKXtqaJzgefb5aTbgbVJlrYbx2tbTZI0Sxb3uP/HgS8nOQV4HPgYnYC5Kckm4CngQ23sbcBFwBjw0zaWqjqc5NPAfW3cVVV1uMe+JEnT0FMYVNWDwMgEm86fYGwBl03yPtuB7b30IkmaOb+BLEkyDCRJhoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEn0IgySLkjyQ5BttfVWSe5KMJflqklNa/TVtfaxtH+56jytb/dEkF/TakyRpevpxZnA5sK9r/bPAtVX1FuBZYFOrbwKebfVr2ziSnAVcArwdWAd8McmiPvQlSZqinsIgyQrgYuBLbT3Ae4Gb25AdwIa2vL6t07af38avB26sqp9X1RPAGLCml74kSdPT65nB54FPAr9s628CnquqI239ALC8LS8H9gO07c+38S/VJ9jnZZJsTjKaZHR8fLzH1iVJR804DJK8DzhUVXv62M8xVdXWqhqpqpGhoaHZ+lhJmvcW97DvecD7k1wEnAq8AfgCsCTJ4vbb/wrgYBt/EFgJHEiyGHgj8KOu+lHd+0iSZsGMzwyq6sqqWlFVw3RuAN9ZVR8BdgMfaMM2Are05Z1tnbb9zqqqVr+kPW20ClgN3DvTviRJ09fLmcFkPgXcmOQzwAPAtlbfBvxTkjHgMJ0Aoar2JrkJeAQ4AlxWVS+egL4kSZPoSxhU1V3AXW35cSZ4GqiqfgZ8cJL9rwau7kcvkqTp8xvIkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAk0UMYJFmZZHeSR5LsTXJ5q5+eZFeSx9rr0lZPkuuSjCV5KMnZXe+1sY1/LMnG3g9LkjQdvZwZHAH+qqrOAs4FLktyFnAFcEdVrQbuaOsAFwKr289m4HrohAewBTgHWANsORogkqTZMeMwqKqnq+r+tvwTYB+wHFgP7GjDdgAb2vJ64IbquBtYkuRM4AJgV1UdrqpngV3Aupn2JUmavr7cM0gyDLwLuAdYVlVPt03PAMva8nJgf9duB1ptsrokaZb0HAZJXg98DfhEVf24e1tVFVC9fkbXZ21OMppkdHx8vF9vK0kLXk9hkOTVdILgy1X19Vb+Qbv8Q3s91OoHgZVdu69otcnqv6KqtlbVSFWNDA0N9dK6JKlLL08TBdgG7Kuqz3Vt2gkcfSJoI3BLV/3S9lTRucDz7XLS7cDaJEvbjeO1rSZJmiWLe9j3POCjwMNJHmy1vwGuAW5Ksgl4CvhQ23YbcBEwBvwU+BhAVR1O8mngvjbuqqo63ENfkqRpmnEYVNW3gUyy+fwJxhdw2STvtR3YPtNeJEm98RvIkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkMYfCIMm6JI8mGUtyxaD7kaSFZE6EQZJFwN8DFwJnAR9OctZgu5KkhWPxoBto1gBjVfU4QJIbgfXAIyf6g4evuPWl5SevufhEf5wkzUlzJQyWA/u71g8A58x2E93BcCJ0h81UQsigkjRb5koYTEmSzcDmtvpCkkdn+FZnAD/sT1dTl89Orz7dMV0GcnyzZD4fG3h8J7u5fnxvnmzDXAmDg8DKrvUVrfYyVbUV2NrrhyUZraqRXt9nrprPxzefjw08vpPdyXx8c+IGMnAfsDrJqiSnAJcAOwfckyQtGHPizKCqjiT5M+B2YBGwvar2DrgtSVow5kQYAFTVbcBts/RxPV9qmuPm8/HN52MDj+9kd9IeX6pq0D1IkgZsrtwzkCQN0IIKg/k+5UWS7UkOJfnuoHvptyQrk+xO8kiSvUkuH3RP/ZTk1CT3JvlOO76/HXRP/ZZkUZIHknxj0L30W5Inkzyc5MEko4PuZyYWzGWiNuXFfwF/SOdLbfcBH66qE/4t59mS5D3AC8ANVfWOQffTT0nOBM6sqvuT/BqwB9gwX/77JQlwWlW9kOTVwLeBy6vq7gG31jdJ/hIYAd5QVe8bdD/9lORJYKSq5vJ3DI5pIZ0ZvDTlRVX9Ajg65cW8UVXfAg4Puo8Toaqerqr72/JPgH10vrk+L1THC2311e1n3vymlmQFcDHwpUH3ooktpDCYaMqLefOPyUKSZBh4F3DPgFvpq3YZ5UHgELCrqubT8X0e+CTwywH3caIU8O9J9rSZEk46CykMNA8keT3wNeATVfXjQffTT1X1YlX9Np1v4K9JMi8u9SV5H3CoqvYMupcT6Peq6mw6My9f1i7ZnlQWUhhMacoLzV3tWvrXgC9X1dcH3c+JUlXPAbuBdQNupV/OA97frqvfCLw3yT8PtqX+qqqD7fUQ8K90LkufVBZSGDjlxUms3WDdBuyrqs8Nup9+SzKUZElbfi2dBx2+N9Cm+qSqrqyqFVU1TOf/uzur6o8H3FbfJDmtPdRAktOAtcBJ90TfggmDqjoCHJ3yYh9w03yb8iLJV4D/BN6a5ECSTYPuqY/OAz5K57fKB9vPRYNuqo/OBHYneYjOLy67qmrePYI5Ty0Dvp3kO8C9wK1V9c0B9zRtC+bRUknS5BbMmYEkaXKGgSTJMJAkGQaSJAwDSRq4fk8ymeTFrqfupvQIvU8TSdKA9XuSySQvVNXrp7OPZwaSNGATTTKZ5DeTfLPNd/QfSd52InswDCRpbtoKfLyq3g38NfDFaex7apLRJHcn2TCVHebM30CWJHW0CRl/F/iXzkwsALymbfsj4KoJdjtYVRe05TdX1cEkvwHcmeThqvrvY32mYSBJc8+rgOfaLLYv0yZpPOZEjV0T5z2e5C46U74fMwy8TCRJc0ybnv2JJB+EzkSNSd45lX2TLE1y9CziDDrzeh33LwIaBpI0YJNMMvkRYFObAG8vU//LjL8FjLb9dgPXTOXPw/poqSTJMwNJkmEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCfg/eoht2K7d/0kAAAAASUVORK5CYII=\n" + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "merged_results['diff_squared'] = np.square(merged_results.value_sim - merged_results.value_obs)\n", + "print(merged_results['diff_squared'].max())\n", + "plt.hist(merged_results.diff_squared.values, bins=100);" + ], "metadata": { "collapsed": false, "pycharm": { @@ -878,11 +914,94 @@ } } }, + { + "cell_type": "markdown", + "source": [ + "### single trip-mode convergence to expected value" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 431, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trip_mode_choice tour_type 'othdiscr' (10000 trips)\n", + "Done\n" + ] + }, + { + "data": { + "text/plain": " trip_mode drawn exact\n0 WALK_LRF 0.3210 0.322184\n1 WALK_HVY 0.2812 0.278911\n2 TNC_SINGLE 0.2170 0.219172\n3 WALK_LOC 0.1780 0.176455\n4 TNC_SHARED 0.0018 0.002318\n5 WALK 0.0008 0.000713\n6 SHARED2FREE 0.0001 0.000179\n7 SHARED3FREE 0.0001 0.000063\n20 TAXI 0.0000 0.000005", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_modedrawnexact
0WALK_LRF0.32100.322184
1WALK_HVY0.28120.278911
2TNC_SINGLE0.21700.219172
3WALK_LOC0.17800.176455
4TNC_SHARED0.00180.002318
5WALK0.00080.000713
6SHARED2FREE0.00010.000179
7SHARED3FREE0.00010.000063
20TAXI0.00000.000005
\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "(1e-08, 1)" + }, + "execution_count": 431, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAECCAYAAAD9z2x7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7D0lEQVR4nO3dd5hU1fnA8e+Z2d7ZZWGB3aV3aYKgYMFCsaLGSiyx15j4S6KmWBI1lhhji7ELJlFBJDawK9JUuvReF9hGWXYXtp/fH3dmdvrO7PTd9/M8+2Tnzp07Z2/wnTPnPec9SmuNEEKIts8U6QYIIYQIDwn4QgjRTkjAF0KIdkICvhBCtBMS8IUQop2QgC+EEO2EBHwhhGgnJOALIUQ7EReuN1JKpQIvAnXAPK31f8P13kIIIQLs4Sul3lBKlSql1jodn6yU2qSU2qqUus9y+GJgltb6JuCCQN5XCCGE/wId0pkGTLY/oJQyA/8EzgYGAVcqpQYB+cAey2mNAb6vEEIIPwUU8LXW84GDTodHA1u11tu11nXAu8AUoAgj6Af8vkIIIfwXijH8bjT35MEI9GOA54AXlFLnAh97erFS6mbgZoDU1NSRAwYM8L8FDbVw7JDx01ADygRJGZDcARIzQSn/rymEEDFi+fLl5VrrXOfjYUvaaq2rget8OO8V4BWAUaNG6WXLlgXyprBvJayeCWvfh+pSSKqDQVNgyGXQfRyY5MuGEKJtUUrtcnc8FAF/L1Bg9zjfciz8lIJuxxs/Ex+BHfNg9Xuw5n1Y8RZk5MOQn8HQy6Hz4Ig0UQghwiUUAX8p0Fcp1RMj0F8BTA3B+/jHHAd9zjJ+6p6GTZ8aPf/FL8CiZ6HTYBh6GQy5BDLzW76eEELEGBXIBihKqXeA8UBHoAR4UGv9ulLqHOAZwAy8obV+tDXXD3hIxxfV5bDuf7B6BhQtBRT0OBmGXGoM/SRnhfb9hRAiyJRSy7XWo1yOR/OOV2EJ+PYObrcM+cyEA1vBnAD9Jhnj/f0mQVxi+NoihBCtJAHfH1rDvhVG8LclezONHv/Qy6FwrCR7hRBRq10F/KamIP5NTQ2w/TvU2pmw4RNUfTU6Ix815BJjzF+SvUKIKOMp4IdtWmY43fP+amYtLwryVaeQzCQmmFZwXe0Shi9+HrXoGUn2CiFiRpsM+BMHdaagQ0pIrn2kZiAXLRzL1MHJPNpvC2rNe/DVg/DVQ5LsFUJEtbYZ8AfnMXFwXsiun52awN8+30Re1zO468Zb4MA2WDPLSPZ+fBfM/a2R5B16OfSdKMleIURUaJMBP9RuH9+bbaVVPP3lZnrlpnLe0N4w/l447R67ZO8s2PCxJHuFEFGjTSZtw6G2oZGpr/7I2r0VzLzlJIYVZDme0NhgWdlrJHupr7as7JVkrxAitNrVLJ1wKa+qZcoLi6hvbOLDO8fRJTPZ/Yl11ZaVvTNg69egG6HzccZ4vyR7hRBBJgE/RDYWH+FnLy6mR8dU3rv1JFISWhglqyozVvaumSkre4UQISEBP4S+2VjCjdOXMXFQHi/+/HhMJh/LL1uTvatnwMFtzSt7JdkrhAiABPwQe23Bdh6Zs4E7Tu/N7yb5WcPfYWXvLKgusyR7LzTG+yXZK4TwQ7taeBUJN5zck62lVfzz2230zk3j4uP9GJdXCrqNNH5sZZxnGr3/FdMl2SuECArp4QdRXUMT17zxIyt2Headm8cwsnt2gBesho1zjfF+SfYKIXwkQzphcqi6joteXERlTQMf3DGOguwgrfj1lOwdehkMvECSvUIIGwn4YbS1tIqLXlxE18xk3r99LGmJQR45c0n2JkK/iZLsFUIAEvDDbuGWcq59cwmn9cvl1WtGYfZ15o4/bMle6569kuwVQkjAj4h//7CL+z9Yy40n9+RP5w0K7Zs1NsD2ecaQj8vK3suhc4jfXwgRNWSWTgRcfWJ3tpVW8drCHfTplMYVowtD92bmOOh7lvFjn+xd/DwsesYu2XspZHYLXTuEEFFLevgh1tDYxPXTl7F4azkf3DGO47plhrcB1mTv6hmwdxmS7BWi7ZMhnQiqOFrPmU/PI79DCrNvG+v7StxgO7AN1rxnjPnbkr2TjOAvyV4h2gxPAV8yemGQmRLPH84ZyKo9h5mxbE/kGpLTG8bfB79cDjd9A6Oug93fw4yr4Km+8NFdsHMhNDVFro1CiJCRHn6YaK25/JUf2FRcyTe/OY2ctCjpTUuyV4g2R4Z0osDmkkrOeXYBFx/fjScvGRbp5riyJntXz4Bt3zSv7B16GRx3iSR7hYgRMqQTBfp1TueGU3oyc1kRy3YejHRzXCWkwtBL4apZ8JtNcPbfIC4JvnwA/jEYpp0HK96CY4cj3VIhRCtIDz/MqmsbmPD0d2Qkx/PJL08mzhwDn7mS7BUipkgPP0qkJsbxwPmD2FhcybTFOyPdHN/YJ3tv9JTsXSTJXiGinPTwI0BrzXXTlrJ0x0G+/s148jKTIt0k/7lL9mYWGMneIZdJsleICJKkbZTZdaCaCf+Yz8RBnXlh6vGRbk5gJNkrRFSRIZ0o0z0nlTvG9+GT1ftZsKUs0s0JjEuy90lJ9goRhaSHH0E19Y1MfmY+Sik++/UpJMaZI92k4PKY7L0c+k6QZK8QISI9/CiUFG/mL1OOY0d5Na98tz3SzQk+d8neXYthxs8l2StEBEgPPwrc8d8VfLWhhC/vPo3CnCDtkBWtrMne1TNg4ydQf1SSvUIEmSRto1hxRQ1n/n0eo3tm88YvTkCpCBVXCze3yd4hRj5Akr1CtJoM6USxvMwk7p7Qj283lfH5upJINyd83CZ7EyXZK0SISA8/StQ3NnH2swtIjDMx565TIt2cyLIle2fAwe2S7BXCT9LDj3LxZhNXjSlk3b4jrN93JNLNiSxbsneF+2Tvx7+SZK8QrSABP4pMGd6NBLOJWcuLIt2U6KAU5I+Es58whnx+/j70nWRM85x2Djw7FL56CErWR7qlQsQECfhRpENqAmcN6sQHq/ZS1yC9VwfWPXt/9ir8dgtc/CrkDoBFz8G/ToJ/nQyLnoWKvZFuqRBRSwJ+lLl0ZAEHq+v4ZmNppJsSvRLTjLINXpO9/4aaiki3VIioIgE/ypzStyOd0hOZtTyCWyHGkrRcGHML3PS1MeY//j44shc+uhP+1hdmXmMUd2uojXRLhYi4uEg3QDiKM5u46PhuvLZgB2WVteSmy4wUn1mTvafdC3tXGLN81r4P6z+EpCwYfKGxuKvwJDBJX0e0P/KvPgpdOrKAxibNBytlPNpexbF6/v7FJiqO1ns/0ZrsPedJ+M1G+PksY6MW52Rv6YawtFuIaCEBPwr16ZTGiMIs3lu+h2hdJ1HX0MSr87dzoCo8QyV1DU1c+tJinv9mK3PW7Afgu81lzNvkPtdRWlnDK/O30ajijLn77pK9L55oSfY+B0f2heXvECKSJOBHqUtG5rO5pIrVRdGZeHx0znoenbuBD1eFJ1D+5ZN1bC6pAmDtvgqW7zrItW8s4RdvLnU5t6a+kVOf/Ja/zt3Iqj2Hmp9wSPZutCR7E+DL++HpQZLsFW1eWAO+UupCpdSrSqkZSqmJ4XzvWHP+sK4kxkXnnPwPVu5l+ve7ANhcUhny95u1vIj//LCbW07txdjeOfyw7QB3/Hel7fkjNY5DPI/N3UBNvTGt1eMitrROlmTvN0ay97R7Jdkr2jyfA75S6g2lVKlSaq3T8clKqU1Kqa1Kqfu8XUNr/YHW+ibgVuDy1jW5fchIimfycXl8uGovNfWNkW6OzZaSSn4/ew2je2QzqnuHkAf8raVV/PF/axjbO4ffTerP0PwstpdXc7C6jnsm9wdg7d7mHvlna4uZ/v0urh/Xk6yUeNb5smo5pzec/vvmlb0jf2Gs5J3xc3iqn7Gyd9diWdkrYp4/PfxpwGT7A0opM/BP4GxgEHClUmqQUmqIUuoTp59Odi/9k+V1wotLRxZwpKaBL9dHR0G1pibNPe+vJjnBzAtTRzC4awabS6r8yjNU1tRz0YuL+NQyDu9NY5Pmnlk/kZxg5pkrhhNnNjGyewcA/nDOAC4fVQA0B/ySIzXcM+snhuZnct/ZAxjcNYP1+/0oU+Et2fvm2ZLsFTHP54CvtZ4PHHQ6PBrYqrXerrWuA94Fpmit12itz3P6KVWGJ4BPtdYrgvdntE1je+fQNTOJ96JkWOe/S3azcvdh7j9vIJ0ykuiXl05VbQP7Kmps51Qcq+fG6UtZt8/9OPjfPt/Eyt2H+dYp2frSd9t47ustDsfeXLSDFbsP89D5g+mUbmz0fuaATnx05ziuHduDnLREumQm2Xrxf/54HbUNTTx7xQgS4kwM6pLBxuJK6htb0TM3x7tJ9vaXZK+IaYGO4XcD7FcIFVmOefJL4CzgEqXUre5OUErdrJRappRaVlYW43u9BshkUvxsZD4LtpSxv+JYRNtSeqSGJz/dyLg+OVw43Pi/uF/ndAA2FzcP67zwzRa+2lDKNxtcZ8+s2nOYf/9gjP1vLa2yHV+/7whPfraRt3/cbTu29/AxnvpiE2cN7MSU4V1tx00mxdD8LNueAf06p7O5pIqvN5Qwd00xd53Zl54dUwEY3DWTuoYmtpdVB/bH25K97xs9/8lP2JK9+ulBMP18SfaKmBDWpK3W+jmt9Uit9a1a65c8nPOK1nqU1npUbm5uOJsXlS4ZmY/WMHtFZOfkPzxnA7WNTTxy4ZDmYNvJCPibLOP4O8qrmbZ4JwA7Dxx1eL3Wmkc+WU/HtESmDO/K1lJjKEhrzUMfraNJQ/GRGqpqGwB45svNNGn485TjvG4I069zGtvKqnh0zgZ656Zy0ym9bM8N6poB4PHbhtXW0irfK5SmdYITb6X6mi8YX/t3ZqZOhYoiSfaKmBBowN8LFNg9zrccE0HSPSeV0T2zmbW8KGJz8jcVV/LxT/u45dRett4zQGZKPHkZSbYe/qNzNpBgNtGvcxo7Dzj2qr/aUMqyXYe4+6x+DMvP4khNA+VVdczbVMaSnQc5tZ/x4b69rIotJZW8v6KIa0/qTresZK9t69s53ejFl1dz7+QBJMQ1/5Pu1TGVeLOyTed0p66hibOe/o5znlvg1z2Z/v1Oduou3HvgXLZc9h3c+LUke0XUCzTgLwX6KqV6KqUSgCuAjwJvlrB36ch8dpRXs3zXoZZPDoGXvttGSoKZG07u6fJcv7x0NpVUsnZvBV9tKOG28b05vrADO8ubA35Tk+bJzzbSq2Mql43Kp0+nNAC2lFby1BebKMxO4fdnDwBgW1kVT32xiZSEOG4b36fFtvW3DCuN7N6BCYM6OzwXZzbRPSeV7WWeA/7sFc35kaYm3z5QK2vqeWV+86bz324ug/xRTsneCU7J3j9LsldEnD/TMt8Bvgf6K6WKlFI3aK0bgDuBz4ENwEyt9brQNLX9OmdIF1ISzLy3LPzJ2z0Hj/LRT/uYOrqQrJQEl+f7d05ja2kV/5q3jbTEOK4Z24MeHVM5UF1nmx//9cZStpRW8auz+hJnNtkC/qvzt7Nu3xHuOrMvvXPTMJsUs1fs5fN1Jdx8ai+yU13fz9nALhlcPKIbD3sY+umdm8oX60t44rONLs81Nmletgvc+3zMk0xfvJPDR+v53+1jASg9Yjd8Y0v2vmYkey96xZLsfVaSvSLifC6eprW+0sPxucDcoLVIuEhNjOOcIV34ZPU+/nLhYBLjzGF775fnb8OsFDfajY3b69c5ndqGJuas2c/Np/YiIymeHjnGsM/O8mqG5mfx6oLtdMtK5twhXQDokplEaoKZbzeV0TUziQuHdzV649kpLNhSTnpSHNe7+TbhTkKciacvH+7xeZPlQ+Bf87bRr3MaY3rmcOhoHfFmE5tLKtlRXs21J3Vn+ve72Fl+lPwOKRw+WkdGUjwmk+sHSG1DI28u2snp/XMZUdiB/p3TXfIVNolpMOxy46eqFNbOhjUzjZW9Xz4APU8xirkNugCSMn36e4UIhJRWiBHj+uRQXdfInoPhm61TWlnDzGVF/GxkN/Iyk9ye0z/PGFKJMymuG9cDgB4dUwAjcbu66DBLdhzkunE9iDMb/9yUUvS29PKvGdt8vFeuceyKEwpISwxOIdcbT2n+4Lh7xk9MW7yTc59byMR/zOeNhTvo2TGVW07rDcCO8ip2Hahm+F++tM0mcjZ3zX4OVNfZPpB6dExxyVcAHK1rcDxgSfY6rOw9vMcx2btxDjTUBeXvFsIdCfgxojDbCKJ7DnroTYbAf3/YTUNjE7ec2tvjOX06pRFvVlwwvCtdMo0Ea/fs5h7+9MW7SEuM4/ITChxeNyAvnZQEM1eeUOhwzKTgmpN6BO1vGNk9mwfOG2R7PGNp8yziFbsPM3V0IV0yk0iON7Oj/Kht2Oz7bQdobNLUNjiucp6+eBe9clMZ17sjAD1yUtlaWsXBaiNQ7zl4lAc+XMugBz5nU7GHVcjWlb13rbQke681kr3vTm3es1eSvSIEpB5+jCjoYAn4h8IX8Oeu2c+Ynjn0sJuZ4ywlIY6Zt5xkG5cHSE4w0yUziQ37jzB/cxnnD+tKelK8w+t+N2kA15/ck8yU5uM3ndqLiYM7U2D5cAsW6zcOMBaGWcWbFRcd3w2lFD06prK1rIpNxcb0zM/WFXPVaz9y6Ggdn/36VMBYL7Bqz2EePH+QbbgnI9lo//EPf8klI/Mdah/938xVzLnrFM8NU8pI9uaPgkl/he3zjBr+q2fC8mmQWQBDLjXWAHQaGKS7IdozCfgxIjc9kcQ4E7s9jRcH2dbSKraUVnHVid1bPHdEYQeXYz1yjGRpY5PmwhGua/Fy0xNdNnfJTI5naH5Wq9vsybD8LIblZ5IQZ2LpzkPkZSRRfKSGswZ2pmOa0YbC7GS+3lBKg91Mne+3H0ApqG9sIt5s4qOf9mE2KaYMb/57zh3Shb99vgnApdCddQWwdfaPu5yAjTXZ23cC1FYZwztrZsKiZ2Dh05A3xBjvH3IJZHT1fB0hvJAhnRihlKIwO4XdYRrS+XxdMQCTBue16vU9OqbS2KTpmpnE6B7ZwWya33LSEvnwzpNtf8v95w3CpOBquw+zrlnJNDRpUhLM/OeGMbbjWhs1erTWfPzTPk7u09Fh9pD3bz9Gcv3W/yznxMe+9r3B1mTvVe8be/ZOfgJM8c1lnGVlr2glCfgxpCA7hT2HwpO0/WxtMSMKszwma1vSI8cYRjl/eFfvPdswunJ0IbNvH8u5Q7uw8v6JjO3T0facdYHXhEGdOblvRzpnNH/72F9Rw4rdh9l7+BgXDHPtXX/6q+Zhm7yMJH56cCK/mdCPo3WNVNc28MX6Ekora/lw1V7W7avgPz/sYsVuH9dUWJO9N38Ldy6XZK8IiAzpxJDC7BSW7DiI1tpruYFAFR06ypq9FbbFUK0xorADCWYTlxyfH8SWBSY1MY7jLcNP9rkDMFY0A5w/1Ajol4zM52B1He8s2cOlL33PgLx0EuJMTBjsuLgLjLUAPz0wkTizIjnejMmkbB+Uc+yqgv7q3VUAmBRcMKyrrS0+69jHSPaOvw/2Lrfs2Tvbbs/ei4zx/oITZc9e4ZYE/BhSkJ1CVW0Dh4/W08GHRUmt9dlaYzhn8nGtG84BGN0zm9UPTSQpPnxrBgJxev9cXr92FGcMMKp4/27SAKpqG3hniTGrZ2NxJaf2yyXDKfls5fwB0tXyjeGeWatdzm3SUFoZQK0d52Tvtm+N8f7VM2D5m5BZaIz1S7JXOJGAH0MKOhhBZPfBo34H/Mc+3cCYntmcMcC1h+rs83XFDOySYev1tlasBHswyjCcOdDx3jivBTitn+/F/LrYDYUN6ZbJsfpGhwqhAQV8e+Z46DfR+JFkr2iBfO+LIYWWcXF/E7eLt5Xz8nfb+d/Klpfzl1bWsGzXISa3MlnblvkT8LvaFX0b1CWDL+8+lbvOaK4NVHKkxt3LAuNLsnflfyTZ245JwI8hrZmLr7XmH19uBqCkouUg892mMrSGiW7GqtujH/9wJpmWufa9c33/xpMUb+wKFmdS3HRqT5RStumpA/LSqaxp4EBVCEsouyR77zGSvR/eYUn2XivJ3nZIhnRiSGpiHDmpCX6ttl209QBLdx4iJcFMsQ+9yo3FlSTGmWybm7R3nTOS+O534zlW3+h3ovy8oV05b2jzMEqv3DQ++/UpvLesiI3FlYx85Ct2PHZOSBPwgCXZ+wcY/3unZO8HkuxtZ+T/3RhTkJ3icz0drTX/+GozXTKTuGxUAcWW+eTebCmtok8no3KlMGSlJNjKRgRqQF6GQ5kJ64YvYWFN9p7zN6OM89T3jIVeP71rKeM8TMo4t3ES8GNMgR+LrxZsKWf5rkPcfnofCrNTqGto4tDReq+v2VJSKb37ELO/v4eqHf//KD1Sw+Kt5aFvhDXZ+7PX4HdbjTLOHfsayd4XT4SXpIxzWyQBP8YUZiez9/AxGlrYmFtrzdNfbqZbVjKXjcq3zQsv9jKOf6Smnv0VNQ51cURo/G5SfwAOHnUcQz/3+YVMfe3H8O5uZk32Xj1bkr1tnAT8GFPQIYXGJs3+FhKw8zaXsWrPYe44vQ+JcWY6ZxgB39vsEOu0Qenhh97Y3jkA/Pa9nxjz16/YsP8Ii7eVU2aZrhnWoR57kuxt0yRpG2PsyyR7qiqpteYZS+/+kpHGSldbD99LwN9i2Yy8X2fp4YeatR6P9UP27Gcd99Q9fLTepcJo2Nkne4uWGfP7177vlOy9HArGSLI3RkjAjzHWIO9tauYP2w/yU1EFj188xLapd6f0RJTyPqSzpaSKxDgT+R2CW55YuGpp4VzFsXoKvJ4RRkpBwQnGj/3K3p/edVrZezl0an05DhF68rEcY7pkJmE2Ka+JW2thrnOGdrEdizebyElN9Dqks1lm6IRNegs7er2xcAfLdx3iua+3hKlFPvKa7B1jJHsXPy/J3iglPfwYE2c20S0rmd1epmZu2H+EblnJLnVf8jITvQ7pbC2pZHTPyJYybi88zb3vmJZAeVUds1fuZfbKvQDccXqf6PwQtt+zt7IE1s02Nm/54k/wxf3Q81Rjfv/A82XP3ighPfwYVJid4nXx1cbiSgZ2yXA5npeR7HFIp7Kmnn0VNfSVhG1E/XZif5djgx74LAIt8VN6ZzjxNqdk724j2ftUP0n2RgkJ+DGoIDvZY8CvqW9ke1kVg7q4Bm5vPfwtMkMnYrY+ejZ3n9UPgHF2NfqtahuauNdN1U1v9lcc46XvtoV3eqeVNdl710q44Ss4/hrYucDYs/fv/eDjX8Ou72XP3giQgB+DCrJTOFBdR7WbqXtbSqpo0jDAbQ8/icNH66mpb3R5bmuJEfD7yhz8sMlIMkZU48wmfnlGH1Y9MIGC7BSW/PFMl3NnLNvjcsybG6cv4/FPN7L3cHg2zHHLmuw952/G/P6p70HvMy0reycbK3u//guUboxcG9sZGcOPQfZF1AbkOQb2DfuNfVTdDenYz8V3Ln28ucSooRPsDcSFZwvuOYMGSy/XZFJkpRgzdzqlt26XMavahkbbfrqvLdjBr87sG9L9E3ziUMa50hjeWT0TFv4DFvwd8oYa4/3HXQIZXVq+nmgV6eHHIOtcfHcbmq/ff4TkeLPtHHveVttuKa2id67M0AmnzJR4ctIS3T738tUjATyWqd5RXs3ROveLs+7/YK3t92mLd3Lhi4vocd8c3lmyO8AWB0liOgy7wljZ+38bYfLjYIozkr1PD4TpF8jK3hCRgB+DbIuv3Oxvu7H4CP3z0t0G7rwMz4uvjBo6MpwTLSYNzmPn4+fyt0uHkp4Uh1LYxuMbGps4/al5/PLtlbbzl+w4SI/75jDy4S+ZuazI4Vq7LB2D389e4/a99h0+xoylEfowcJvs3eWU7J0ryd4gkYAfg7JS4klLjHNJ3Gqt2bDf/QwdgM6Z7ssryAyd6JWeFM9t43ujtZG8Bfh6Y6nD/wJc9vL3AByo9j8wXvX6j9z7/hqO1HgvrBdytmTvKiPZO+JqS7L3SiPZ+8ndkuwNkIzhxyCllKVMsmPA319RQ8Wxega6maEDxmKf1AQzxRWOG29Yl/dLwjY6WbdarKptICnezC3/Xh7U628vqwZgweZyzh0aBePn9it7Jz9mrOxdPQNWvQPL3jBW9g691Ni6UVb2+kUCfowq6JDMjvJqh2Mbiz0nbMH4oOicmUTxEcehoC0lMiUzmqUkGP+ZHq1tBKfP5M/WFpPfwfda/ZuKK1m7t4KKY/Vcf3JPh2mbd7y9gnOHnhuUNgeNJHuDSgJ+jCrMTmH+ljK01rZVmxv2G8XP+ud5Dtx5GUkuSduNxZUkxcsMnWiVlmhsBl9d1+AypfbW/3ju7X/zm9PomJ5IemIcv56xig9X7WPSM/Ntz3fNSnLZ2KW61ngPT8nkiLIme4ddYbeyd4bTyt7LLSt73Xd62jsJ+DGqMCeFmvomyqpqbdP4Nuw/Qn4H15IK9vIykvhxx0GHYz8VHea4rpkyQydKWXv4Zz+7gIuP7+bTa644oYBeuc1fB+ZvLnM559b/rHA5NvjBzwHY9MhkEuPMrWlueFiTvSfeBuVbYM17RvD/8HaY83/Qb7IR/PucBXERnpIaRSRpG6Nsc/HtxvE37D/icTjHqnNmEiVHamhqMr7K1zc2sXZvBcMLskLWVhGYVLtCa7NX7PV43nNXjuCFqSPITk3g0YuGODw3vn8nv96zsiZC9fhbo2NfSfb6SHr4Mco6/LL74FFGds+mpr6RHeXVnDvE+zhmXkYSDU2aA9V15KYnsnF/JbUNTQwvzApDq0VrJMf71tM+sWc2nTKSHDZOt/r7pcP430rPHxbOjtW5rsaOei7J3m+M8X5J9tpIDz9GWRN11g3NN5dU0qQ9J2ytnHe+WrXHKKUsPfzolZzgGvCfv3KEy//X2V5W05r8HK6L2I5bwWKOh36T4JLX4Xdb4KKXjW8CC/9hKeN8iqWM8/5ItzSsJODHqKR4M50zEm118a0lFdzV0LHXxWm17co9h+mYlkC3LN9neojwSop3/c/0/GFdyUx2/IIeZ27df85/mTLY5ZjzKt45q/e7rcEUE9yu7DU7rez9L9QciXRLQ04CfgyzL5O8YX8lKQlmurcw08ZaXmG/rYd/mOEFWR7rs4vIS3JKnna1/H+4psi/0gPOK6kzk+NZeO/pXHNSD+bcdbLDc2//2Fys7YftB7jj7RU8/mlki5w1NWmXDyKtNUVedn9zYVvZOw/uXAan/s6ysvd2eKovvPeLNr2yV8bwY1hBhxR+2H4AMHr4/fPSW/zq3jEtEbNJUVJRQ8XReraXVXPxCN9mfojI6JCawAd3jKO44hi3/mcFKZYkbrXdOPu8345v8TpPXzac855fCBglme2/EQzu6rhByfsrinh/RRFpiXG24Z3SSs+b54RDrz/MBeCr/zuVPp2MqcfXvLGEBVvKefvGMYx1U1raq4594Yw/GgnfoqXGeP+62bDuf5Dcwdizd8hlbWrP3rbxV7RTBdkp7D9SQ21DIxv2H3GpnOmO2aTITTPq4v9UdBiA4QUdQtxSEajhBVlkJhtj9KluxvR7dEx1OeasU3rz3Hp3wz//vXGMyzH7sfxomab53nKjVtCuA9Us2FIOwNTXfmz9BZWCgtFw7lOWMs4zjTLOq94xyjg/13bKOEvAj2GF2SloDct3HuJITYPbTU/csU7NXLXnMErB0ALZfi4WWFfFukvi+sIasK15HGfj+nRkyR9ca/E3vz5y4aKuoXlK5ZBuxr/X0/42L/hv5C7Zm9PHKdn7QswmeyXgxzDr1Mwv1pcALSdsrfIyEimuMAJ+79w0rwu1RPQ4ahnCSU1wHIl11zN3JzMlng/vGMc3vxnv8ZxOGZ5r8c/bVMaCLa4LuLz5ac9hetw3h8Vby/16nbPZK5orgN759krqG13n1D/9xSZq6htZty9IZZVtyd7/GcneSY9Zkr1/jNlkrwT8GGYtk/zFumIABngpqWCvS6axt+1PloStiA0n9+3IlOFd+cuFxwHGTB1wvy2iJ8MKslr8hvDclSPcHi8+UsPVry/x+b0ApvxzEWAMuSxxWuHtj/ucSjt/urbY5ZznvtnKve+v5tznFrK/Isg7faV3hpNuj/lkrwT8GNYpPZGEOBP7KmooyE4m3ceeeueMJCprGzhQXccwCfgxIynezLNXjLBNoX36smGsfmhi0N/n/KFdOKWvnwlQH/zz262tep11Vbi9u95p3gvg9P65tt8/XLUPgHv83APYL9Zk712r4IYvjZW9279zXNm7+weIxH7CLZCAH8NMJmVbgOVLwtYqL7M5eTdCAn7MijebQjIcp5RiyvDAZm4t2lrOnNWO49wNrSxtMPW1H7w+/9q1J7gcsyZzQ8o+2fvbzZZk7xlGsveNSfDsUCPZW7Yp9G3xkUzLjHGF2SlsL6tucYWtPetq28Q4k9fKmqL90gH2Tn/uZtbM6X7W87Gy38pzXJ8cFm09YHu88/EoKedsTfb2m2RXxnmGUxnny+G4n0W0jLP08GOctYjaQD8Ct3WrwyHdMolv5epM0bY12A2j9HKa8rm9rKpV12zt4r5fntnX9vvDU47z+XURKw/RUrL3rSkRS/aG9b92pVSqUmqZUuq8cL5vW9Y9xwj4g7r6M6SThEnBCCmYJjxosMyCyUlN4J2bT3R4rrWBtLWlGdbsNWbdXDIyn64eSoC4SzQfZyn1HFHukr2HdkYs2etTwFdKvaGUKlVKrXU6PlkptUkptVUpdZ8Pl7oXmNmahgr3LjuhgJevHkn3nJYX3lilJMTx+i9O4NbTeoewZSKWZaYYi7yuP7mnbQjQKqGF+fjuhoOUgtpWBvy3fzQ2WP/zBYNJsqsceuaA5iGiMT2zbb8/fGHztwBPs3W01rYPtWmLdtDjvjnsdNpBLuiiINnr6xj+NOAF4C3rAaWUGfgnMAEoApYqpT4CzMBjTq+/HhgGrAc8T/QVfstIimfS4Dy/X9fa8VTRPpw/tAtaa7fltluKR3Vu5sgDVNUGVnwtyalM9MtXj7T9bj/V9BS7aaonPfaN23H+1xfu4JE5G7jqxEL+84PxgTL+qXmtygkUV9SwtbSK0T2zW/wwBJqTvQWj7co42+3Zm1UIQy6FUddDZr7f7fHGp4CvtZ6vlOrhdHg0sFVrvd34G9S7wBSt9WOAy5CNUmo8kAoMAo4ppeZqrV3+ZSilbgZuBigsLPT5DxFCBI/zTJ2pYwptPe2GRu8R/63FuxweJ8SZqGto4o1FO7htfG9OePQrh3o43izfZczdTzCbbDuyuQvKaQlxnN4/l6ljutuGOb15ZM4GAFuwt7LfMtQX28qqOPPv39ke+/2B4Zzs3fAJrLHs2TvgvKAH/EDG8LsBe+weF1mOuaW1/qPW+tfA28Cr7oK95bxXtNajtNajcnNz3Z0ihAizv140hDd/YUx/PP+FhV7P/WpDicNj+6HD05+aB8BNb3nei9fez/71PQC9cr0PWZpMijevG82EQZ1RSnFRKwsCWj8IfHWN00K0RjdrBnyWmA7DrzSSvb/ZBF3dL4ALRNinaGitp2mtPwn3+wohAmM/o2vXAc/j3Uftqng+fdkw7j6rLwPy0pk4qLMt4etc5rglzsM5Lfm/Cf38Ot/q9YU7/EpK7z3smCM4dDRIyde0TsbQT5AFEvD3AgV2j/Mtx4QQbZB9heDDR+s9ntc5o3lhX99O6SiliDebHOrfmHwIZvYrbHcf9KPmPY4F4uaucVwA9sI3W7y+9rgHP/f7A8nqcLACfogEEvCXAn2VUj2VUgnAFcBHwWmWECLa7CxvDrpmL/suHDlmBMtfn9WX47oZ04X3V9SweFvzgqn9FS3X1h9vGf4BOFjtXyC1L//8wIfG5MKHP1nP2Me+5qkvNrf4+pZ6+VW1DVxgN7R186m9ADjr6fl+tTPcfJ2W+Q7wPdBfKVWklLpBa90A3Al8DmwAZmqt14WuqUKISNI097jX7vVckXLJTiPR+qsz+9oSoOVVtdQ2OKbtbpy+1Ov7+dur96S8yviweH3hDvZ5+KC58/Q+Do9r6ryXgTjuwc9ZbbfjWFcPJaejjU8BX2t9pda6i9Y6Xmudr7V+3XJ8rta6n9a6t9b60dA2VQgRSfZlmZ2rV7rT0myXrzaUBtwmX7krwAZw2ah8nr1iOL+d1N/huL+Ly64c0zyj8KnPo6d2jjNZVy+E8MkFlnLMseiHHQdcjt02vjdPXjLMNv3UfoOXc55bwMvfbfPp2jseO8dhN7AXWlkVNBwk4AshfGIyKW45rVeL52WlxLvsqvXLM/p4ONs3fzxnYECv31bmOqvo2pN6ODy+/uSeDo8f83HTdnffZGrqGwMuQBcKEvCFED47w26FtnWYpLahkdqG5qmYh4/WuyRlEwIo0rfz8XO56dSWP2iczb3rFNvv93+w1uX5sspah8c93ewL3OO+Oexzmnr5057DLb73gPs/o+fv57a6flCoSMAXQvjMfvbLDstc/KEPfUH/P33m9XXxfu6H++r87f43zklLBQVz0hIcHp831H3Z4unf73R4fMu/3S8a+8XYHi7HyqtqXU+MIAn4Qgif2ffU40yKuoYm2+ybq177kWN17nu0Zg8JXE8rUx+d69+K19bomJbo8DglIc5taYRlOw/Zft9eVkXxkeZvL+/depLtd3eLwzaXVHKwui5qAr8EfCGEz/YedpyLX2M3lLNwaznvLNnt7mX84yv3c9/dbUYeTLeNd18R9vkrR/hW6AxYvssI+N9uKuUMu7o50647gRN6NFfpvP101/fqlJ7E8Q9/yahHvvKn2SEjAV8I4TP7sgk19U0u0x3/8sn6Fl9nL9QB376Esr3W7Nn7sWW/XKvxThVn3W03GbFNWDyQgC+E8Nlwuz2Q/zVvG/UtVM5syZoizwu4gmGUXQ/cXlZKgtvjAAvuOd3t8dkr/a8cc8Ur3vfjDTcJ+EIIn9kHytqGRo8bkztXt7zFbpbNoxc1b1BiHS6xF+npjAXZKSz701mteu2r14zy+Fyk/y6QgC+E8EO82TH5Wt/gPogNy89yeNzfbs9l+8JpjW6CoP1+ug+dP6g1zXTw8JTBDo8/uGNci69xTug++Zlvc/KdZ/7Y87QxTDhJwBdC+CzdbpxaKUW9hx5+epLj3krOcf3eyQMAeOarLS55APtx/eGFHQJpLgCd7LZofPD8QQ7DUt68f9tY2+8vzvNt1a11O8hxfXJcnqtrkIAvhIgxd53ZF4C0RLPDbk/2OjiNkfd0GuIZlp9p+73J6dPA/ltDnJeqnL6yv4Lz6lpvRnb3/GGz6ZHJbo93y0pm0X1n8Nb1Y1yecy4eFwkS8IUQfpk4qDMA7yzZ43DcvlfrXD75+MIOvH3jGMb3z+W8oV0cnm9w7uHbfWuIMwce8O2vbvLzA+T+81yHlJ66dJhD7Rxn3bKSMZsU6/8yyeF4NKy6lYAvhPCLp1r49ouoLj+hwOX5sX06Mu260aQnxTsE8gH3f8am4kp63DeHt3/c7TCkE4w8p6dKmb5IdDNX/5KRvu0zm5LgOKwlPXwhRMzxtFuV/XB+5wzv9eHNJsfQ89oCo5TCawu3Owzp/Ljdtcqlv9wlhn1l3bg9GKSHL4SIOZ5GRawbn/jCeWze+hmicBzSGdwtk0A1BLBWIKBNyXHcW7e2oSniUzMl4Ash/NLSxia+cB4WUpbUqlLKIUA7l1lujR5uqmD6KjPFcfXsm9ed4Nfr7zqzL/+90UjgXvziYnr+fm6r2xIMcS2fIoQQzYIwcYbKGseSA9YRHpNynJaZ3yEl4PcaXpDFuzefyPGtmOJpXy7hNxP6cXp/96UavPl+W+DDUsEiPXwhhF+C0cNvcFqEtH5/pXFtlMusnWA4sVeOz8XS7NkXX9te7rqJii/s9wqAyK64lYAvhPCL87z51nD+0LBuKrKppNL2YfCvnx8f8PsEKiWhefrl/1pRSwfgbrtxfAg8LxAICfhCCL+466H6u6OVt2Eha0E25/HzSAjGwq9kpzr5ofgG4ysJ+EIIv7iLV+7mq3szoIvn3aisBdniA9gWMVj8XajljvO3mc/WFrO/4piHs0Mr8ndUCBFT3I3oZKX61xtPTfC8UnXvISMYBqN3HSj7nbo8LTjz169nrGLC0/ODci1/ScAXQvjF3Rj+iAL/ZsB4S/wesyxQClaADYT93zrKS20df0VqYxQJ+EIIv6S46Z37G5x9OVv5dFZo2Q9fndovN3INCRIJ+EIIv3TPcV3IdKC6zq9reJvZaS3dEITZn0HQHPFzUj3Xuo8VEvCFEAGbv7nMr/ODMZc/HHJSmzdCiYYkcqBi/y8QQkSVjl52fbI3trfrJiEQLT17Qwe7Xv24Pv5vfG5lX/8/kiTgCyGC6rkrR/h0nrtcADSP70dL4LfOFjIFEC1n3z6O6dePDlKLWk8CvhAiqDKTfZui2dKC3WhI2oJdTiGA9phNitOiIOkrAV8IEVS+1n33FO8jW0DYlfWbRjhmiTY2aTaXVIbs+hLwhRBB5ltk9FREzLpDVbQM6TQH/NA36MnPNzLxH/PZXlYVkutLwBdCBJWvK2R/Pqa72+PWj4FoCfjWQB+OgP/52mIA9hwKTekFCfhCiIAMdZqB4uvG42cN6szGhye7HP/zx+uD0q5gCdWK36JDR12O7TxgHLtx+tKQvKcEfCFEQF6+eqTD42AFyGhJ2lrn3zc0BXcT8pOf+Nbjc/UBbMvojQR8IURA0pPi6WBXytifoQ9vp0bLkM7143oAkJoY+AaBqx+aaPvdeVpqj/vmBHz9lkjAF0IEROG4KMmfDn44xsUDdecZfdn5+LkkxXuu8Okr+y0Tj9b5NpspmCTgCyECEmdWzNtkX1rBjx5+K59ra5bvOhSW95GAL4QISGJc63u+3mrqxEDnP2h+9q/FLsdCsfetBHwhhN+6ZCY5PLYPTv4E6nYU0/32warW7aHrjQR8IYTfgjX27v0y7fvj4ECVfyWnfSEBXwjhN6+za/y6TvPZCU774ranIZ1wCXyekRCi3XEOxvaBu2dH1w1SfOE8u6c9xPub31rG+P6d3D4Xij0DJOALIfzmPKTjOIbfukAVC1M0g+2L9SV8sb7E7XOhuBthG9JRSpmUUo8qpZ5XSl0brvcVQgRfKIKz87z0WNkVy1+Xjyrw6bxeua37puSNTwFfKfWGUqpUKbXW6fhkpdQmpdRWpdR9LVxmCpAP1ANFrWuuECIaeBvSCdp7BP2K0eGvFw/x6bw8p5lQweDrkM404AXgLesBpZQZ+CcwASOAL1VKfQSYgcecXn890B9YrLV+WSk1C/g6sKYLISLFuYffVoNzKPhaa8gcqTF8rfV8pVQPp8Ojga1a6+0ASql3gSla68eA85yvoZQqAqzzjMK/plgIETQuoSgEEb+Njuj4rLyqjr6dg3vNQMbwuwF77B4XWY55MhuYpJR6Hpjv6SSl1M1KqWVKqWVlZWWeThNCRFA4evjRUi0zUtbsPRz0a4Ztlo7W+ihwgw/nvQK8AjBq1Kho2+1MCIH0vsPh8NH6oF8zkB7+XsA+3ZxvOSaEEAFr7x8qwajO6SyQgL8U6KuU6qmUSgCuAD4KTrOEENHMeVZOMGbpdMtKDvgabYnzTmLB4Ou0zHeA74H+SqkipdQNWusG4E7gc2ADMFNrvS7oLRRCRB3n8J4UH/iSnrsn9Av4Gm1Jbnpi0K/p6yydKz0cnwvMDWqLhBAx568XDeGG6csCuka82flbQ0CXizkmBU12WUvr1opBfY+gX1EI0eY5B+Ps1IQgXDP4w0Sx5KyBjnMwQ7GaWQK+EMJvzrEoFKV821e4d72nQdoL3vGawb+kEKKtc54jv/vg0SBcs31zvqe+rsj1hwR8IYTfXGvpBH7NlbsPB/2ascTkFI1lSEcI0Wa9sWiHw+P2ttLWOWdhkh6+ECIauCRYI9SOtiQ90XHSpIzhCyGiQjgCfPsb0lEsvPd0OqTEG49lSEcIEQ2kHn7wHayqI79Dii1ZKwFfCBEV2lswDofaBqNqvHW3yFAM6cietkKIgIVk+KWdfqq8fdOJzF5RFJTFbM4k4AshAib18ANnrarQPy+d358zMCTvIUM6QggRBZLigl8O2ZkEfCFEVGpvs3QykkM/4CIBXwjht3BsRdfO4n1YSMAXQvityTnih2JaZjvr4ocjZyEBXwgROB14n39cn5wgNER4IwFfCBEVDlY7btrdvvr34SEBXwjhvyD06J05LzRqZyM6YSEBXwjht1Akbdt7gA/H3y8BXwgRFZyTlu1t4VWjSyY8+CTgCyEClpIQ+Bxylx5u+4r3LNt1KOTvIQFfCBGwE3sHPsMmBGkB4UQCvhAiKoRi20ThSAK+ECIqtPddtMLx90rAF0L4LRTDL+0twLuQWTpCiGikQzAxMxy7aEUL6zaG9gqzU0L+vhLwhRB+c54yqUPQ5W+74d69rGTXD4Fgk4AvhPBbKDrfLrMy21vEDwMJ+EIIv4VkDF8ifMhJwBdC+C0UY/gutXTa8KCOu7sXjmUIEvCFEAELRo/fpbRC2433btU1NIX8PSTgCyGiQii+NcSST9cWh/w9JOALIUQ7IQFfCCHaCQn4Qgi/hWalbTsbtI8ACfhCiIAF5QNA4n3IScAXQvgtFD38OOd5mW3YtSf1iMj7SsAXQvgtFFMmu2UlB/+iUeruCf3Y+fi5YX9fCfhCCNFOSMAXQvjN05BOl8ykVl9TNkAJPQn4QoigiTNLlI5mEvCFEH5z7n0HY5WsTMsMPQn4Qgi/eRrSCSRo56Yntvq1wjcS8IUQfnOO94lxZgAKsls/02Z8/9wAWiR8EReuN1JKFQLPAQeBzVrrx8P13kKI0MrLTOKlq0ZyUq+coF1ThniCz6cevlLqDaVUqVJqrdPxyUqpTUqprUqp+1q4zBBgltb6emBEK9srhIhSk4/LI9PNXq0ievjaw58GvAC8ZT2glDID/wQmAEXAUqXUR4AZeMzp9dcDPwCzlFLXA/8OrNlCCCH85VPA11rPV0r1cDo8Gtiqtd4OoJR6F5iitX4MOM/5Gkqp3wIPWq41C3gzoJYLISImFIMtcWZJKYZaIGP43YA9do+LgDFezv8MeEgpNRXY6ekkpdTNwM2Wh1VKqU2W3zOBCqfTnY/ZP+4IlHtpTyDctSVYr/F2nqfnfLk37o5F8/3y9XXBul/ujsv98v5cJlChnvB4jl/3a8QTjo8Tn3B/no9tC9brWjonkP8mXR6rJ4L276u726Naa59+gB7AWrvHlwCv2T2+GnjB1+v5+wO80tIx+8fAsnC2JViv8Xaep+d8uTexdr98fV2w7ldL96c93y9Pz7X3++Xt+Wi9X4F8h9oLFNg9zrccC5WPfTjm7pxQaM37+Poab+d5es6Xe+PuWDTfL19fF6z75e643C/vz7X3++Xt+ai8X8ryadLyicYY/ida6+Msj+OAzcCZGIF+KTBVa70uNE31j1JqmdZ6VKTbESvkfvlH7pd/5H75J1T3y9dpme8A3wP9lVJFSqkbtNYNwJ3A58AGYGa0BHuLVyLdgBgj98s/cr/8I/fLPyG5Xz738IUQQsQ2mQclhBDthAR8IYRoJyTgCyFEO9FuAr5SKlUpNV0p9apS6ueRbk+0U0r1Ukq9blkVLVqglLrQ8m9rhlJqYqTbE+2UUgOVUi8ppWYppW6LdHtigSWGLVNKuVQy8FVMB3w/i7pdjFG87SbggrA3Ngr4c7+01tu11jdEpqXRwc/79YHl39atwOWRaG+k+Xm/NmitbwUuA8ZFor2R1oqilPcCMwN5z5gO+BhF3SbbH7Ar6nY2MAi4Uik1CGNhmLUURGMY2xhNpuH7/RKtu19/sjzfHk3Dj/ullLoAmAPMDW8zo8Y0fLxfSqkJwHqgNJA3jOmAr7Wej1Ff356tqJvWug54F5iCUesn33JOTP/dreXn/Wr3/LlfyvAE8KnWekW42xoN/P33pbX+SGt9NtAuh1j9vF/jgROBqcBNSqlWxbCwbYASRp6Kuj0HvKCUOpfwLfmOBW7vl1IqB3gUGKGU+r02qqAKz/++fgmcBWQqpfporV+KROOikKd/X+MxhlkTab89fHfc3i+t9Z0ASqlfAOVa66bWXLwtBny3tNbVwHWRbkes0FofwBiPFj7QWj+H0akQPtBazwPmRbgZMUdrPS2Q17fFoY1wF3WLdXK//CP3yz9yv/wT0vvVFgP+UqCvUqqnUioBuAL4KMJtimZyv/wj98s/cr/8E9L7FdMBP0aLukWM3C//yP3yj9wv/0TifknxNCGEaCdiuocvhBDCdxLwhRCinZCAL4QQ7YQEfCGEaCck4AshRDshAV8IIdoJCfhCCNFOSMAXQoh2QgK+EEK0E/8PfxFLENh4RckAAAAASUVORK5CYII=\n" + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_samples_single = 10000\n", + "trip_id_to_check = 1767182865 #bp.index.values #[100:103]\n", + "\n", + "_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=True,\n", + " trip_id_to_check=trip_id_to_check, num_samples=num_samples_single)\n", + "c_ = c_[['trip_mode']].reset_index(drop=True)\n", + "\n", + "temp = bp.loc[bp.index == trip_id_to_check]\n", + "temp.index = [\"exact\"]\n", + "temp = temp.T.reset_index().rename(columns={\"index\": \"trip_mode\"})\n", + "ms = (c_.value_counts() / c_.shape[0]).to_frame(\"drawn\").reset_index().merge(temp, on=\"trip_mode\", how=\"outer\").fillna(0)\n", + "ms = ms.loc[(ms.exact != 0)]\n", + "display(ms)\n", + "c_test = c_.copy()\n", + "c_test.index = np.arange(1, c_test.index.shape[0]+1)\n", + "check_mode = \"WALK_LRF\"\n", + "ms_exact = ms.loc[ms.trip_mode == check_mode].exact.values[0]\n", + "\n", + "c_test['trip_mode'] = np.where(c_test['trip_mode'] == check_mode, 1, 0)\n", + "c_test = c_test.cumsum()\n", + "c_test[\"share\"] = c_test.trip_mode / c_test.index\n", + "c_test[\"diff\"] = np.square(c_test[\"share\"] - ms_exact)\n", + "\n", + "plt.plot(c_test.index.values, c_test['diff'].values)\n", + "plt.plot(np.arange(1,num_samples_single), 1.0/np.arange(1,num_samples_single))\n", + "plt.yscale('log')\n", + "plt.xscale('log')\n", + "plt.ylim([1e-8,1])" + ], "metadata": { "collapsed": false, "pycharm": { From 7694563d6574c78920919a1bb99b26addee5cefb Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 15:48:51 +1000 Subject: [PATCH 034/135] simple siumlate mnl frozen rand ind util --- activitysim/core/logit.py | 13 + activitysim/core/simulate.py | 42 ++- notebooks/frozen_impl_dev.ipynb | 485 +++++++++++++++++++++++++++ notebooks/validate_frozen_impl.ipynb | 30 +- 4 files changed, 550 insertions(+), 20 deletions(-) create mode 100644 notebooks/frozen_impl_dev.ipynb diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index de292dc3a1..aecae4e3f9 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -221,6 +221,7 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter next_level_alts = nest_alternatives_by_name[choice_this_level] raise ValueError("This should never happen - no alternative found") + # Note: this is relatively slow due to the apply. # It could *maybe* be sped up by using the fact that the nesting structure is the same for all rows: Add ev1(0,1) to # all entries (as is currently being done). Then, at each level, pick the maximum of the available composite @@ -245,6 +246,18 @@ def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_ return choices +# TODO: integrate with nested impl above +# TODO: make everything in nested and here numpy from beginning to make choices consistent with previous impl ( +# want column index and not alternative name) +def make_choices_ru_frozen_mnl(utilities, trace_label=None): + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen_mnl') + utilities_incl_unobs = add_ev1_random(utilities) + choices = np.argmax(utilities_incl_unobs.to_numpy(), axis=1) + assert not np.isnan(choices).any(), "No choice for XXX - implement reporting" + choices = pd.Series(choices, index=utilities_incl_unobs.index) + return choices + + def make_choices(probs, trace_label=None, trace_choosers=None, allow_bad_probs=False): """ Make choices for each chooser from among a set of alternatives. diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index aba8d2340c..f233339082 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -887,7 +887,8 @@ def compute_base_probabilities(nested_probabilities, nests, spec): def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, log_alt_losers=False, want_logsums=False, trace_label=None, - trace_choice_name=None, trace_column_names=None): + trace_choice_name=None, trace_column_names=None, + choose_individual_max_utility=False): """ Run a simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -922,6 +923,8 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, This is the column label to be used in trace file csv dump of choices trace_column_names: str or list of str chooser columns to include when tracing expression_values + choose_individual_max_utility: bool + apply frozen randomness at the individual utility level Returns ------- @@ -949,25 +952,29 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, tracing.trace_df(utilities, '%s.utilities' % trace_label, column_labels=['alternative', 'utility']) - probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) - chunk.log_df(trace_label, "probs", probs) + if choose_individual_max_utility: + choices = logit.make_choices_ru_frozen_mnl(utilities, trace_label=trace_label) + rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up + else: + probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) + chunk.log_df(trace_label, "probs", probs) - del utilities - chunk.log_df(trace_label, 'utilities', None) + del utilities + chunk.log_df(trace_label, 'utilities', None) - if have_trace_targets: - # report these now in case make_choices throws error on bad_choices - tracing.trace_df(probs, '%s.probs' % trace_label, - column_labels=['alternative', 'probability']) + if have_trace_targets: + # report these now in case make_choices throws error on bad_choices + tracing.trace_df(probs, '%s.probs' % trace_label, + column_labels=['alternative', 'probability']) - if custom_chooser: - choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, - trace_label=trace_label) - else: - choices, rands = logit.make_choices(probs, trace_label=trace_label) + if custom_chooser: + choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, + trace_label=trace_label) + else: + choices, rands = logit.make_choices(probs, trace_label=trace_label) - del probs - chunk.log_df(trace_label, 'probs', None) + del probs + chunk.log_df(trace_label, 'probs', None) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, @@ -1318,7 +1325,8 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, want_logsums=want_logsums, estimator=estimator, trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, + choose_individual_max_utility=choose_individual_max_utility) else: if choose_individual_max_utility: choices = eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, diff --git a/notebooks/frozen_impl_dev.ipynb b/notebooks/frozen_impl_dev.ipynb new file mode 100644 index 0000000000..7bf198c2b5 --- /dev/null +++ b/notebooks/frozen_impl_dev.ipynb @@ -0,0 +1,485 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", + "metadata": {}, + "source": [ + "# validate results\n", + "\n", + "## TODO\n", + "what happened to tracing when I fixed probability calcs - it seems like all trip ids are attached when I add one by\n", + "hand below - why?" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.385153Z", + "iopub.status.busy": "2022-05-01T10:17:57.384881Z", + "iopub.status.idle": "2022-05-01T10:17:57.534433Z", + "shell.execute_reply": "2022-05-01T10:17:57.533096Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3dba451-1e10-403e-8614-35d57e6577f4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.536623Z", + "iopub.status.busy": "2022-05-01T10:17:57.536012Z", + "iopub.status.idle": "2022-05-01T10:17:57.542755Z", + "shell.execute_reply": "2022-05-01T10:17:57.541685Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:00.350944Z", + "iopub.status.busy": "2022-05-01T10:18:00.350730Z", + "iopub.status.idle": "2022-05-01T10:18:12.760977Z", + "shell.execute_reply": "2022-05-01T10:18:12.760013Z", + "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "from datetime import datetime\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from numpy.random import default_rng\n", + "from statsmodels.stats.gof import powerdiscrepancy\n", + "from statsmodels.stats.proportion import multinomial_proportions_confint\n", + "import numba\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from activitysim.cli import run\n", + "from activitysim.core import inject\n", + "\n", + "from fru_utils import run_trip_mode_choice, comp_mode_shares, mode_choice_for_trip" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:16.944537Z", + "iopub.status.busy": "2022-05-01T10:18:16.944291Z", + "iopub.status.idle": "2022-05-01T10:18:17.124764Z", + "shell.execute_reply": "2022-05-01T10:18:17.123725Z", + "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "pd.set_option(\"max_columns\", 500)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "outputs": [ + { + "data": { + "text/plain": "array([2, 1, 2])" + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.argmax(np.array([[1,2,3],[4,6,5],[7,8,9]]), axis=1)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 34, + "outputs": [ + { + "data": { + "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "0 c\n1 b\n2 c\ndtype: object" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", + "display(df_)\n", + "ch_ = df_.idxmax(1)\n", + "display(ch_)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:17.839947Z", + "iopub.status.busy": "2022-05-01T10:18:17.839070Z", + "iopub.status.idle": "2022-05-01T10:18:18.019676Z", + "shell.execute_reply": "2022-05-01T10:18:18.018689Z", + "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_example_mtc\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:18.894533Z", + "iopub.status.busy": "2022-05-01T10:18:18.894303Z", + "iopub.status.idle": "2022-05-01T10:18:19.078807Z", + "shell.execute_reply": "2022-05-01T10:18:19.077951Z", + "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "os.chdir(example_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:22.111723Z", + "iopub.status.busy": "2022-05-01T10:18:22.111490Z", + "iopub.status.idle": "2022-05-01T10:18:22.297437Z", + "shell.execute_reply": "2022-05-01T10:18:22.296501Z", + "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "run.add_run_args(parser)\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", + "#run.run(args) # 2mins full example run\n", + "if not inject.is_injectable('preload_injectables'):\n", + " from activitysim import abm # register abm steps and other abm-specific injectables\n", + "run.handle_standard_args(args) # possibly update injectables" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "outputs": [], + "source": [ + "from activitysim.core import inject\n", + "from activitysim.core import pipeline\n", + "from activitysim.core import config\n", + "from activitysim.core import expressions\n", + "from activitysim.core import simulate\n", + "from activitysim.abm.models.util import estimation\n", + "\n", + "def run_mandatory_tour_frequency(choose_individual_max_utility=False):\n", + " resume_after = \"cdap_simulate\"\n", + " model_name = \"mandatory_tour_frequency\"\n", + " chunk_size = 0 # test_mtc means no chunking\n", + "\n", + " pipeline.open_pipeline(resume_after)\n", + " # preload any bulky injectables (e.g. skims) not in pipeline\n", + " inject.get_injectable('preload_injectables', None)\n", + " pipeline._PIPELINE.rng().begin_step(model_name)\n", + " #step_name = model_name\n", + " args = {}\n", + " #checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + " inject.set_step_args(args)\n", + "\n", + " #network_los = inject.get_injectable('network_los')\n", + " persons_merged = inject.get_table('persons_merged')\n", + "\n", + " trace_label = 'mandatory_tour_frequency'\n", + " model_settings_file_name = 'mandatory_tour_frequency.yaml'\n", + " model_settings = config.read_model_settings(model_settings_file_name)\n", + " choosers = persons_merged.to_frame()\n", + " # filter based on results of CDAP\n", + " choosers = choosers[choosers.cdap_activity == 'M']\n", + " print(\"Running mandatory_tour_frequency with %d persons\", len(choosers))\n", + " preprocessor_settings = model_settings.get('preprocessor', None)\n", + " if preprocessor_settings:\n", + " locals_dict = {}\n", + " expressions.assign_columns(\n", + " df=choosers,\n", + " model_settings=preprocessor_settings,\n", + " locals_dict=locals_dict,\n", + " trace_label=trace_label)\n", + "\n", + " estimator = estimation.manager.begin_estimation('mandatory_tour_frequency')\n", + " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + " coefficients_df = simulate.read_model_coefficients(model_settings)\n", + " model_spec = simulate.eval_coefficients(model_spec, coefficients_df, estimator)\n", + " nest_spec = config.get_logit_model_settings(model_settings)\n", + " constants = config.get_model_constants(model_settings)\n", + " choices = simulate.simple_simulate(\n", + " choosers=choosers,\n", + " spec=model_spec,\n", + " nest_spec=nest_spec,\n", + " locals_d=constants,\n", + " chunk_size=chunk_size,\n", + " trace_label=trace_label,\n", + " trace_choice_name='mandatory_tour_frequency',\n", + " estimator=estimator,\n", + " choose_individual_max_utility=choose_individual_max_utility)\n", + "\n", + " # convert indexes to alternative names\n", + " choices = pd.Series(model_spec.columns[choices.values], index=choices.index)\n", + "\n", + " finalise = True\n", + " if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n", + "\n", + " return choices" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "#pipeline.close_pipeline()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 47, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle mandatory_tour_frequency not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running mandatory_tour_frequency with %d persons 89\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle mandatory_tour_frequency not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running mandatory_tour_frequency with %d persons 89\n" + ] + } + ], + "source": [ + "ch_prev = run_mandatory_tour_frequency(choose_individual_max_utility=False)\n", + "ch_new = run_mandatory_tour_frequency(choose_individual_max_utility=True)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 48, + "outputs": [ + { + "data": { + "text/plain": "work1 61\nschool1 22\nwork_and_school 4\nwork2 2\ndtype: int64" + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ch_prev.value_counts()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 49, + "outputs": [ + { + "data": { + "text/plain": "work1 65\nschool1 22\nwork2 2\ndtype: int64" + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ch_new.value_counts()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index c267d16757..62f8405acd 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -28,7 +28,6 @@ }, "outputs": [], "source": [ - "import numba\n", "%load_ext autoreload" ] }, @@ -161,8 +160,6 @@ "source": [ "parser = argparse.ArgumentParser()\n", "run.add_run_args(parser)\n", - "# args = parser.parse_args()\n", - "# parser.parse_args(['--sum', '7', '-1', '42'])\n", "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", "#run.run(args) # 2mins full example run\n", "if not inject.is_injectable('preload_injectables'):\n", @@ -914,6 +911,33 @@ } } }, + { + "cell_type": "code", + "execution_count": 448, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEFCAYAAAAYKqc0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAANl0lEQVR4nO3df6zd9V3H8edrMDbDXDXSGEJhZZbgGhOd3rBEjDHGH0VWIPiLOv3DNDQYWWaMMV3iP5oY8R9jljBNI4QYFwiyaajUsSWD4BLmaJE5WMVUZKGNSTvJqvgPAd/+cc/wer2nnHvP9/Tb8+7zkdxwz/fe8z3vbwsvPnmf9/l8U1VIknp5x9gFSJKGZ7hLUkOGuyQ1ZLhLUkOGuyQ1dOnYBQBcccUVtXPnzrHLkKSlcuzYsW9U1faNfnZBhPvOnTs5evTo2GVI0lJJ8vVpP7MtI0kNGe6S1JDhLkkNGe6S1NDg4Z7kA0n+NMkjSX5t6PNLkt7eTOGe5P4kp5M8v+74niQvJjmR5CBAVR2vqruAXwBuHL5kSdLbmXXl/gCwZ+2BJJcA9wI3AbuBfUl2T352C/AYcGSwSiVJM5sp3KvqKeDVdYdvAE5U1UtV9TrwEHDr5PcfraqbgI9MO2eSA0mOJjl65syZrVUvSdrQPB9iugp4Zc3jk8CHkvwYcDvwLs6xcq+qQ8AhgJWVlS1vKr/z4GNvff/yPTdv9TSS1Mrgn1CtqieBJ4c+ryRpdvNMy5wCrl7zeMfk2MyS7E1y6OzZs3OUIUlab55wfwa4Lsm1SS4D7gAe3cwJqupwVR3Ytm3bHGVIktabdRTyQeBp4PokJ5Psr6o3gLuBx4HjwMNV9cLiSpUkzWqmnntV7Zty/AhzjDsm2Qvs3bVr11ZPIUnawKjbD9iWkaTFcG8ZSWrIcJekhkYNd0chJWkx7LlLUkO2ZSSpIcNdkhqy5y5JDdlzl6SGbMtIUkOGuyQ1ZLhLUkOGuyQ15LSMJDXktIwkNWRbRpIaMtwlqSHDXZIaMtwlqSGnZSSpIadlJKkh2zKS1JDhLkkNGe6S1JDhLkkNGe6S1JDhLkkNOecuSQ055y5JDdmWkaSGDHdJashwl6SGDHdJashwl6SGDHdJashwl6SGDHdJashwl6SGDHdJasi9ZSSpIfeWkaSGbMtIUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOXLuKkSW4DbgbeC9xXVZ9bxOtIkjY288o9yf1JTid5ft3xPUleTHIiyUGAqvrrqroTuAv4xWFLliS9nc20ZR4A9qw9kOQS4F7gJmA3sC/J7jW/8juTn0uSzqOZw72qngJeXXf4BuBEVb1UVa8DDwG3ZtUfAn9bVc9udL4kB5IcTXL0zJkzW61fkrSBed9QvQp4Zc3jk5NjHwV+Avi5JHdt9MSqOlRVK1W1sn379jnLkCSttZA3VKvqE8AnFnFuSdLbm3flfgq4es3jHZNjM0myN8mhs2fPzlmGJGmtecP9GeC6JNcmuQy4A3h01idX1eGqOrBt27Y5y5AkrbWZUcgHgaeB65OcTLK/qt4A7gYeB44DD1fVC4spVZI0q5l77lW1b8rxI8CRwSqSJM1t1O0H7LlL0mKMGu723CVpMdw4TJIaWsic+6yS7AX27tq1a5Dz7Tz42Fvfv3zPzYOcU5KWkW0ZSWrItowkNWS4S1JDhrskNeScuyQ1NOq0TFUdBg6vrKzcOfS5nZyRdDGzLSNJDRnuktSQ4S5JDfmGqiQ15CdUJakh2zKS1JDhLkkNGe6S1JDhLkkNGe6S1JCjkJLUUNu9ZaZxzxlJFwPbMpLUkOEuSQ2N2pY5X9a2YiTpYuDKXZIaMtwlqSHDXZIacs5dkhpyy19JauiimJaZZv0UjR9qktSFPXdJauiiXrmv59YEkrpw5S5JDblyn8JVvKRl5spdkhoy3CWpIcNdkhoy3CWpIcNdkhpybxlJasi9ZSSpIdsyktSQ4S5JDfkJ1RlMuwern1yVdKFy5S5JDblyH4h70Ui6kLhyl6SGDHdJashwl6SG7LmfR/blJZ0vrtwlqSHDXZIaMtwlqSF77gs27dOtFzvff5AWy5W7JDU0eLgneX+S+5I8MvS5JUmzmSnck9yf5HSS59cd35PkxSQnkhwEqKqXqmr/IorVcth58LG3viSNY9aV+wPAnrUHklwC3AvcBOwG9iXZPWh1kqQtmSncq+op4NV1h28ATkxW6q8DDwG3DlyfJGkL5pmWuQp4Zc3jk8CHknwX8PvAB5N8vKr+YKMnJzkAHAC45ppr5ihjPNPaDrYjJI1t8FHIqvp34K4Zfu8QcAhgZWWlhq5Dki5m80zLnAKuXvN4x+SYJGlk86zcnwGuS3Itq6F+B/BLmzlBkr3A3l27ds1RxvKb5QM9fuhH0mbMOgr5IPA0cH2Sk0n2V9UbwN3A48Bx4OGqemEzL15Vh6vqwLZt2zZbtyTpHGZauVfVvinHjwBHBq1IkjS3UfeWuZjbMrNM1Dh1I2mrRt1bxraMJC2GG4dJUkOGuyQ1NGq4J9mb5NDZs2fHLEOS2rHnLkkN2ZaRpIYMd0lqyDn3JTfPtgRuaSD1Zc9dkhqyLSNJDRnuktSQ4S5JDRnuktSQ0zIXmHl2izwf0y9DTedcaJwcUjdOy0hSQ7ZlJKkhw12SGjLcJakhw12SGnJa5iIw7yTIhTDl4jTLMPxzvHg4LSNJDdmWkaSGDHdJashwl6SGDHdJashwl6SGDHdJasg596Zm2Tly2nHnn5fT+fw79N+XC59z7pLUkG0ZSWrIcJekhgx3SWrIcJekhgx3SWrIcJekhgx3SWrIcJekhgx3SWrIcJekhtxbRlu26Hurbvb80/Y7GWoflPX1zHKuzV5D531a3I/m/HJvGUlqyLaMJDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDU0+M06klwOfBJ4HXiyqj419GtIks5tppV7kvuTnE7y/Lrje5K8mOREkoOTw7cDj1TVncAtA9crSZrBrG2ZB4A9aw8kuQS4F7gJ2A3sS7Ib2AG8Mvm1N4cpU5K0GTO1ZarqqSQ71x2+AThRVS8BJHkIuBU4yWrAP8c5/ueR5ABwAOCaa67ZbN1aoEXfG/VcrzfUfUmn/c6Qr7Xoe4LOc0/YoX5nURbx2tP+rmb5szsffxaz1Deked5QvYr/XaHDaqhfBXwG+NkkfwIcnvbkqjpUVStVtbJ9+/Y5ypAkrTf4G6pV9V/Arw59XknS7OZZuZ8Crl7zeMfk2MyS7E1y6OzZs3OUIUlab55wfwa4Lsm1SS4D7gAe3cwJqupwVR3Ytm3bHGVIktabdRTyQeBp4PokJ5Psr6o3gLuBx4HjwMNV9cLiSpUkzWrWaZl9U44fAY5s9cWT7AX27tq1a6unkCRtYNTtB2zLSNJiuLeMJDVkuEtSQ6mqsWsgyRng61t8+hXANwYs50Lj9S23ztfX+dpgOa7vfVW14adAL4hwn0eSo1W1MnYdi+L1LbfO19f52mD5r8+2jCQ1ZLhLUkMdwv3Q2AUsmNe33DpfX+drgyW/vqXvuUuS/r8OK3dJ0jqGuyQ1tNThPuUeri1Mu29tB0muTvJEkq8leSHJx8auaUhJ3p3ky0m+Mrm+3x27pkVIckmSf0jyN2PXMrQkLyf5apLnkhwdu56tWNqe++Qerv8M/CSrd4F6BthXVV8btbCBJPlR4DXgz6vq+8auZ0hJrgSurKpnk3w7cAy4rdHfXYDLq+q1JO8Evgh8rKq+NHJpg0rym8AK8N6q+vDY9QwpycvASlVd6B9immqZV+5v3cO1ql4HvnUP1xaq6ing1bHrWISq+reqenby/X+yumX0VeNWNZxa9drk4TsnX8u5ipoiyQ7gZuDPxq5FG1vmcJ92D1ctkcmN1z8I/P3IpQxq0rJ4DjgNfL6qWl0f8MfAbwP/PXIdi1LA55IcS3Jg7GK2YpnDXUsuyXuATwO/UVX/MXY9Q6qqN6vqB1i9/eQNSdq01pJ8GDhdVcfGrmWBfqSqfhC4Cfj1SZt0qSxzuM99D1eNZ9KL/jTwqar6zNj1LEpVfRN4AtgzcilDuhG4ZdKXfgj48SR/MW5Jw6qqU5N/ngb+itU28FJZ5nCf+x6uGsfkDcf7gONV9Udj1zO0JNuTfMfk+29j9U3/fxq1qAFV1cerakdV7WT1v7svVNUvj1zWYJJcPnmjnySXAz8FLN3U2tKGe/d7uG5039qxaxrQjcCvsLrie27y9TNjFzWgK4Enkvwjq4uQz1dVu3HBxr4b+GKSrwBfBh6rqs+OXNOmLe0opCRpuqVduUuSpjPcJakhw12SGjLcJakhw12SBjb0xn9J3lwzWTbTyLfTMpI0sKE3/kvyWlW9ZzPPceUuSQPbaOO/JN+T5LOT/Wr+Lsn3LrIGw12Szo9DwEer6oeA3wI+uYnnvjvJ0SRfSnLbLE+4dAsFSpI2YbJJ3g8Df7m6+wYA75r87Hbg9zZ42qmq+unJ9++rqlNJ3g98IclXq+pfzvWahrskLd47gG9Odgr9PyYb551z87w1G5m9lORJVrfJPme425aRpAWbbGn9r0l+HlY3z0vy/bM8N8l3JvnWKv8KVvdmetu7lhnukjSwKRv/fQTYP9mQ7AVmv3PcB4Cjk+c9Adwzyy0pHYWUpIZcuUtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ/8D+hlGbxrz6lMAAAAASUVORK5CYII=\n" + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# remove where both are unavailable\n", + "plt.hist(merged_results.loc[(merged_results.value_obs != 0) & (merged_results.value_sim != 0)].diff_squared.values, bins=100);\n", + "plt.yscale('log')" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "markdown", "source": [ From 0c6f4c8eb5a81505469dab3b6d531a4361682054 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 19:03:06 +1000 Subject: [PATCH 035/135] nested now with alternative numbers, not names, change structure to be more consistent with current impl --- activitysim/core/logit.py | 35 +++- activitysim/core/simulate.py | 361 ++++++++++++++++---------------- notebooks/frozen_impl_dev.ipynb | 71 +++++-- 3 files changed, 261 insertions(+), 206 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index aecae4e3f9..f9be97d77a 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -228,9 +228,8 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter # alternatives and set the corresponding entry to 1 for each row, set all other alternatives at this level to zero. # Once the tree is walked (all alternatives have been processed), take the product of the alternatives in each # leaf's alternative list. Then pick the only alternative with entry 1, all others must be 0. -def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_choosers=None): +def make_choices_ru_frozen_nl(nested_utilities, nest_spec): """ walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" - trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') nest_utils_for_choice = add_ev1_random(nested_utilities) all_alternatives = set(nest.name for nest in each_nest(nest_spec, type='leaf')) @@ -243,22 +242,34 @@ def make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=None, trace_ ) assert not choices.isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=nest_utils_for_choice.index) + + # TODO [janzill Jun2022]: REMOVE HACK, make this numpy and positional indexes from the beginning + choices = choices.map({v: k for k,v in enumerate(nest_utils_for_choice.columns)}) + return choices -# TODO: integrate with nested impl above -# TODO: make everything in nested and here numpy from beginning to make choices consistent with previous impl ( -# want column index and not alternative name) -def make_choices_ru_frozen_mnl(utilities, trace_label=None): - trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen_mnl') +# TODO [janzill Jun2022]: integrate with nested impl above +# TODO [janzill Jun2022]: make everything in nested and here numpy from beginning to make choices consistent with +# previous impl (want column index and not alternative name) +def make_choices_ru_frozen_mnl(utilities): utilities_incl_unobs = add_ev1_random(utilities) choices = np.argmax(utilities_incl_unobs.to_numpy(), axis=1) assert not np.isnan(choices).any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=utilities_incl_unobs.index) return choices +def make_choices_ru_frozen(utilities, nest_spec=None, trace_label=None): + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen_mnl') + if nest_spec is None: + choices = make_choices_ru_frozen_mnl(utilities) + else: + choices = make_choices_ru_frozen_nl(utilities, nest_spec) + return choices + -def make_choices(probs, trace_label=None, trace_choosers=None, allow_bad_probs=False): +def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_choosers=None, allow_bad_probs=False, + choose_individual_max_utility=False): """ Make choices for each chooser from among a set of alternatives. @@ -286,8 +297,14 @@ def make_choices(probs, trace_label=None, trace_choosers=None, allow_bad_probs=F """ trace_label = tracing.extend_trace_label(trace_label, 'make_choices') - # probs should sum to 1 across each row + if choose_individual_max_utility: + choices = make_choices_ru_frozen(utilities, nest_spec, trace_label) + # TODO: rands + rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) + return choices, rands + + # probs should sum to 1 across each row BAD_PROB_THRESHOLD = 0.001 bad_probs = \ probs.sum(axis=1).sub(np.ones(len(probs.index))).abs() \ diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index f233339082..33ab7fa2da 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -952,29 +952,29 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, tracing.trace_df(utilities, '%s.utilities' % trace_label, column_labels=['alternative', 'utility']) - if choose_individual_max_utility: - choices = logit.make_choices_ru_frozen_mnl(utilities, trace_label=trace_label) - rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up - else: - probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) - chunk.log_df(trace_label, "probs", probs) + #if choose_individual_max_utility: + # choices = logit.make_choices_ru_frozen_mnl(utilities, trace_label=trace_label) + # rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up + #else: + probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) + chunk.log_df(trace_label, "probs", probs) - del utilities - chunk.log_df(trace_label, 'utilities', None) + if have_trace_targets: + # report these now in case make_choices throws error on bad_choices + tracing.trace_df(probs, '%s.probs' % trace_label, + column_labels=['alternative', 'probability']) - if have_trace_targets: - # report these now in case make_choices throws error on bad_choices - tracing.trace_df(probs, '%s.probs' % trace_label, - column_labels=['alternative', 'probability']) + if custom_chooser: + choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, + trace_label=trace_label) + else: + choices, rands = logit.make_choices(probs, utilities=utilities, trace_label=trace_label) - if custom_chooser: - choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, - trace_label=trace_label) - else: - choices, rands = logit.make_choices(probs, trace_label=trace_label) + del utilities + chunk.log_df(trace_label, 'utilities', None) - del probs - chunk.log_df(trace_label, 'probs', None) + del probs + chunk.log_df(trace_label, 'probs', None) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, @@ -984,151 +984,152 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, return choices - -def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, - log_alt_losers=False, - want_logsums=False, trace_label=None, - trace_choice_name=None, trace_column_names=None): - """ - Run a nested-logit simulation for when the model spec does not involve alternative - specific data, e.g. there are no interactions with alternative - properties and no need to sample from alternatives. - - Parameters - ---------- - choosers : pandas.DataFrame - spec : pandas.DataFrame - A table of variable specifications and coefficient values. - Variable expressions should be in the table index and the table - should have a column for each alternative. - nest_spec: - dictionary specifying nesting structure and nesting coefficients - (from the model spec yaml file) - locals_d : Dict or None - This is a dictionary of local variables that will be the environment - for an evaluation of an expression that begins with @ - custom_chooser : function(probs, choosers, spec, trace_label) returns choices, rands - custom alternative to logit.make_choices - estimator : Estimator object - called to report intermediate table results (used for estimation) - trace_label: str - This is the label to be used for trace log file entries and dump file names - when household tracing enabled. No tracing occurs if label is empty or None. - trace_choice_name: str - This is the column label to be used in trace file csv dump of choices - trace_column_names: str or list of str - chooser columns to include when tracing expression_values - - Returns - ------- - choices : pandas.Series - Index will be that of `choosers`, values will match the columns - of `spec`. - """ - - trace_label = tracing.extend_trace_label(trace_label, 'eval_nl_fixed_ru') - assert trace_label - have_trace_targets = tracing.has_trace_targets(choosers) - - logit.validate_nest_spec(nest_spec, trace_label) - - if have_trace_targets: - tracing.trace_df(choosers, '%s.choosers' % trace_label) - - raw_utilities = eval_utilities(spec, choosers, locals_d, - log_alt_losers=log_alt_losers, - trace_label=trace_label, have_trace_targets=have_trace_targets, - estimator=estimator, trace_column_names=trace_column_names) - chunk.log_df(trace_label, "raw_utilities", raw_utilities) - - if have_trace_targets: - tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label, - column_labels=['alternative', 'utility']) - - # utilities of leaves and nests - nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) - chunk.log_df(trace_label, "nested_utilities", nested_utilities) - - del raw_utilities - chunk.log_df(trace_label, 'raw_utilities', None) - - if have_trace_targets: - tracing.trace_df(nested_utilities, '%s.nested_utilities' % trace_label, - column_labels=['alternative', 'utility']) - - - ########### comparison only - # probabilities of alternatives relative to siblings sharing the same nest - nested_probabilities = \ - compute_nested_probabilities(np.exp(nested_utilities), nest_spec, trace_label=trace_label) - chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) - - if have_trace_targets: - tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, - column_labels=['alternative', 'probability']) - - base_probabilities = compute_base_probabilities(nested_probabilities, nest_spec, spec) - chunk.log_df(trace_label, "base_probabilities", base_probabilities) - - del nested_probabilities - chunk.log_df(trace_label, "nested_probabilities", None) - - if have_trace_targets: - tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label, - column_labels=['alternative', 'probability']) - - del(base_probabilities) - chunk.log_df(trace_label, "base_probabilities", None) - ########### end comparison only - - # TODO - check this is correct - if want_logsums: - # logsum of nest root - logsums = pd.Series(nested_utilities.root, index=choosers.index) - chunk.log_df(trace_label, "logsums", logsums) - - # TODO: add checks on utilities? - # # note base_probabilities could all be zero since we allowed all probs for nests to be zero - # # check here to print a clear message but make_choices will raise error if probs don't sum to 1 - # BAD_PROB_THRESHOLD = 0.001 - # no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD - # - # if no_choices.any(): - # - # logit.report_bad_choices( - # no_choices, base_probabilities, - # trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'), - # trace_choosers=choosers, - # msg="base_probabilities do not sum to one") - - # TODO: add custom_chooser - # if custom_chooser: - # choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, - # trace_label=trace_label) - # else: - - choices = logit.make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=trace_label) - - if have_trace_targets: - tracing.trace_df(choices, '%s.choices' % trace_label, - columns=[None, trace_choice_name]) - #tracing.trace_df(rands, '%s.rands' % trace_label, - # columns=[None, 'rand']) - if want_logsums: - tracing.trace_df(logsums, '%s.logsums' % trace_label, - columns=[None, 'logsum']) - - if want_logsums: - choices = choices.to_frame('choice') - choices['logsum'] = logsums - - return choices +# +# def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, +# log_alt_losers=False, +# want_logsums=False, trace_label=None, +# trace_choice_name=None, trace_column_names=None): +# """ +# Run a nested-logit simulation for when the model spec does not involve alternative +# specific data, e.g. there are no interactions with alternative +# properties and no need to sample from alternatives. +# +# Parameters +# ---------- +# choosers : pandas.DataFrame +# spec : pandas.DataFrame +# A table of variable specifications and coefficient values. +# Variable expressions should be in the table index and the table +# should have a column for each alternative. +# nest_spec: +# dictionary specifying nesting structure and nesting coefficients +# (from the model spec yaml file) +# locals_d : Dict or None +# This is a dictionary of local variables that will be the environment +# for an evaluation of an expression that begins with @ +# custom_chooser : function(probs, choosers, spec, trace_label) returns choices, rands +# custom alternative to logit.make_choices +# estimator : Estimator object +# called to report intermediate table results (used for estimation) +# trace_label: str +# This is the label to be used for trace log file entries and dump file names +# when household tracing enabled. No tracing occurs if label is empty or None. +# trace_choice_name: str +# This is the column label to be used in trace file csv dump of choices +# trace_column_names: str or list of str +# chooser columns to include when tracing expression_values +# +# Returns +# ------- +# choices : pandas.Series +# Index will be that of `choosers`, values will match the columns +# of `spec`. +# """ +# +# trace_label = tracing.extend_trace_label(trace_label, 'eval_nl_fixed_ru') +# assert trace_label +# have_trace_targets = tracing.has_trace_targets(choosers) +# +# logit.validate_nest_spec(nest_spec, trace_label) +# +# if have_trace_targets: +# tracing.trace_df(choosers, '%s.choosers' % trace_label) +# +# raw_utilities = eval_utilities(spec, choosers, locals_d, +# log_alt_losers=log_alt_losers, +# trace_label=trace_label, have_trace_targets=have_trace_targets, +# estimator=estimator, trace_column_names=trace_column_names) +# chunk.log_df(trace_label, "raw_utilities", raw_utilities) +# +# if have_trace_targets: +# tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label, +# column_labels=['alternative', 'utility']) +# +# # utilities of leaves and nests +# nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) +# chunk.log_df(trace_label, "nested_utilities", nested_utilities) +# +# del raw_utilities +# chunk.log_df(trace_label, 'raw_utilities', None) +# +# if have_trace_targets: +# tracing.trace_df(nested_utilities, '%s.nested_utilities' % trace_label, +# column_labels=['alternative', 'utility']) +# +# +# ########### comparison only +# # probabilities of alternatives relative to siblings sharing the same nest +# nested_probabilities = \ +# compute_nested_probabilities(np.exp(nested_utilities), nest_spec, trace_label=trace_label) +# chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) +# +# if have_trace_targets: +# tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, +# column_labels=['alternative', 'probability']) +# +# base_probabilities = compute_base_probabilities(nested_probabilities, nest_spec, spec) +# chunk.log_df(trace_label, "base_probabilities", base_probabilities) +# +# del nested_probabilities +# chunk.log_df(trace_label, "nested_probabilities", None) +# +# if have_trace_targets: +# tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label, +# column_labels=['alternative', 'probability']) +# +# del(base_probabilities) +# chunk.log_df(trace_label, "base_probabilities", None) +# ########### end comparison only +# +# # TODO - check this is correct +# if want_logsums: +# # logsum of nest root +# logsums = pd.Series(nested_utilities.root, index=choosers.index) +# chunk.log_df(trace_label, "logsums", logsums) +# +# # TODO: add checks on utilities? +# # # note base_probabilities could all be zero since we allowed all probs for nests to be zero +# # # check here to print a clear message but make_choices will raise error if probs don't sum to 1 +# # BAD_PROB_THRESHOLD = 0.001 +# # no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD +# # +# # if no_choices.any(): +# # +# # logit.report_bad_choices( +# # no_choices, base_probabilities, +# # trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'), +# # trace_choosers=choosers, +# # msg="base_probabilities do not sum to one") +# +# # TODO: add custom_chooser +# # if custom_chooser: +# # choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, +# # trace_label=trace_label) +# # else: +# +# choices = logit.make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=trace_label) +# +# if have_trace_targets: +# tracing.trace_df(choices, '%s.choices' % trace_label, +# columns=[None, trace_choice_name]) +# #tracing.trace_df(rands, '%s.rands' % trace_label, +# # columns=[None, 'rand']) +# if want_logsums: +# tracing.trace_df(logsums, '%s.logsums' % trace_label, +# columns=[None, 'logsum']) +# +# if want_logsums: +# choices = choices.to_frame('choice') +# choices['logsum'] = logsums +# +# return choices def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, log_alt_losers=False, want_logsums=False, trace_label=None, - trace_choice_name=None, trace_column_names=None): + trace_choice_name=None, trace_column_names=None, + choose_individual_max_utility=False): """ Run a nested-logit simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -1189,6 +1190,11 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, nested_exp_utilities = compute_nested_exp_utilities(raw_utilities, nest_spec) chunk.log_df(trace_label, "nested_exp_utilities", nested_exp_utilities) + # TODO [janzill Jun2022: combine with nested_exp_utilities somehow?] + # utilities of leaves and nests + nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) + chunk.log_df(trace_label, "nested_utilities", nested_utilities) + del raw_utilities chunk.log_df(trace_label, 'raw_utilities', None) @@ -1241,7 +1247,11 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, trace_label=trace_label) else: - choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label) + choices, rands = logit.make_choices(base_probabilities, utilities=nested_utilities, nest_spec=nest_spec, + trace_label=trace_label, choose_individual_max_utility=choose_individual_max_utility) + + del(nested_utilities) + chunk.log_df(trace_label, "nested_utilities", None) del base_probabilities chunk.log_df(trace_label, 'base_probabilities', None) @@ -1319,7 +1329,6 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, set_skim_wrapper_targets(choosers, skims) if nest_spec is None: - # TODO: add frozen individual ru for mnl choices = eval_mnl(choosers, spec, locals_d, custom_chooser, log_alt_losers=log_alt_losers, want_logsums=want_logsums, @@ -1328,20 +1337,22 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, choose_individual_max_utility=choose_individual_max_utility) else: - if choose_individual_max_utility: - choices = eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, - log_alt_losers=log_alt_losers, - want_logsums=want_logsums, - estimator=estimator, - trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) - else: - choices = eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, - log_alt_losers=log_alt_losers, - want_logsums=want_logsums, - estimator=estimator, - trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) + ##make_choices now carries choose_individual_max_utility -> incorporate there + #if choose_individual_max_utility: + # choices = eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, + # log_alt_losers=log_alt_losers, + # want_logsums=want_logsums, + # estimator=estimator, + # trace_label=trace_label, + # trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) + #else: + choices = eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, + log_alt_losers=log_alt_losers, + want_logsums=want_logsums, + estimator=estimator, + trace_label=trace_label, + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, + choose_individual_max_utility=choose_individual_max_utility) return choices diff --git a/notebooks/frozen_impl_dev.ipynb b/notebooks/frozen_impl_dev.ipynb index 7bf198c2b5..bd8ec4db33 100644 --- a/notebooks/frozen_impl_dev.ipynb +++ b/notebooks/frozen_impl_dev.ipynb @@ -336,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 50, "outputs": [ { "name": "stderr", @@ -380,19 +380,22 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 53, "outputs": [ { "data": { - "text/plain": "work1 61\nschool1 22\nwork_and_school 4\nwork2 2\ndtype: int64" + "text/plain": " count_prev count_new\nschool1 22 22.0\nwork1 61 65.0\nwork2 2 2.0\nwork_and_school 4 0.0", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
count_prevcount_new
school12222.0
work16165.0
work222.0
work_and_school40.0
\n
" }, - "execution_count": 48, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ch_prev.value_counts()" + "ch_prev.value_counts().to_frame(\"count_prev\").merge(\n", + " ch_new.value_counts().to_frame(\"count_new\"), how=\"outer\", left_index=True, right_index=True\n", + ").fillna(0)" ], "metadata": { "collapsed": false, @@ -403,20 +406,9 @@ }, { "cell_type": "code", - "execution_count": 49, - "outputs": [ - { - "data": { - "text/plain": "work1 65\nschool1 22\nwork2 2\ndtype: int64" - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ch_new.value_counts()" - ], + "execution_count": null, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { @@ -438,9 +430,44 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 58, + "outputs": [ + { + "data": { + "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", + "display(df_)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 64, + "outputs": [ + { + "data": { + "text/plain": "{'a': 0, 'b': 1, 'c': 2}" + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "{v: k for k,v in enumerate(df_.columns)}" + ], "metadata": { "collapsed": false, "pycharm": { From f725ca69093e52d25cfe3d98d0bbf7f5b477af64 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 19:05:13 +1000 Subject: [PATCH 036/135] pass through frozen_impl flag --- activitysim/core/simulate.py | 3 ++- notebooks/frozen_impl_dev.ipynb | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 33ab7fa2da..9c4999957a 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -968,7 +968,8 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, trace_label=trace_label) else: - choices, rands = logit.make_choices(probs, utilities=utilities, trace_label=trace_label) + choices, rands = logit.make_choices(probs, utilities=utilities, trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) del utilities chunk.log_df(trace_label, 'utilities', None) diff --git a/notebooks/frozen_impl_dev.ipynb b/notebooks/frozen_impl_dev.ipynb index bd8ec4db33..050c604cb8 100644 --- a/notebooks/frozen_impl_dev.ipynb +++ b/notebooks/frozen_impl_dev.ipynb @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 69, "outputs": [], "source": [ "from activitysim.core import inject\n", @@ -322,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "outputs": [], "source": [ "#pipeline.close_pipeline()" @@ -336,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 71, "outputs": [ { "name": "stderr", @@ -380,14 +380,14 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 72, "outputs": [ { "data": { "text/plain": " count_prev count_new\nschool1 22 22.0\nwork1 61 65.0\nwork2 2 2.0\nwork_and_school 4 0.0", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
count_prevcount_new
school12222.0
work16165.0
work222.0
work_and_school40.0
\n
" }, - "execution_count": 53, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } From 1a786f26ac8bfcae0855a3a64c44f2bb47cde81c Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 19:33:05 +1000 Subject: [PATCH 037/135] implement flag in settings, apply to all simple simulate models --- .../abm/models/atwork_subtour_frequency.py | 3 +- activitysim/abm/models/auto_ownership.py | 3 +- activitysim/abm/models/free_parking.py | 3 +- .../abm/models/joint_tour_composition.py | 3 +- .../abm/models/joint_tour_frequency.py | 3 +- .../abm/models/joint_tour_participation.py | 3 +- .../abm/models/mandatory_tour_frequency.py | 3 +- activitysim/abm/models/stop_frequency.py | 3 +- activitysim/abm/models/util/mode.py | 3 +- activitysim/core/logit.py | 1 - activitysim/core/simulate.py | 6 +- .../extensions/telecommute_frequency.py | 3 +- .../extensions/transit_pass_ownership.py | 3 +- .../extensions/transit_pass_subsidy.py | 3 +- .../extensions/work_from_home.py | 3 +- notebooks/frozen_impl_dev.ipynb | 149 ++++++++++++++++++ 16 files changed, 179 insertions(+), 16 deletions(-) diff --git a/activitysim/abm/models/atwork_subtour_frequency.py b/activitysim/abm/models/atwork_subtour_frequency.py index cb9d3ca4c7..ed62db14cd 100644 --- a/activitysim/abm/models/atwork_subtour_frequency.py +++ b/activitysim/abm/models/atwork_subtour_frequency.py @@ -88,7 +88,8 @@ def atwork_subtour_frequency(tours, chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='atwork_subtour_frequency', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/auto_ownership.py b/activitysim/abm/models/auto_ownership.py index 12f8fbd2bb..a6a1e2106a 100644 --- a/activitysim/abm/models/auto_ownership.py +++ b/activitysim/abm/models/auto_ownership.py @@ -57,7 +57,8 @@ def auto_ownership_simulate(households, trace_label=trace_label, trace_choice_name='auto_ownership', log_alt_losers=log_alt_losers, - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if estimator: estimator.write_choices(choices) diff --git a/activitysim/abm/models/free_parking.py b/activitysim/abm/models/free_parking.py index b144ed0543..9f37872f9a 100644 --- a/activitysim/abm/models/free_parking.py +++ b/activitysim/abm/models/free_parking.py @@ -68,7 +68,8 @@ def free_parking( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='free_parking_at_work', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) free_parking_alt = model_settings['FREE_PARKING_ALT'] choices = (choices == free_parking_alt) diff --git a/activitysim/abm/models/joint_tour_composition.py b/activitysim/abm/models/joint_tour_composition.py index f21b8f8677..3e6cbc10f4 100644 --- a/activitysim/abm/models/joint_tour_composition.py +++ b/activitysim/abm/models/joint_tour_composition.py @@ -96,7 +96,8 @@ def joint_tour_composition( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='composition', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/joint_tour_frequency.py b/activitysim/abm/models/joint_tour_frequency.py index d8930d9834..8cf63e332d 100644 --- a/activitysim/abm/models/joint_tour_frequency.py +++ b/activitysim/abm/models/joint_tour_frequency.py @@ -87,7 +87,8 @@ def joint_tour_frequency( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='joint_tour_frequency', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index c69b5093c5..23d13ac9e5 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -314,7 +314,8 @@ def joint_tour_participation( trace_label=trace_label, trace_choice_name='participation', custom_chooser=participants_chooser, - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # choice is boolean (participate or not) choice_col = model_settings.get('participation_choice', 'participate') diff --git a/activitysim/abm/models/mandatory_tour_frequency.py b/activitysim/abm/models/mandatory_tour_frequency.py index bbe1df599e..de3c6f2a5a 100644 --- a/activitysim/abm/models/mandatory_tour_frequency.py +++ b/activitysim/abm/models/mandatory_tour_frequency.py @@ -96,7 +96,8 @@ def mandatory_tour_frequency(persons_merged, chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='mandatory_tour_frequency', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/stop_frequency.py b/activitysim/abm/models/stop_frequency.py index 14ddeb0669..db967f8b77 100644 --- a/activitysim/abm/models/stop_frequency.py +++ b/activitysim/abm/models/stop_frequency.py @@ -143,7 +143,8 @@ def stop_frequency( chunk_size=chunk_size, trace_label=tracing.extend_trace_label(trace_label, segment_name), trace_choice_name='stops', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # convert indexes to alternative names choices = pd.Series(segment_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/util/mode.py b/activitysim/abm/models/util/mode.py index 37afe6d47d..c13b9cc66b 100644 --- a/activitysim/abm/models/util/mode.py +++ b/activitysim/abm/models/util/mode.py @@ -60,7 +60,8 @@ def mode_choice_simulate( trace_label=trace_label, trace_choice_name=trace_choice_name, estimator=estimator, - trace_column_names=trace_column_names) + trace_column_names=trace_column_names, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # for consistency, always return dataframe, whether or not logsums were requested if isinstance(choices, pd.Series): diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index f9be97d77a..516add8aa9 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -303,7 +303,6 @@ def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_ rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) return choices, rands - # probs should sum to 1 across each row BAD_PROB_THRESHOLD = 0.001 bad_probs = \ diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 9c4999957a..0162d20616 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1425,7 +1425,8 @@ def simple_simulate_by_chunk_id(choosers, spec, nest_spec, want_logsums=False, estimator=None, trace_label=None, - trace_choice_name=None): + trace_choice_name=None, + choose_individual_max_utility=False): """ chunk_by_chunk_id wrapper for simple_simulate """ @@ -1443,7 +1444,8 @@ def simple_simulate_by_chunk_id(choosers, spec, nest_spec, want_logsums=want_logsums, estimator=estimator, trace_label=chunk_trace_label, - trace_choice_name=trace_choice_name) + trace_choice_name=trace_choice_name, + choose_individual_max_utility=choose_individual_max_utility) result_list.append(choices) diff --git a/activitysim/examples/example_semcog/extensions/telecommute_frequency.py b/activitysim/examples/example_semcog/extensions/telecommute_frequency.py index bbe72e11f1..6246820e5d 100755 --- a/activitysim/examples/example_semcog/extensions/telecommute_frequency.py +++ b/activitysim/examples/example_semcog/extensions/telecommute_frequency.py @@ -75,7 +75,8 @@ def telecommute_frequency( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='telecommute_frequency', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py b/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py index 1b9d3c1fae..1f111e3dec 100644 --- a/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py +++ b/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py @@ -69,7 +69,8 @@ def transit_pass_ownership( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='transit_pass_ownership', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if estimator: estimator.write_choices(choices) diff --git a/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py b/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py index ddbcfbef32..ae02208c83 100644 --- a/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py +++ b/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py @@ -69,7 +69,8 @@ def transit_pass_subsidy( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='transit_pass_subsidy', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if estimator: estimator.write_choices(choices) diff --git a/activitysim/examples/example_semcog/extensions/work_from_home.py b/activitysim/examples/example_semcog/extensions/work_from_home.py index f6e2728b23..38182fcd39 100755 --- a/activitysim/examples/example_semcog/extensions/work_from_home.py +++ b/activitysim/examples/example_semcog/extensions/work_from_home.py @@ -87,7 +87,8 @@ def work_from_home( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='work_from_home', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if iterations_target_percent is not None: choices_for_filter = choices[choosers[iterations_chooser_filter]] diff --git a/notebooks/frozen_impl_dev.ipynb b/notebooks/frozen_impl_dev.ipynb index 050c604cb8..d4838d8b09 100644 --- a/notebooks/frozen_impl_dev.ipynb +++ b/notebooks/frozen_impl_dev.ipynb @@ -404,6 +404,155 @@ } } }, + { + "cell_type": "code", + "execution_count": 73, + "outputs": [ + { + "data": { + "text/plain": "{'input_table_list': [{'tablename': 'households',\n 'filename': 'households.csv',\n 'index_col': 'household_id',\n 'rename_columns': {'HHID': 'household_id',\n 'PERSONS': 'hhsize',\n 'workers': 'num_workers',\n 'VEHICL': 'auto_ownership',\n 'TAZ': 'home_zone_id'},\n 'keep_columns': ['home_zone_id',\n 'income',\n 'hhsize',\n 'HHT',\n 'auto_ownership',\n 'num_workers']},\n {'tablename': 'persons',\n 'filename': 'persons.csv',\n 'index_col': 'person_id',\n 'rename_columns': {'PERID': 'person_id'},\n 'keep_columns': ['household_id',\n 'age',\n 'PNUM',\n 'sex',\n 'pemploy',\n 'pstudent',\n 'ptype']},\n {'tablename': 'land_use',\n 'filename': 'land_use.csv',\n 'index_col': 'zone_id',\n 'rename_columns': {'TAZ': 'zone_id', 'COUNTY': 'county_id'},\n 'keep_columns': ['DISTRICT',\n 'SD',\n 'county_id',\n 'TOTHH',\n 'TOTPOP',\n 'TOTACRE',\n 'RESACRE',\n 'CIACRE',\n 'TOTEMP',\n 'AGE0519',\n 'RETEMPN',\n 'FPSEMPN',\n 'HEREMPN',\n 'OTHEMPN',\n 'AGREMPN',\n 'MWTEMPN',\n 'PRKCST',\n 'OPRKCST',\n 'area_type',\n 'HSENROLL',\n 'COLLFTE',\n 'COLLPTE',\n 'TOPOLOGY',\n 'TERMINAL']}],\n 'households_sample_size': 100,\n 'check_for_variability': False,\n 'use_shadow_pricing': False,\n 'want_dest_choice_sample_tables': False,\n 'log_alt_losers': False,\n 'chunk_size': 0,\n 'min_available_chunk_ratio': 0.05,\n 'default_initial_rows_per_chunk': 500,\n 'chunk_method': 'hybrid_uss',\n 'chunk_training_mode': 'disabled',\n 'keep_chunk_logs': True,\n 'keep_mem_logs': True,\n 'trace_hh_id': [982875],\n 'trace_od': None,\n 'resume_after': None,\n 'checkpoints': True,\n 'models': ['initialize_landuse',\n 'initialize_households',\n 'compute_accessibility',\n 'school_location',\n 'workplace_location',\n 'auto_ownership_simulate',\n 'free_parking',\n 'cdap_simulate',\n 'mandatory_tour_frequency',\n 'mandatory_tour_scheduling',\n 'joint_tour_frequency',\n 'joint_tour_composition',\n 'joint_tour_participation',\n 'joint_tour_destination',\n 'joint_tour_scheduling',\n 'non_mandatory_tour_frequency',\n 'non_mandatory_tour_destination',\n 'non_mandatory_tour_scheduling',\n 'tour_mode_choice_simulate',\n 'atwork_subtour_frequency',\n 'atwork_subtour_destination',\n 'atwork_subtour_scheduling',\n 'atwork_subtour_mode_choice',\n 'stop_frequency',\n 'trip_purpose',\n 'trip_destination',\n 'trip_purpose_and_destination',\n 'trip_scheduling',\n 'trip_mode_choice',\n 'write_data_dictionary',\n 'track_skim_usage',\n 'write_trip_matrices',\n 'write_tables'],\n 'output_tables': {'h5_store': False,\n 'action': 'include',\n 'prefix': 'final_',\n 'tables': ['checkpoints',\n 'accessibility',\n 'land_use',\n 'households',\n 'persons',\n 'tours',\n 'trips',\n 'joint_tour_participants']},\n 'urban_threshold': 4,\n 'cbd_threshold': 2,\n 'rural_threshold': 6,\n 'min_value_of_time': 1,\n 'max_value_of_time': 50,\n 'distributed_vot_mu': 0.684,\n 'distributed_vot_sigma': 0.85,\n 'household_median_value_of_time': {1: 6.01, 2: 8.81, 3: 10.44, 4: 12.86},\n 'source_file_paths': ['configs/settings.yaml']}" + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inject.get_injectable('settings')" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 74, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle mandatory_tour_frequency not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running mandatory_tour_frequency with %d persons 89\n" + ] + } + ], + "source": [ + "resume_after = \"cdap_simulate\"\n", + "model_name = \"mandatory_tour_frequency\"\n", + "chunk_size = 0 # test_mtc means no chunking\n", + "\n", + "pipeline.open_pipeline(resume_after)\n", + "# preload any bulky injectables (e.g. skims) not in pipeline\n", + "inject.get_injectable('preload_injectables', None)\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "#step_name = model_name\n", + "args = {}\n", + "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "#network_los = inject.get_injectable('network_los')\n", + "persons_merged = inject.get_table('persons_merged')\n", + "\n", + "trace_label = 'mandatory_tour_frequency'\n", + "model_settings_file_name = 'mandatory_tour_frequency.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "choosers = persons_merged.to_frame()\n", + "# filter based on results of CDAP\n", + "choosers = choosers[choosers.cdap_activity == 'M']\n", + "print(\"Running mandatory_tour_frequency with %d persons\", len(choosers))\n", + "preprocessor_settings = model_settings.get('preprocessor', None)\n", + "if preprocessor_settings:\n", + " locals_dict = {}\n", + " expressions.assign_columns(\n", + " df=choosers,\n", + " model_settings=preprocessor_settings,\n", + " locals_dict=locals_dict,\n", + " trace_label=trace_label)\n", + "\n", + "estimator = estimation.manager.begin_estimation('mandatory_tour_frequency')\n", + "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", + "coefficients_df = simulate.read_model_coefficients(model_settings)\n", + "model_spec = simulate.eval_coefficients(model_spec, coefficients_df, estimator)\n", + "nest_spec = config.get_logit_model_settings(model_settings)\n", + "constants = config.get_model_constants(model_settings)\n", + "choices = simulate.simple_simulate(\n", + " choosers=choosers,\n", + " spec=model_spec,\n", + " nest_spec=nest_spec,\n", + " locals_d=constants,\n", + " chunk_size=chunk_size,\n", + " trace_label=trace_label,\n", + " trace_choice_name='mandatory_tour_frequency',\n", + " estimator=estimator,\n", + " choose_individual_max_utility=True)\n", + "\n", + "# convert indexes to alternative names\n", + "choices = pd.Series(model_spec.columns[choices.values], index=choices.index)\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 1d4ca9a20ae3c25dbc25c6be76e83ec0cafb86e3 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 19:39:22 +1000 Subject: [PATCH 038/135] froxen rand ind util in interaction simulate --- activitysim/core/interaction_simulate.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/activitysim/core/interaction_simulate.py b/activitysim/core/interaction_simulate.py index 292c9ccaef..b0a74ab1e0 100644 --- a/activitysim/core/interaction_simulate.py +++ b/activitysim/core/interaction_simulate.py @@ -240,7 +240,8 @@ def _interaction_simulate( skims=None, locals_d=None, sample_size=None, trace_label=None, trace_choice_name=None, log_alt_losers=False, - estimator=None): + estimator=None, + choose_individual_max_utility=False): """ Run a MNL simulation in the situation in which alternatives must be merged with choosers because there are interaction terms or @@ -379,9 +380,6 @@ def _interaction_simulate( probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) chunk.log_df(trace_label, 'probs', probs) - del utilities - chunk.log_df(trace_label, 'utilities', None) - if have_trace_targets: tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), column_labels=['alternative', 'probability']) @@ -390,7 +388,12 @@ def _interaction_simulate( # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample positions, rands = \ - logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers) + logit.make_choices(probs, utilities=utilities, trace_label=trace_label, trace_choosers=choosers, + choose_individual_max_utility=choose_individual_max_utility) + + del utilities + chunk.log_df(trace_label, 'utilities', None) + chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) @@ -420,7 +423,8 @@ def interaction_simulate( log_alt_losers=False, skims=None, locals_d=None, sample_size=None, chunk_size=0, trace_label=None, trace_choice_name=None, - estimator=None): + estimator=None, + choose_individual_max_utility=False): """ Run a simulation in the situation in which alternatives must @@ -486,7 +490,8 @@ def interaction_simulate( trace_label=chunk_trace_label, trace_choice_name=trace_choice_name, log_alt_losers=log_alt_losers, - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=choose_individual_max_utility) result_list.append(choices) From f3e16107d070ba2c787308eab2a2a42baafd2dc6 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 19:41:11 +1000 Subject: [PATCH 039/135] add setting to interaction_simulate call --- activitysim/abm/models/non_mandatory_tour_frequency.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index 814bec97e6..0cf58e4618 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -231,7 +231,8 @@ def non_mandatory_tour_frequency(persons, persons_merged, chunk_size=chunk_size, trace_label='non_mandatory_tour_frequency.%s' % segment_name, trace_choice_name='non_mandatory_tour_frequency', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if estimator: estimator.write_choices(choices) From 9ac51c91c1438f183daa0736e451efed782416a0 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 15 Jun 2022 20:22:41 +1000 Subject: [PATCH 040/135] usage of interaction_sample_simulate --- activitysim/abm/models/location_choice.py | 4 +++- activitysim/abm/models/parking_location_choice.py | 3 ++- activitysim/abm/models/trip_destination.py | 4 +++- activitysim/abm/models/trip_scheduling_choice.py | 3 ++- activitysim/abm/models/util/tour_destination.py | 3 ++- activitysim/abm/models/util/tour_od.py | 3 ++- .../abm/models/util/vectorize_tour_scheduling.py | 3 ++- activitysim/core/interaction_sample_simulate.py | 15 ++++++++------- 8 files changed, 24 insertions(+), 14 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 5b3b7d1e47..4968a8dcce 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -528,7 +528,9 @@ def run_location_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name=model_settings['DEST_CHOICE_COLUMN_NAME'], - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/activitysim/abm/models/parking_location_choice.py b/activitysim/abm/models/parking_location_choice.py index 5c13969eed..c81eb89691 100644 --- a/activitysim/abm/models/parking_location_choice.py +++ b/activitysim/abm/models/parking_location_choice.py @@ -124,7 +124,8 @@ def parking_destination_simulate( locals_d=locals_dict, chunk_size=chunk_size, trace_label=trace_label, - trace_choice_name='parking_loc') + trace_choice_name='parking_loc', + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # drop any failed zero_prob destinations if (parking_locations == NO_DESTINATION).any(): diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index e4dbed4444..a5724c3540 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -699,7 +699,9 @@ def trip_destination_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name='trip_dest', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/activitysim/abm/models/trip_scheduling_choice.py b/activitysim/abm/models/trip_scheduling_choice.py index c950b0726f..b1e6f72626 100644 --- a/activitysim/abm/models/trip_scheduling_choice.py +++ b/activitysim/abm/models/trip_scheduling_choice.py @@ -261,7 +261,8 @@ def run_trip_scheduling_choice(spec, tours, skims, locals_dict, locals_d=locals_dict, trace_label=chunk_trace_label, trace_choice_name='trip_schedule_stage_1', - estimator=None + estimator=None, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) ) assert len(choices.index) == len(choosers.index) diff --git a/activitysim/abm/models/util/tour_destination.py b/activitysim/abm/models/util/tour_destination.py index 7482aeb6db..bafa6c5eed 100644 --- a/activitysim/abm/models/util/tour_destination.py +++ b/activitysim/abm/models/util/tour_destination.py @@ -632,7 +632,8 @@ def run_destination_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name='destination', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/activitysim/abm/models/util/tour_od.py b/activitysim/abm/models/util/tour_od.py index 0db7e01d74..1b47e02cdf 100644 --- a/activitysim/abm/models/util/tour_od.py +++ b/activitysim/abm/models/util/tour_od.py @@ -853,7 +853,8 @@ def run_od_simulate( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='origin_destination', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if not want_logsums: choices = choices.to_frame('choice') diff --git a/activitysim/abm/models/util/vectorize_tour_scheduling.py b/activitysim/abm/models/util/vectorize_tour_scheduling.py index 199059e2af..ca77ab6c08 100644 --- a/activitysim/abm/models/util/vectorize_tour_scheduling.py +++ b/activitysim/abm/models/util/vectorize_tour_scheduling.py @@ -600,7 +600,8 @@ def _schedule_tours( locals_d=locals_d, chunk_size=0, trace_label=tour_trace_label, - estimator=estimator + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) ) chunk.log_df(tour_trace_label, 'choices', choices) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 9f7a275ba5..573ac89b8d 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -22,7 +22,7 @@ def _interaction_sample_simulate( want_logsums, skims, locals_d, trace_label, trace_choice_name, - estimator): + estimator, choose_individual_max_utility): """ Run a MNL simulation in the situation in which alternatives must @@ -215,9 +215,6 @@ def _interaction_sample_simulate( logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) chunk.log_df(trace_label, 'logsums', logsums) - del utilities_df - chunk.log_df(trace_label, 'utilities_df', None) - if have_trace_targets: tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), column_labels=['alternative', 'probability']) @@ -232,7 +229,11 @@ def _interaction_sample_simulate( # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample positions, rands = \ - logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers) + logit.make_choices(probs, utilities_df, trace_label=trace_label, trace_choosers=choosers, + choose_individual_max_utility=choose_individual_max_utility) + + del utilities_df + chunk.log_df(trace_label, 'utilities_df', None) chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) @@ -287,7 +288,7 @@ def interaction_sample_simulate( want_logsums=False, skims=None, locals_d=None, chunk_size=0, chunk_tag=None, trace_label=None, trace_choice_name=None, - estimator=None): + estimator=None, choose_individual_max_utility=False): """ Run a simulation in the situation in which alternatives must @@ -357,7 +358,7 @@ def interaction_sample_simulate( want_logsums, skims, locals_d, chunk_trace_label, trace_choice_name, - estimator) + estimator, choose_individual_max_utility) result_list.append(choices) From e65635d1fb6c033b551e37fadc058142dd3c815b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 16 Jun 2022 12:55:01 +1000 Subject: [PATCH 041/135] interaction sample notebook --- .../frozen_impl_interaction_sample_dev.ipynb | 906 ++++++++++++++++++ 1 file changed, 906 insertions(+) create mode 100644 notebooks/frozen_impl_interaction_sample_dev.ipynb diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb new file mode 100644 index 0000000000..4d5db96446 --- /dev/null +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -0,0 +1,906 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", + "metadata": {}, + "source": [ + "# validate results\n", + "\n", + "## TODO\n", + "what happened to tracing when I fixed probability calcs - it seems like all trip ids are attached when I add one by\n", + "hand below - why?" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.385153Z", + "iopub.status.busy": "2022-05-01T10:17:57.384881Z", + "iopub.status.idle": "2022-05-01T10:17:57.534433Z", + "shell.execute_reply": "2022-05-01T10:17:57.533096Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%load_ext autoreload" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3dba451-1e10-403e-8614-35d57e6577f4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:17:57.536623Z", + "iopub.status.busy": "2022-05-01T10:17:57.536012Z", + "iopub.status.idle": "2022-05-01T10:17:57.542755Z", + "shell.execute_reply": "2022-05-01T10:17:57.541685Z", + "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:00.350944Z", + "iopub.status.busy": "2022-05-01T10:18:00.350730Z", + "iopub.status.idle": "2022-05-01T10:18:12.760977Z", + "shell.execute_reply": "2022-05-01T10:18:12.760013Z", + "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "from datetime import datetime\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from activitysim.cli import run\n", + "from activitysim.core import inject" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:16.944537Z", + "iopub.status.busy": "2022-05-01T10:18:16.944291Z", + "iopub.status.idle": "2022-05-01T10:18:17.124764Z", + "shell.execute_reply": "2022-05-01T10:18:17.123725Z", + "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "pd.set_option(\"max_columns\", 500)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "outputs": [ + { + "data": { + "text/plain": "array([2, 1, 2])" + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.argmax(np.array([[1,2,3],[4,6,5],[7,8,9]]), axis=1)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 34, + "outputs": [ + { + "data": { + "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "0 c\n1 b\n2 c\ndtype: object" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", + "display(df_)\n", + "ch_ = df_.idxmax(1)\n", + "display(ch_)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 58, + "outputs": [ + { + "data": { + "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", + "display(df_)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 64, + "outputs": [ + { + "data": { + "text/plain": "{'a': 0, 'b': 1, 'c': 2}" + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "{v: k for k,v in enumerate(df_.columns)}" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:17.839947Z", + "iopub.status.busy": "2022-05-01T10:18:17.839070Z", + "iopub.status.idle": "2022-05-01T10:18:18.019676Z", + "shell.execute_reply": "2022-05-01T10:18:18.018689Z", + "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_example_mtc\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:18.894533Z", + "iopub.status.busy": "2022-05-01T10:18:18.894303Z", + "iopub.status.idle": "2022-05-01T10:18:19.078807Z", + "shell.execute_reply": "2022-05-01T10:18:19.077951Z", + "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "os.chdir(example_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T10:18:22.111723Z", + "iopub.status.busy": "2022-05-01T10:18:22.111490Z", + "iopub.status.idle": "2022-05-01T10:18:22.297437Z", + "shell.execute_reply": "2022-05-01T10:18:22.296501Z", + "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "run.add_run_args(parser)\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", + "#run.run(args) # 2mins full example run\n", + "if not inject.is_injectable('preload_injectables'):\n", + " from activitysim import abm # register abm steps and other abm-specific injectables\n", + "run.handle_standard_args(args) # possibly update injectables" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "outputs": [], + "source": [ + "from activitysim.core import inject\n", + "from activitysim.core import pipeline\n", + "from activitysim.core import config\n", + "from activitysim.core import simulate\n", + "from activitysim.abm.models.util import estimation\n", + "from activitysim.abm.tables import shadow_pricing\n", + "from activitysim.core import interaction_simulate\n", + "from activitysim.core import logit\n", + "from activitysim.core.simulate import set_skim_wrapper_targets\n", + "from activitysim.core import chunk" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 64, + "outputs": [], + "source": [ + "def hack_make_sample_choices(\n", + " choosers, probs,\n", + " alternatives,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs,\n", + " trace_label):\n", + " assert isinstance(probs, pd.DataFrame)\n", + " assert probs.shape == (len(choosers), alternative_count)\n", + "\n", + " assert isinstance(alternatives, pd.DataFrame)\n", + " assert len(alternatives) == alternative_count\n", + "\n", + " if allow_zero_probs:\n", + " zero_probs = (probs.sum(axis=1) == 0)\n", + " if zero_probs.all():\n", + " return pd.DataFrame(columns=[alt_col_name, 'rand', 'prob', choosers.index.name])\n", + " if zero_probs.any():\n", + " # remove from sample\n", + " probs = probs[~zero_probs]\n", + " choosers = choosers[~zero_probs]\n", + "\n", + " cum_probs_array = probs.values.cumsum(axis=1)\n", + "\n", + " # alt probs in convenient layout to return prob of chose alternative\n", + " # (same layout as cum_probs_arr)\n", + " alt_probs_array = probs.values.flatten()\n", + "\n", + " # get sample_size rands for each chooser\n", + " rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size)\n", + "\n", + " # transform as we iterate over alternatives\n", + " # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr\n", + " # i.e rands[i] is a 2-D array of one alt choice rand for each chooser\n", + " rands = rands.T.reshape(sample_size, -1, 1)\n", + "\n", + " # the alternative value chosen\n", + " choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype)\n", + " # chunk log these later after we populate them...\n", + " # the probability of the chosen alternative\n", + " choice_probs_array = np.empty([sample_size, len(choosers)])\n", + " # chunk log these later after we populate them...\n", + " alts = np.tile(alternatives.index.values, len(choosers))\n", + "\n", + " # FIXME - do this all at once rather than iterate?\n", + " for i in range(sample_size):\n", + " # FIXME - do this in numpy, not pandas?\n", + " # rands for this alt in broadcastable shape\n", + " r = rands[i]\n", + "\n", + " # position of first occurrence of positive value\n", + " positions = np.argmax(cum_probs_array > r, axis=1)\n", + "\n", + " # FIXME - leave positions as numpy array, not pandas series?\n", + " # positions is series with the chosen alternative represented as a column index in probs\n", + " # which is an integer between zero and num alternatives in the alternative sample\n", + " positions = pd.Series(positions, index=probs.index)\n", + "\n", + " # need to get from an integer offset into the alternative sample to the alternative index\n", + " # that is, we want the index value of the row that is offset by rows into the\n", + " # tranche of this choosers alternatives created by cross join of alternatives and choosers\n", + "\n", + " # offsets is the offset into model_design df of first row of chooser alternatives\n", + " offsets = np.arange(len(positions)) * alternative_count\n", + "\n", + " # choices and choice_probs have one element per chooser and is in same order as choosers\n", + " choices_array[i] = np.take(alts, positions + offsets)\n", + " choice_probs_array[i] = np.take(alt_probs_array, positions + offsets)\n", + "\n", + " del positions\n", + " del offsets\n", + "\n", + " del alts\n", + " del cum_probs_array\n", + " del alt_probs_array\n", + "\n", + " # explode to one row per chooser.index, alt_zone_id\n", + " choices_df = pd.DataFrame(\n", + " {alt_col_name: choices_array.flatten(order='F'),\n", + " 'rand': rands.flatten(order='F'),\n", + " 'prob': choice_probs_array.flatten(order='F'),\n", + " choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", + " })\n", + "\n", + " del choices_array\n", + " del rands\n", + " del choice_probs_array\n", + " return choices_df" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 65, + "outputs": [], + "source": [ + "def hack_interaction_sample(\n", + " choosers, alternatives,\n", + " spec, sample_size, alt_col_name,\n", + " allow_zero_probs=False,\n", + " log_alt_losers=False,\n", + " skims=None,\n", + " locals_d=None,\n", + " trace_label=None):\n", + "\n", + " num_choosers = len(choosers.index)\n", + " assert num_choosers > 0\n", + " if len(spec.columns) > 1:\n", + " raise RuntimeError('spec must have only one column')\n", + " # if using skims, copy index into the dataframe, so it will be\n", + " # available as the \"destination\" for set_skim_wrapper_targets\n", + " if skims is not None and alternatives.index.name not in alternatives:\n", + " alternatives = alternatives.copy()\n", + " alternatives[alternatives.index.name] = alternatives.index\n", + "\n", + " chooser_index_id = interaction_simulate.ALT_CHOOSER_ID if log_alt_losers else None\n", + "\n", + " # - cross join choosers and alternatives (cartesian product)\n", + " # for every chooser, there will be a row for each alternative\n", + " # index values (non-unique) are from alternatives df\n", + " alternative_count = alternatives.shape[0]\n", + " interaction_df = \\\n", + " logit.interaction_dataset(choosers, alternatives, sample_size=alternative_count,\n", + " chooser_index_id=chooser_index_id)\n", + "\n", + " assert alternative_count == len(interaction_df.index) / len(choosers.index)\n", + "\n", + " if skims is not None:\n", + " set_skim_wrapper_targets(interaction_df, skims)\n", + "\n", + " # evaluate expressions from the spec multiply by coefficients and sum\n", + " # spec is df with one row per spec expression and one col with utility coefficient\n", + " # column names of interaction_df match spec index values\n", + " # utilities has utility value for element in the cross product of choosers and alternatives\n", + " # interaction_utilities is a df with one utility column and one row per row in interaction_df\n", + " trace_rows = trace_ids = None\n", + "\n", + " # interaction_utilities is a df with one utility column and one row per interaction_df row\n", + " interaction_utilities, trace_eval_results = interaction_simulate.eval_interaction_utilities(\n", + " spec, interaction_df, locals_d, trace_label, trace_rows, estimator=None,\n", + " log_alt_losers=log_alt_losers\n", + " )\n", + " # ########### HWM - high water mark (point of max observed memory usage)\n", + " del interaction_df\n", + "\n", + " # reshape utilities (one utility column and one row per row in interaction_utilities)\n", + " # to a dataframe with one row per chooser and one column per alternative\n", + " utilities = pd.DataFrame(\n", + " interaction_utilities.values.reshape(len(choosers), alternative_count),\n", + " index=choosers.index)\n", + " del interaction_utilities\n", + "\n", + " # convert to probabilities (utilities exponentiated and normalized to probs)\n", + " # probs is same shape as utilities, one row per chooser and one column for alternative\n", + " probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", + " trace_label=trace_label, trace_choosers=choosers)\n", + " del utilities\n", + "\n", + " if sample_size == 0:\n", + " # FIXME return full alternative set rather than sample\n", + " print(\"Estimation mode for %s using unsampled alternatives\" % (trace_label, ))\n", + " index_name = probs.index.name\n", + " choices_df = \\\n", + " pd.melt(probs.reset_index(), id_vars=[index_name])\\\n", + " .sort_values(by=index_name, kind='mergesort')\\\n", + " .set_index(index_name)\\\n", + " .rename(columns={'value': 'prob'})\\\n", + " .drop(columns='variable')\n", + " choices_df['pick_count'] = 1\n", + " choices_df.insert(0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index)))\n", + " return choices_df\n", + " else:\n", + " choices_df = hack_make_sample_choices(\n", + " choosers, probs, alternatives,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs=allow_zero_probs,\n", + " trace_label=trace_label)\n", + "\n", + " del probs\n", + "\n", + " # pick_count and pick_dup\n", + " # pick_count is number of duplicate picks\n", + " # pick_dup flag is True for all but first of duplicates\n", + " pick_group = choices_df.groupby([choosers.index.name, alt_col_name])\n", + " # number each item in each group from 0 to the length of that group - 1.\n", + " choices_df['pick_count'] = pick_group.cumcount(ascending=True)\n", + " # flag duplicate rows after first\n", + " choices_df['pick_dup'] = choices_df['pick_count'] > 0\n", + " # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", + " choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", + " # drop the duplicates\n", + " choices_df = choices_df[~choices_df['pick_dup']]\n", + " del choices_df['pick_dup']\n", + " # set index after groupby so we can trace on it\n", + " choices_df.set_index(choosers.index.name, inplace=True)\n", + "\n", + " # don't need this after tracing\n", + " del choices_df['rand']\n", + "\n", + " # - NARROW\n", + " choices_df['prob'] = choices_df['prob'].astype(np.float32)\n", + " assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty)\n", + " choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32)\n", + "\n", + " return choices_df" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 86, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "estimation bundle school_location not in settings file estimation.yaml\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "running university, 3\n", + "dropping 19 of 25 rows where size_term is zero\n", + "Running school_location with 17 persons\n", + "running highschool, 2\n", + "dropping 23 of 25 rows where size_term is zero\n", + "Running school_location with 5 persons\n", + "running gradeschool, 1\n", + "dropping 0 of 25 rows where size_term is zero\n", + "Running school_location with 17 persons\n", + "CPU times: user 219 ms, sys: 0 ns, total: 219 ms\n", + "Wall time: 264 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "resume_after = \"compute_accessibility\"\n", + "model_name = \"school_location\"\n", + "chunk_size = 0 # test_mtc means no chunking\n", + "\n", + "pipeline.open_pipeline(resume_after)\n", + "# preload any bulky injectables (e.g. skims) not in pipeline\n", + "inject.get_injectable('preload_injectables', None)\n", + "pipeline._PIPELINE.rng().begin_step(model_name)\n", + "#step_name = model_name\n", + "args = {}\n", + "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", + "inject.set_step_args(args)\n", + "\n", + "\n", + "persons_merged = inject.get_table('persons_merged')\n", + "network_los = inject.get_injectable('network_los')\n", + "households = inject.get_table('households')\n", + "persons = inject.get_table('persons')\n", + "locutor = inject.get_injectable('locutor')\n", + "\n", + "trace_label = model_name #'school_location'\n", + "model_settings_file_name = f\"{model_name}.yaml\" #'school_location.yaml'\n", + "model_settings = config.read_model_settings(model_settings_file_name)\n", + "\n", + "estimator = estimation.manager.begin_estimation(model_name)\n", + "\n", + "# iterate_location_choice(\n", + "# model_settings,\n", + "# persons_merged, persons, households,\n", + "# network_los,\n", + "# estimator,\n", + "# chunk_size, trace_hh_id, locutor, trace_label\n", + "# )\n", + "\n", + "chunk_tag = trace_label\n", + "\n", + "# boolean to filter out persons not needing location modeling (e.g. is_worker, is_student)\n", + "chooser_filter_column = model_settings['CHOOSER_FILTER_COLUMN_NAME']\n", + "\n", + "dest_choice_column_name = model_settings['DEST_CHOICE_COLUMN_NAME']\n", + "logsum_column_name = model_settings.get('DEST_CHOICE_LOGSUM_COLUMN_NAME')\n", + "\n", + "sample_table_name = model_settings.get('DEST_CHOICE_SAMPLE_TABLE_NAME')\n", + "want_sample_table = config.setting('want_dest_choice_sample_tables') and sample_table_name is not None\n", + "\n", + "persons_merged_df = persons_merged.to_frame()\n", + "\n", + "persons_merged_df = persons_merged_df[persons_merged_df[chooser_filter_column]]\n", + "\n", + "persons_merged_df.sort_index(inplace=True) # interaction_sample expects chooser index to be monotonic increasing\n", + "\n", + "# chooser segmentation allows different sets coefficients for e.g. different income_segments or tour_types\n", + "chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']\n", + "\n", + "assert chooser_segment_column in persons_merged_df, f\"CHOOSER_SEGMENT_COLUMN '{chooser_segment_column}' not in \" \\\n", + " f\"persons_merged table.\"\n", + "\n", + "shadow_price_calculator = shadow_pricing.load_shadow_price_calculator(model_settings)\n", + "\n", + "chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']\n", + "\n", + "# maps segment names to compact (integer) ids\n", + "segment_ids = model_settings['SEGMENT_IDS']\n", + "\n", + "#choices_list = []\n", + "sample_list = []\n", + "for segment_name, segment_id in segment_ids.items():\n", + " print(f\"running {segment_name}, {segment_id}\")\n", + " choosers = persons_merged_df[persons_merged_df[chooser_segment_column] == segment_id]\n", + "\n", + " # size_term and shadow price adjustment - one row per zone\n", + " dest_size_terms = shadow_price_calculator.dest_size_terms(segment_name)\n", + "\n", + " assert dest_size_terms.index.is_monotonic_increasing, f\"shadow_price_calculator.dest_size_terms({segment_name}) \" \\\n", + " f\"not monotonic_increasing\"\n", + " if choosers.shape[0] == 0:\n", + " print(f\"{trace_label} skipping segment {segment_name}: no choosers\")\n", + " continue\n", + "\n", + " print(f\"dropping {(~(dest_size_terms.size_term > 0)).sum()} \"\n", + " f\"of {len(dest_size_terms)} rows where size_term is zero\")\n", + " dest_size_terms = dest_size_terms[dest_size_terms.size_term > 0]\n", + "\n", + " chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS']\n", + " choosers_location_sample = choosers[chooser_columns]\n", + " skim_dict = network_los.get_default_skim_dict()\n", + " skims = skim_dict.wrap('home_zone_id', 'zone_id')\n", + " alt_dest_col_name = model_settings['ALT_DEST_COL_NAME']\n", + "\n", + " assert not choosers_location_sample.empty\n", + " print(\"Running %s with %d persons\" % (trace_label, len(choosers_location_sample.index)))\n", + " sample_size = model_settings[\"SAMPLE_SIZE\"]\n", + " locals_d = {\n", + " 'skims': skims,\n", + " 'segment_size': segment_name\n", + " }\n", + " constants = config.get_model_constants(model_settings)\n", + " locals_d.update(constants)\n", + " spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC',\n", + " segment_name=segment_name, estimator=estimator)\n", + " ### choices = interaction_sample(\n", + " alt_col_name=alt_dest_col_name\n", + " allow_zero_probs=False\n", + " log_alt_losers=False\n", + "\n", + " # we return alternatives ordered in (index, alt_col_name)\n", + " # if choosers index is not ordered, it is probably a mistake, since the alts wont line up\n", + " assert alt_col_name is not None\n", + " assert choosers.index.is_monotonic_increasing\n", + "\n", + " # FIXME - legacy logic - not sure this is needed or even correct?\n", + " sample_size = min(sample_size, len(dest_size_terms.index))\n", + "\n", + " result_list = []\n", + " for i, chooser_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers(choosers_location_sample, chunk_size, trace_label,\n", + " chunk_tag):\n", + "\n", + " ### choices = hack_interaction_sample\n", + " # chooser = chooser_chunk\n", + " #alternatives = dest_size_terms\n", + " #trace_label=chunk_trace_label\n", + "\n", + " num_choosers = len(chooser_chunk.index)\n", + " assert num_choosers > 0\n", + " if len(spec.columns) > 1:\n", + " raise RuntimeError('spec must have only one column')\n", + " # if using skims, copy index into the dataframe, so it will be\n", + " # available as the \"destination\" for set_skim_wrapper_targets\n", + " if skims is not None and dest_size_terms.index.name not in dest_size_terms:\n", + " dest_size_terms = dest_size_terms.copy()\n", + " dest_size_terms[dest_size_terms.index.name] = dest_size_terms.index\n", + "\n", + " chooser_index_id = interaction_simulate.ALT_CHOOSER_ID if log_alt_losers else None\n", + "\n", + " # - cross join choosers and alternatives (cartesian product)\n", + " # for every chooser, there will be a row for each alternative\n", + " # index values (non-unique) are from alternatives df\n", + " alternative_count = dest_size_terms.shape[0]\n", + " interaction_df =\\\n", + " logit.interaction_dataset(chooser_chunk, dest_size_terms, sample_size=alternative_count,\n", + " chooser_index_id=chooser_index_id)\n", + "\n", + " assert alternative_count == len(interaction_df.index) / len(chooser_chunk.index)\n", + "\n", + " if skims is not None:\n", + " set_skim_wrapper_targets(interaction_df, skims)\n", + "\n", + " # evaluate expressions from the spec multiply by coefficients and sum\n", + " # spec is df with one row per spec expression and one col with utility coefficient\n", + " # column names of interaction_df match spec index values\n", + " # utilities has utility value for element in the cross product of choosers and alternatives\n", + " # interaction_utilities is a df with one utility column and one row per row in interaction_df\n", + " trace_rows = trace_ids = None\n", + "\n", + " # interaction_utilities is a df with one utility column and one row per interaction_df row\n", + " interaction_utilities, trace_eval_results = interaction_simulate.eval_interaction_utilities(\n", + " spec, interaction_df, locals_d, chunk_trace_label, trace_rows, estimator=None,\n", + " log_alt_losers=log_alt_losers\n", + " )\n", + " # ########### HWM - high water mark (point of max observed memory usage)\n", + " #del interaction_df\n", + "\n", + " # reshape utilities (one utility column and one row per row in interaction_utilities)\n", + " # to a dataframe with one row per chooser and one column per alternative\n", + " utilities = pd.DataFrame(\n", + " interaction_utilities.values.reshape(len(chooser_chunk), alternative_count),\n", + " index=chooser_chunk.index)\n", + " #del interaction_utilities\n", + "\n", + " # convert to probabilities (utilities exponentiated and normalized to probs)\n", + " # probs is same shape as utilities, one row per chooser and one column for alternative\n", + " probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", + " trace_label=chunk_trace_label, trace_choosers=chooser_chunk)\n", + " #del utilities\n", + "\n", + " choices_df = hack_make_sample_choices(\n", + " chooser_chunk, probs, dest_size_terms,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs=allow_zero_probs,\n", + " trace_label=chunk_trace_label)\n", + "\n", + " # pick_count and pick_dup\n", + " # pick_count is number of duplicate picks\n", + " # pick_dup flag is True for all but first of duplicates\n", + " pick_group = choices_df.groupby([choosers.index.name, alt_col_name])\n", + " # number each item in each group from 0 to the length of that group - 1.\n", + " choices_df['pick_count'] = pick_group.cumcount(ascending=True)\n", + " # flag duplicate rows after first\n", + " choices_df['pick_dup'] = choices_df['pick_count'] > 0\n", + " # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", + " choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", + " # drop the duplicates\n", + " choices_df = choices_df[~choices_df['pick_dup']]\n", + " del choices_df['pick_dup']\n", + " # set index after groupby so we can trace on it\n", + " choices_df.set_index(choosers.index.name, inplace=True)\n", + " # don't need this after tracing\n", + " del choices_df['rand']\n", + " # - NARROW\n", + " choices_df['prob'] = choices_df['prob'].astype(np.float32)\n", + " assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty)\n", + " choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32)\n", + "\n", + "\n", + "\n", + "\n", + " if choices_df.shape[0] > 0:\n", + " result_list.append(choices_df)\n", + "\n", + " if len(result_list) > 1:\n", + " choices_df = pd.concat(result_list)\n", + " assert allow_zero_probs or (len(choosers_location_sample.index) == len(np.unique(choices_df.index.values))), \\\n", + " \"what is this\"\n", + " # keep alts in canonical order so choices based on their probs are stable across runs\n", + " choices_df = choices_df.sort_values(by=alt_col_name).sort_index(kind='mergesort')\n", + "\n", + " sample_list.append(choices_df)\n", + "\n", + "finalise = True\n", + "if finalise:\n", + " inject.set_step_args(None)\n", + " #\n", + " pipeline._PIPELINE.rng().end_step(model_name)\n", + " pipeline.add_checkpoint(model_name)\n", + " if not pipeline.intermediate_checkpoint():\n", + " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", + "\n", + " pipeline.close_pipeline()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 91, + "outputs": [ + { + "data": { + "text/plain": "10" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": "person_id\n386008 10\n418442 10\n595686 10\n644292 10\n644478 10\n1958678 10\n2159059 10\n2219998 10\n2458502 10\n2458503 10\n2566700 10\n2566701 10\n2566702 10\n2877287 10\n3596365 10\n3891104 10\n4171620 10\nName: pick_count, dtype: uint32" + }, + "execution_count": 91, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "display(sample_size)\n", + "sample_list[2].groupby('person_id').pick_count.sum()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 71, + "outputs": [ + { + "ename": "AssertionError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mAssertionError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/tmp/ipykernel_10980/63825533.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 43\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 44\u001B[0m \u001B[0;31m# interaction_utilities is a df with one utility column and one row per interaction_df row\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 45\u001B[0;31m interaction_utilities, trace_eval_results = interaction_simulate.eval_interaction_utilities(\n\u001B[0m\u001B[1;32m 46\u001B[0m \u001B[0mspec\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0minteraction_df\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlocals_d\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrace_label\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrace_rows\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mestimator\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mNone\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 47\u001B[0m \u001B[0mlog_alt_losers\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mlog_alt_losers\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/interaction_simulate.py\u001B[0m in \u001B[0;36meval_interaction_utilities\u001B[0;34m(spec, df, locals_d, trace_label, trace_rows, estimator, log_alt_losers)\u001B[0m\n\u001B[1;32m 66\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"Running eval_interaction_utilities on %s rows\"\u001B[0m \u001B[0;34m%\u001B[0m \u001B[0mdf\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mshape\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 67\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 68\u001B[0;31m \u001B[0;32mwith\u001B[0m \u001B[0mchunk\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchunk_log\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrace_label\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 69\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 70\u001B[0m \u001B[0;32massert\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mspec\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcolumns\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/.pyenv/versions/3.8.2/lib/python3.8/contextlib.py\u001B[0m in \u001B[0;36m__enter__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 111\u001B[0m \u001B[0;32mdel\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkwds\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfunc\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 112\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 113\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mnext\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgen\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 114\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mStopIteration\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 115\u001B[0m \u001B[0;32mraise\u001B[0m \u001B[0mRuntimeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"generator didn't yield\"\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/chunk.py\u001B[0m in \u001B[0;36mchunk_log\u001B[0;34m(trace_label, chunk_tag, base)\u001B[0m\n\u001B[1;32m 927\u001B[0m \u001B[0;31m# avoids breaking the assertion below.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 928\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 929\u001B[0;31m \u001B[0;32massert\u001B[0m \u001B[0mbase\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mCHUNK_SIZERS\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 930\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 931\u001B[0m \u001B[0mtrace_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34mf\"{trace_label}.chunk_log\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mAssertionError\u001B[0m: " + ] + } + ], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 18, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 9950f846fe56926df8cf9fd167155e38fb104678 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 16 Jun 2022 13:28:45 +1000 Subject: [PATCH 042/135] interaction_sample dev work --- .../frozen_impl_interaction_sample_dev.ipynb | 206 +++++++----------- 1 file changed, 78 insertions(+), 128 deletions(-) diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index 4d5db96446..e82c93c057 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -412,129 +412,7 @@ }, { "cell_type": "code", - "execution_count": 65, - "outputs": [], - "source": [ - "def hack_interaction_sample(\n", - " choosers, alternatives,\n", - " spec, sample_size, alt_col_name,\n", - " allow_zero_probs=False,\n", - " log_alt_losers=False,\n", - " skims=None,\n", - " locals_d=None,\n", - " trace_label=None):\n", - "\n", - " num_choosers = len(choosers.index)\n", - " assert num_choosers > 0\n", - " if len(spec.columns) > 1:\n", - " raise RuntimeError('spec must have only one column')\n", - " # if using skims, copy index into the dataframe, so it will be\n", - " # available as the \"destination\" for set_skim_wrapper_targets\n", - " if skims is not None and alternatives.index.name not in alternatives:\n", - " alternatives = alternatives.copy()\n", - " alternatives[alternatives.index.name] = alternatives.index\n", - "\n", - " chooser_index_id = interaction_simulate.ALT_CHOOSER_ID if log_alt_losers else None\n", - "\n", - " # - cross join choosers and alternatives (cartesian product)\n", - " # for every chooser, there will be a row for each alternative\n", - " # index values (non-unique) are from alternatives df\n", - " alternative_count = alternatives.shape[0]\n", - " interaction_df = \\\n", - " logit.interaction_dataset(choosers, alternatives, sample_size=alternative_count,\n", - " chooser_index_id=chooser_index_id)\n", - "\n", - " assert alternative_count == len(interaction_df.index) / len(choosers.index)\n", - "\n", - " if skims is not None:\n", - " set_skim_wrapper_targets(interaction_df, skims)\n", - "\n", - " # evaluate expressions from the spec multiply by coefficients and sum\n", - " # spec is df with one row per spec expression and one col with utility coefficient\n", - " # column names of interaction_df match spec index values\n", - " # utilities has utility value for element in the cross product of choosers and alternatives\n", - " # interaction_utilities is a df with one utility column and one row per row in interaction_df\n", - " trace_rows = trace_ids = None\n", - "\n", - " # interaction_utilities is a df with one utility column and one row per interaction_df row\n", - " interaction_utilities, trace_eval_results = interaction_simulate.eval_interaction_utilities(\n", - " spec, interaction_df, locals_d, trace_label, trace_rows, estimator=None,\n", - " log_alt_losers=log_alt_losers\n", - " )\n", - " # ########### HWM - high water mark (point of max observed memory usage)\n", - " del interaction_df\n", - "\n", - " # reshape utilities (one utility column and one row per row in interaction_utilities)\n", - " # to a dataframe with one row per chooser and one column per alternative\n", - " utilities = pd.DataFrame(\n", - " interaction_utilities.values.reshape(len(choosers), alternative_count),\n", - " index=choosers.index)\n", - " del interaction_utilities\n", - "\n", - " # convert to probabilities (utilities exponentiated and normalized to probs)\n", - " # probs is same shape as utilities, one row per chooser and one column for alternative\n", - " probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", - " trace_label=trace_label, trace_choosers=choosers)\n", - " del utilities\n", - "\n", - " if sample_size == 0:\n", - " # FIXME return full alternative set rather than sample\n", - " print(\"Estimation mode for %s using unsampled alternatives\" % (trace_label, ))\n", - " index_name = probs.index.name\n", - " choices_df = \\\n", - " pd.melt(probs.reset_index(), id_vars=[index_name])\\\n", - " .sort_values(by=index_name, kind='mergesort')\\\n", - " .set_index(index_name)\\\n", - " .rename(columns={'value': 'prob'})\\\n", - " .drop(columns='variable')\n", - " choices_df['pick_count'] = 1\n", - " choices_df.insert(0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index)))\n", - " return choices_df\n", - " else:\n", - " choices_df = hack_make_sample_choices(\n", - " choosers, probs, alternatives,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs=allow_zero_probs,\n", - " trace_label=trace_label)\n", - "\n", - " del probs\n", - "\n", - " # pick_count and pick_dup\n", - " # pick_count is number of duplicate picks\n", - " # pick_dup flag is True for all but first of duplicates\n", - " pick_group = choices_df.groupby([choosers.index.name, alt_col_name])\n", - " # number each item in each group from 0 to the length of that group - 1.\n", - " choices_df['pick_count'] = pick_group.cumcount(ascending=True)\n", - " # flag duplicate rows after first\n", - " choices_df['pick_dup'] = choices_df['pick_count'] > 0\n", - " # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", - " choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", - " # drop the duplicates\n", - " choices_df = choices_df[~choices_df['pick_dup']]\n", - " del choices_df['pick_dup']\n", - " # set index after groupby so we can trace on it\n", - " choices_df.set_index(choosers.index.name, inplace=True)\n", - "\n", - " # don't need this after tracing\n", - " del choices_df['rand']\n", - "\n", - " # - NARROW\n", - " choices_df['prob'] = choices_df['prob'].astype(np.float32)\n", - " assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty)\n", - " choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32)\n", - "\n", - " return choices_df" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 86, + "execution_count": 92, "outputs": [ { "name": "stderr", @@ -556,8 +434,8 @@ "running gradeschool, 1\n", "dropping 0 of 25 rows where size_term is zero\n", "Running school_location with 17 persons\n", - "CPU times: user 219 ms, sys: 0 ns, total: 219 ms\n", - "Wall time: 264 ms\n" + "CPU times: user 188 ms, sys: 31.2 ms, total: 219 ms\n", + "Wall time: 216 ms\n" ] } ], @@ -738,12 +616,14 @@ " trace_label=chunk_trace_label, trace_choosers=chooser_chunk)\n", " #del utilities\n", "\n", - " choices_df = hack_make_sample_choices(\n", + " temp_choices = hack_make_sample_choices(\n", " chooser_chunk, probs, dest_size_terms,\n", " sample_size, alternative_count, alt_col_name,\n", " allow_zero_probs=allow_zero_probs,\n", " trace_label=chunk_trace_label)\n", "\n", + " choices_df = temp_choices.copy()\n", + "\n", " # pick_count and pick_dup\n", " # pick_count is number of duplicate picks\n", " # pick_dup flag is True for all but first of duplicates\n", @@ -767,8 +647,6 @@ " choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32)\n", "\n", "\n", - "\n", - "\n", " if choices_df.shape[0] > 0:\n", " result_list.append(choices_df)\n", "\n", @@ -799,6 +677,78 @@ } } }, + { + "cell_type": "code", + "execution_count": 98, + "outputs": [ + { + "data": { + "text/plain": " alt_dest rand prob person_id\n0 8 0.222922 0.157982 386008\n1 9 0.404176 0.336708 386008\n2 8 0.223154 0.157982 386008\n3 9 0.457402 0.336708 386008\n4 11 0.812506 0.069858 386008\n.. ... ... ... ...\n165 7 0.107363 0.062739 4171620\n166 11 0.517573 0.070848 4171620\n167 10 0.375828 0.097048 4171620\n168 9 0.279810 0.111409 4171620\n169 20 0.848968 0.026642 4171620\n\n[170 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
080.2229220.157982386008
190.4041760.336708386008
280.2231540.157982386008
390.4574020.336708386008
4110.8125060.069858386008
...............
16570.1073630.0627394171620
166110.5175730.0708484171620
167100.3758280.0970484171620
16890.2798100.1114094171620
169200.8489680.0266424171620
\n

170 rows × 4 columns

\n
" + }, + "execution_count": 98, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "temp_choices" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 97, + "outputs": [ + { + "data": { + "text/plain": " alt_dest prob pick_count\nperson_id \n386008 8 0.157982 2\n386008 9 0.336708 5\n386008 10 0.205884 1\n386008 11 0.069858 1\n386008 17 0.009346 1\n... ... ... ...\n4171620 9 0.111409 1\n4171620 10 0.097048 1\n4171620 11 0.070848 1\n4171620 20 0.026642 1\n4171620 25 0.062053 1\n\n[105 rows x 3 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
38600880.1579822
38600890.3367085
386008100.2058841
386008110.0698581
386008170.0093461
............
417162090.1114091
4171620100.0970481
4171620110.0708481
4171620200.0266421
4171620250.0620531
\n

105 rows × 3 columns

\n
" + }, + "execution_count": 97, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_list[-1]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": 91, From 36789a3a2c32f790f3cfafd7a22c2ce155138e56 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 16 Jun 2022 18:54:04 +1000 Subject: [PATCH 043/135] interaction_sample dev work --- .../frozen_impl_interaction_sample_dev.ipynb | 523 ++++++++++++------ 1 file changed, 350 insertions(+), 173 deletions(-) diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index e82c93c057..ad34deebeb 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -98,19 +98,32 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 112, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(3, 3, 2)\n", + "[[2 2]\n", + " [2 1]\n", + " [1 0]]\n" + ] + }, { "data": { - "text/plain": "array([2, 1, 2])" + "text/plain": "array([2, 2, 1, 2, 1, 0])" }, - "execution_count": 35, + "execution_count": 112, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "np.argmax(np.array([[1,2,3],[4,6,5],[7,8,9]]), axis=1)" + "utils = np.array([[1,2,3,4,5,6],[4,6,5,9,9,6],[7,8,9,1,2,3]]).reshape((3,3,2))\n", + "print(utils.shape)\n", + "print(np.argmax(utils, axis=1))\n", + "np.argmax(utils, axis=1).flatten(order=\"F\")" ], "metadata": { "collapsed": false, @@ -121,29 +134,52 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 124, "outputs": [ { "data": { - "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" + "text/plain": "array([[2, 2],\n [2, 1],\n [1, 0]])" }, + "execution_count": 124, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" + } + ], + "source": [ + "ch_array = np.argmax(utils, axis=1)\n", + "ch_array" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 191, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(3, 4)\n" + ] }, { "data": { - "text/plain": "0 c\n1 b\n2 c\ndtype: object" + "text/plain": "array([[0.1 , 0.7 , 0.1 , 0.1 ],\n [0. , 0.5 , 0.25, 0.25],\n [0.3 , 0.3 , 0.2 , 0.2 ]])" }, + "execution_count": 191, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", - "display(df_)\n", - "ch_ = df_.idxmax(1)\n", - "display(ch_)" + "probs = np.array([[0.1,0.7,0.1,0.1], [0.0,0.5,0.25,0.25], [0.3,0.3,0.2,0.2]])\n", + "print(probs.shape)\n", + "probs" ], "metadata": { "collapsed": false, @@ -154,20 +190,19 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 174, "outputs": [ { "data": { - "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" + "text/plain": "(3, 4, 3)" }, + "execution_count": 174, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", - "display(df_)" + "np.repeat(probs[:,:,None], 3, axis=2).shape" ], "metadata": { "collapsed": false, @@ -178,19 +213,19 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 198, "outputs": [ { "data": { - "text/plain": "{'a': 0, 'b': 1, 'c': 2}" + "text/plain": "array([0, 0, 1, 1, 2, 2])" }, - "execution_count": 64, + "execution_count": 198, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "{v: k for k,v in enumerate(df_.columns)}" + "np.repeat(np.arange(0,probs.shape[0]), 2)" ], "metadata": { "collapsed": false, @@ -201,9 +236,20 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 202, + "outputs": [ + { + "data": { + "text/plain": "array([0, 1, 2, 0, 1, 2, 0, 1, 2])" + }, + "execution_count": 202, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.tile(np.arange(0,probs.shape[0]), 3)" + ], "metadata": { "collapsed": false, "pycharm": { @@ -312,18 +358,22 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 218, "outputs": [], "source": [ + "from activitysim.core.logit import inverse_ev1_cdf\n", + "\n", "def hack_make_sample_choices(\n", - " choosers, probs,\n", - " alternatives,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs,\n", - " trace_label):\n", + " choosers, probs,\n", + " alternatives,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs,\n", + " trace_label,\n", + " utilities=None,\n", + " choose_individual_max_utility=False\n", + "):\n", " assert isinstance(probs, pd.DataFrame)\n", " assert probs.shape == (len(choosers), alternative_count)\n", - "\n", " assert isinstance(alternatives, pd.DataFrame)\n", " assert len(alternatives) == alternative_count\n", "\n", @@ -335,72 +385,112 @@ " # remove from sample\n", " probs = probs[~zero_probs]\n", " choosers = choosers[~zero_probs]\n", + " # TODO [janzill Jun2022]: probably need to remove these from the utilities as well for consistency?\n", + " # might need this in other places too?\n", + "\n", + " if choose_individual_max_utility:\n", + " assert isinstance(utilities, pd.DataFrame)\n", + " #print(utilities.head(3))\n", + " assert utilities.shape == (len(choosers), alternative_count)\n", + " #print(utilities.shape)\n", + "\n", + " choice_dimension = (len(choosers), alternative_count, sample_size)\n", + " rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count*sample_size)\n", + " #print(f\"after generation rands shape = {rands.shape}\", flush=True)\n", + " rands = rands.reshape(choice_dimension)\n", + " #print(f\"before inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", + " rands = inverse_ev1_cdf(rands)\n", + " print(f\"after inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", + " utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented\n", + " utilities = np.repeat(utilities[:,:,None], sample_size, axis=2)\n", + " print(f\"after utils reshape: {utilities.shape}\", flush=True)\n", + " utilities += rands\n", + " # this gives us len(choosers), sample_size dimensions, with values the chosen alternative\n", + " choices_array = np.argmax(utilities, axis=1)\n", + " print(choices_array.shape)\n", + "\n", + " choosers_index_rep = np.tile(np.arange(0,choices_array.shape[0]), sample_size)\n", + " #np.repeat(np.arange(0,choices_array.shape[0]), sample_size)\n", + " choices_flattened = choices_array.flatten(order='F')\n", + " print(f\"choices flattened shape = {choices_flattened.shape}\")\n", + "\n", + " print(choosers_index_rep.shape, flush=True)\n", + " print(probs.shape, flush=True)\n", + " print(probs.head(3), flush=True)\n", + "\n", + " print(f\"fake rand shape = {np.zeros_like(choosers_index_rep).shape}\", flush=True)\n", + " probs_look_up = probs.to_numpy()[choosers_index_rep, choices_flattened]\n", + "\n", + " print(f\"probs_look_up shape = {probs_look_up.shape}\", flush=True)\n", + "\n", + " # explode to one row per chooser.index, alt_zone_id\n", + " choices_df = pd.DataFrame({\n", + " alt_col_name: choices_flattened,\n", + " #'rand': rands.flatten(order='F'),\n", + " 'rand': np.zeros_like(choosers_index_rep), # TODO: zero out for now\n", + " 'prob': probs_look_up.flatten(order='F'),\n", + " # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction\n", + " #choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", + " choosers.index.name: np.tile(choosers.index.values, sample_size)\n", + " })\n", + "\n", + " else:\n", + " cum_probs_array = probs.values.cumsum(axis=1)\n", + " # alt probs in convenient layout to return prob of chose alternative\n", + " # (same layout as cum_probs_arr)\n", + " alt_probs_array = probs.values.flatten()\n", + " # get sample_size rands for each chooser\n", + " rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size)\n", + " # transform as we iterate over alternatives\n", + " # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr\n", + " # i.e rands[i] is a 2-D array of one alt choice rand for each chooser\n", + " rands = rands.T.reshape(sample_size, -1, 1)\n", + " # the alternative value chosen\n", + " choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype)\n", + " # chunk log these later after we populate them...\n", + " # the probability of the chosen alternative\n", + " choice_probs_array = np.empty([sample_size, len(choosers)])\n", + " # chunk log these later after we populate them...\n", + " alts = np.tile(alternatives.index.values, len(choosers))\n", + " # FIXME - do this all at once rather than iterate?\n", + " for i in range(sample_size):\n", + " # FIXME - do this in numpy, not pandas?\n", + " # rands for this alt in broadcastable shape\n", + " r = rands[i]\n", + "\n", + " # position of first occurrence of positive value\n", + " positions = np.argmax(cum_probs_array > r, axis=1)\n", + "\n", + " # FIXME - leave positions as numpy array, not pandas series?\n", + " # positions is series with the chosen alternative represented as a column index in probs\n", + " # which is an integer between zero and num alternatives in the alternative sample\n", + " positions = pd.Series(positions, index=probs.index)\n", + "\n", + " # need to get from an integer offset into the alternative sample to the alternative index\n", + " # that is, we want the index value of the row that is offset by rows into the\n", + " # tranche of this choosers alternatives created by cross join of alternatives and choosers\n", + "\n", + " # offsets is the offset into model_design df of first row of chooser alternatives\n", + " offsets = np.arange(len(positions)) * alternative_count\n", + "\n", + " # choices and choice_probs have one element per chooser and is in same order as choosers\n", + " choices_array[i] = np.take(alts, positions + offsets)\n", + " choice_probs_array[i] = np.take(alt_probs_array, positions + offsets)\n", + " del positions\n", + " del offsets\n", + "\n", + " del alts\n", + " del cum_probs_array\n", + " del alt_probs_array\n", + "\n", + " # explode to one row per chooser.index, alt_zone_id\n", + " choices_df = pd.DataFrame(\n", + " {alt_col_name: choices_array.flatten(order='F'),\n", + " 'rand': rands.flatten(order='F'),\n", + " 'prob': choice_probs_array.flatten(order='F'),\n", + " choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", + " })\n", "\n", - " cum_probs_array = probs.values.cumsum(axis=1)\n", - "\n", - " # alt probs in convenient layout to return prob of chose alternative\n", - " # (same layout as cum_probs_arr)\n", - " alt_probs_array = probs.values.flatten()\n", - "\n", - " # get sample_size rands for each chooser\n", - " rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size)\n", - "\n", - " # transform as we iterate over alternatives\n", - " # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr\n", - " # i.e rands[i] is a 2-D array of one alt choice rand for each chooser\n", - " rands = rands.T.reshape(sample_size, -1, 1)\n", - "\n", - " # the alternative value chosen\n", - " choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype)\n", - " # chunk log these later after we populate them...\n", - " # the probability of the chosen alternative\n", - " choice_probs_array = np.empty([sample_size, len(choosers)])\n", - " # chunk log these later after we populate them...\n", - " alts = np.tile(alternatives.index.values, len(choosers))\n", - "\n", - " # FIXME - do this all at once rather than iterate?\n", - " for i in range(sample_size):\n", - " # FIXME - do this in numpy, not pandas?\n", - " # rands for this alt in broadcastable shape\n", - " r = rands[i]\n", - "\n", - " # position of first occurrence of positive value\n", - " positions = np.argmax(cum_probs_array > r, axis=1)\n", - "\n", - " # FIXME - leave positions as numpy array, not pandas series?\n", - " # positions is series with the chosen alternative represented as a column index in probs\n", - " # which is an integer between zero and num alternatives in the alternative sample\n", - " positions = pd.Series(positions, index=probs.index)\n", - "\n", - " # need to get from an integer offset into the alternative sample to the alternative index\n", - " # that is, we want the index value of the row that is offset by rows into the\n", - " # tranche of this choosers alternatives created by cross join of alternatives and choosers\n", - "\n", - " # offsets is the offset into model_design df of first row of chooser alternatives\n", - " offsets = np.arange(len(positions)) * alternative_count\n", - "\n", - " # choices and choice_probs have one element per chooser and is in same order as choosers\n", - " choices_array[i] = np.take(alts, positions + offsets)\n", - " choice_probs_array[i] = np.take(alt_probs_array, positions + offsets)\n", - "\n", - " del positions\n", - " del offsets\n", - "\n", - " del alts\n", - " del cum_probs_array\n", - " del alt_probs_array\n", - "\n", - " # explode to one row per chooser.index, alt_zone_id\n", - " choices_df = pd.DataFrame(\n", - " {alt_col_name: choices_array.flatten(order='F'),\n", - " 'rand': rands.flatten(order='F'),\n", - " 'prob': choice_probs_array.flatten(order='F'),\n", - " choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", - " })\n", - "\n", - " del choices_array\n", - " del rands\n", - " del choice_probs_array\n", " return choices_df" ], "metadata": { @@ -412,7 +502,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 219, "outputs": [ { "name": "stderr", @@ -428,19 +518,85 @@ "running university, 3\n", "dropping 19 of 25 rows where size_term is zero\n", "Running school_location with 17 persons\n", + "after inverse_ev1 rands shape = (17, 6, 6)\n", + "after utils reshape: (17, 6, 6)\n", + "(17, 6)\n", + "choices flattened shape = (102,)\n", + "(102,)\n", + "(17, 6)\n", + " 0 1 2 3 4 5\n", + "person_id \n", + "325623 0.001874 0.005184 0.002695 0.299427 0.508256 0.182563\n", + "386007 0.001584 0.724879 0.145533 0.080880 0.041571 0.005553\n", + "1774265 0.019349 0.146800 0.038209 0.486232 0.273819 0.035591\n", + "fake rand shape = (102,)\n", + "probs_look_up shape = (102,)\n", "running highschool, 2\n", "dropping 23 of 25 rows where size_term is zero\n", "Running school_location with 5 persons\n", + "after inverse_ev1 rands shape = (5, 2, 2)\n", + "after utils reshape: (5, 2, 2)\n", + "(5, 2)\n", + "choices flattened shape = (10,)\n", + "(10,)\n", + "(5, 2)\n", + " 0 1\n", + "person_id \n", + "386062 0.176063 0.823937\n", + "595685 0.057237 0.942763\n", + "2877285 0.126995 0.873005\n", + "fake rand shape = (10,)\n", + "probs_look_up shape = (10,)\n", "running gradeschool, 1\n", "dropping 0 of 25 rows where size_term is zero\n", "Running school_location with 17 persons\n", - "CPU times: user 188 ms, sys: 31.2 ms, total: 219 ms\n", - "Wall time: 216 ms\n" + "after inverse_ev1 rands shape = (17, 25, 10)\n", + "after utils reshape: (17, 25, 10)\n", + "(17, 10)\n", + "choices flattened shape = (170,)\n", + "(170,)\n", + "(17, 25)\n", + " 0 1 2 3 4 5 \\\n", + "person_id \n", + "386008 0.000504 0.001303 0.002645 0.001534 0.007081 0.023018 \n", + "418442 0.000554 0.001434 0.002911 0.001678 0.007704 0.020100 \n", + "595686 0.000877 0.002269 0.004605 0.003020 0.014986 0.025318 \n", + "\n", + " 6 7 8 9 10 11 \\\n", + "person_id \n", + "386008 0.056174 0.157982 0.336708 0.205884 0.069858 0.001223 \n", + "418442 0.042219 0.120700 0.229318 0.288773 0.101255 0.001526 \n", + "595686 0.058311 0.166706 0.116761 0.110817 0.133127 0.003827 \n", + "\n", + " 12 13 14 15 16 17 \\\n", + "person_id \n", + "386008 0.000206 0.000548 0.000380 0.015291 0.009346 0.008790 \n", + "418442 0.000257 0.000683 0.000474 0.019075 0.011592 0.010965 \n", + "595686 0.000525 0.001116 0.000750 0.053224 0.025697 0.033011 \n", + "\n", + " 18 19 20 21 22 23 \\\n", + "person_id \n", + "386008 0.008610 0.032788 0.028567 0.002079 0.000817 0.002728 \n", + "418442 0.012703 0.052444 0.039245 0.002288 0.000899 0.002985 \n", + "595686 0.014173 0.033481 0.147571 0.003620 0.001422 0.004723 \n", + "\n", + " 24 \n", + "person_id \n", + "386008 0.025936 \n", + "418442 0.028217 \n", + "595686 0.040062 \n", + "fake rand shape = (170,)\n", + "probs_look_up shape = (170,)\n", + "CPU times: user 312 ms, sys: 46.9 ms, total: 359 ms\n", + "Wall time: 358 ms\n" ] } ], "source": [ "%%time\n", + "\n", + "choose_individual_max_utility = True\n", + "\n", "resume_after = \"compute_accessibility\"\n", "model_name = \"school_location\"\n", "chunk_size = 0 # test_mtc means no chunking\n", @@ -464,72 +620,48 @@ "trace_label = model_name #'school_location'\n", "model_settings_file_name = f\"{model_name}.yaml\" #'school_location.yaml'\n", "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", "estimator = estimation.manager.begin_estimation(model_name)\n", - "\n", - "# iterate_location_choice(\n", - "# model_settings,\n", - "# persons_merged, persons, households,\n", - "# network_los,\n", - "# estimator,\n", - "# chunk_size, trace_hh_id, locutor, trace_label\n", - "# )\n", - "\n", + "# iterate_location_choice()\n", "chunk_tag = trace_label\n", "\n", "# boolean to filter out persons not needing location modeling (e.g. is_worker, is_student)\n", "chooser_filter_column = model_settings['CHOOSER_FILTER_COLUMN_NAME']\n", - "\n", "dest_choice_column_name = model_settings['DEST_CHOICE_COLUMN_NAME']\n", "logsum_column_name = model_settings.get('DEST_CHOICE_LOGSUM_COLUMN_NAME')\n", - "\n", "sample_table_name = model_settings.get('DEST_CHOICE_SAMPLE_TABLE_NAME')\n", "want_sample_table = config.setting('want_dest_choice_sample_tables') and sample_table_name is not None\n", - "\n", "persons_merged_df = persons_merged.to_frame()\n", - "\n", "persons_merged_df = persons_merged_df[persons_merged_df[chooser_filter_column]]\n", - "\n", "persons_merged_df.sort_index(inplace=True) # interaction_sample expects chooser index to be monotonic increasing\n", "\n", "# chooser segmentation allows different sets coefficients for e.g. different income_segments or tour_types\n", "chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']\n", - "\n", "assert chooser_segment_column in persons_merged_df, f\"CHOOSER_SEGMENT_COLUMN '{chooser_segment_column}' not in \" \\\n", " f\"persons_merged table.\"\n", - "\n", "shadow_price_calculator = shadow_pricing.load_shadow_price_calculator(model_settings)\n", - "\n", "chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']\n", - "\n", "# maps segment names to compact (integer) ids\n", "segment_ids = model_settings['SEGMENT_IDS']\n", "\n", - "#choices_list = []\n", "sample_list = []\n", "for segment_name, segment_id in segment_ids.items():\n", " print(f\"running {segment_name}, {segment_id}\")\n", " choosers = persons_merged_df[persons_merged_df[chooser_segment_column] == segment_id]\n", - "\n", " # size_term and shadow price adjustment - one row per zone\n", " dest_size_terms = shadow_price_calculator.dest_size_terms(segment_name)\n", - "\n", " assert dest_size_terms.index.is_monotonic_increasing, f\"shadow_price_calculator.dest_size_terms({segment_name}) \" \\\n", " f\"not monotonic_increasing\"\n", " if choosers.shape[0] == 0:\n", " print(f\"{trace_label} skipping segment {segment_name}: no choosers\")\n", " continue\n", - "\n", " print(f\"dropping {(~(dest_size_terms.size_term > 0)).sum()} \"\n", " f\"of {len(dest_size_terms)} rows where size_term is zero\")\n", " dest_size_terms = dest_size_terms[dest_size_terms.size_term > 0]\n", - "\n", " chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS']\n", " choosers_location_sample = choosers[chooser_columns]\n", " skim_dict = network_los.get_default_skim_dict()\n", " skims = skim_dict.wrap('home_zone_id', 'zone_id')\n", " alt_dest_col_name = model_settings['ALT_DEST_COL_NAME']\n", - "\n", " assert not choosers_location_sample.empty\n", " print(\"Running %s with %d persons\" % (trace_label, len(choosers_location_sample.index)))\n", " sample_size = model_settings[\"SAMPLE_SIZE\"]\n", @@ -541,11 +673,10 @@ " locals_d.update(constants)\n", " spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC',\n", " segment_name=segment_name, estimator=estimator)\n", - " ### choices = interaction_sample(\n", + " ### choices = interaction_sample()\n", " alt_col_name=alt_dest_col_name\n", " allow_zero_probs=False\n", " log_alt_losers=False\n", - "\n", " # we return alternatives ordered in (index, alt_col_name)\n", " # if choosers index is not ordered, it is probably a mistake, since the alts wont line up\n", " assert alt_col_name is not None\n", @@ -620,7 +751,10 @@ " chooser_chunk, probs, dest_size_terms,\n", " sample_size, alternative_count, alt_col_name,\n", " allow_zero_probs=allow_zero_probs,\n", - " trace_label=chunk_trace_label)\n", + " trace_label=chunk_trace_label,\n", + " utilities=utilities,\n", + " choose_individual_max_utility=choose_individual_max_utility\n", + " )\n", "\n", " choices_df = temp_choices.copy()\n", "\n", @@ -679,20 +813,58 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 221, + "outputs": [ + { + "data": { + "text/plain": " alt_dest rand prob person_id\n0 8 0 0.336708 386008\n51 6 0 0.056174 386008\n85 8 0 0.336708 386008\n102 8 0 0.336708 386008\n34 7 0 0.157982 386008\n.. ... ... ... ...\n50 15 0 0.182639 4171620\n33 5 0 0.032124 4171620\n16 8 0 0.111409 4171620\n152 9 0 0.097048 4171620\n169 6 0 0.062739 4171620\n\n[170 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
0800.336708386008
51600.056174386008
85800.336708386008
102800.336708386008
34700.157982386008
...............
501500.1826394171620
33500.0321244171620
16800.1114094171620
152900.0970484171620
169600.0627394171620
\n

170 rows × 4 columns

\n
" + }, + "execution_count": 221, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "temp_choices.sort_values(by=[\"person_id\"])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 216, "outputs": [ { "data": { "text/plain": " alt_dest rand prob person_id\n0 8 0.222922 0.157982 386008\n1 9 0.404176 0.336708 386008\n2 8 0.223154 0.157982 386008\n3 9 0.457402 0.336708 386008\n4 11 0.812506 0.069858 386008\n.. ... ... ... ...\n165 7 0.107363 0.062739 4171620\n166 11 0.517573 0.070848 4171620\n167 10 0.375828 0.097048 4171620\n168 9 0.279810 0.111409 4171620\n169 20 0.848968 0.026642 4171620\n\n[170 rows x 4 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
080.2229220.157982386008
190.4041760.336708386008
280.2231540.157982386008
390.4574020.336708386008
4110.8125060.069858386008
...............
16570.1073630.0627394171620
166110.5175730.0708484171620
167100.3758280.0970484171620
16890.2798100.1114094171620
169200.8489680.0266424171620
\n

170 rows × 4 columns

\n
" }, - "execution_count": 98, + "execution_count": 216, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "temp_choices" + "#temp_choices" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "choices_df" ], "metadata": { "collapsed": false, @@ -703,20 +875,20 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 217, "outputs": [ { "data": { "text/plain": " alt_dest prob pick_count\nperson_id \n386008 8 0.157982 2\n386008 9 0.336708 5\n386008 10 0.205884 1\n386008 11 0.069858 1\n386008 17 0.009346 1\n... ... ... ...\n4171620 9 0.111409 1\n4171620 10 0.097048 1\n4171620 11 0.070848 1\n4171620 20 0.026642 1\n4171620 25 0.062053 1\n\n[105 rows x 3 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
38600880.1579822
38600890.3367085
386008100.2058841
386008110.0698581
386008170.0093461
............
417162090.1114091
4171620100.0970481
4171620110.0708481
4171620200.0266421
4171620250.0620531
\n

105 rows × 3 columns

\n
" }, - "execution_count": 97, + "execution_count": 217, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "sample_list[-1]" + "#choices_df" ], "metadata": { "collapsed": false, @@ -751,28 +923,21 @@ }, { "cell_type": "code", - "execution_count": 91, - "outputs": [ - { - "data": { - "text/plain": "10" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": "person_id\n386008 10\n418442 10\n595686 10\n644292 10\n644478 10\n1958678 10\n2159059 10\n2219998 10\n2458502 10\n2458503 10\n2566700 10\n2566701 10\n2566702 10\n2877287 10\n3596365 10\n3891104 10\n4171620 10\nName: pick_count, dtype: uint32" - }, - "execution_count": 91, - "metadata": {}, - "output_type": "execute_result" + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" } - ], - "source": [ - "display(sample_size)\n", - "sample_list[2].groupby('person_id').pick_count.sum()" - ], + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], "metadata": { "collapsed": false, "pycharm": { @@ -782,23 +947,20 @@ }, { "cell_type": "code", - "execution_count": 71, - "outputs": [ - { - "ename": "AssertionError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mAssertionError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/tmp/ipykernel_10980/63825533.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 43\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 44\u001B[0m \u001B[0;31m# interaction_utilities is a df with one utility column and one row per interaction_df row\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 45\u001B[0;31m interaction_utilities, trace_eval_results = interaction_simulate.eval_interaction_utilities(\n\u001B[0m\u001B[1;32m 46\u001B[0m \u001B[0mspec\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0minteraction_df\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlocals_d\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrace_label\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrace_rows\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mestimator\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mNone\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 47\u001B[0m \u001B[0mlog_alt_losers\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mlog_alt_losers\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/interaction_simulate.py\u001B[0m in \u001B[0;36meval_interaction_utilities\u001B[0;34m(spec, df, locals_d, trace_label, trace_rows, estimator, log_alt_losers)\u001B[0m\n\u001B[1;32m 66\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"Running eval_interaction_utilities on %s rows\"\u001B[0m \u001B[0;34m%\u001B[0m \u001B[0mdf\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mshape\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 67\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 68\u001B[0;31m \u001B[0;32mwith\u001B[0m \u001B[0mchunk\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchunk_log\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrace_label\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 69\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 70\u001B[0m \u001B[0;32massert\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mspec\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcolumns\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m~/.pyenv/versions/3.8.2/lib/python3.8/contextlib.py\u001B[0m in \u001B[0;36m__enter__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 111\u001B[0m \u001B[0;32mdel\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkwds\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfunc\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 112\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 113\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mnext\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgen\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 114\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mStopIteration\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 115\u001B[0m \u001B[0;32mraise\u001B[0m \u001B[0mRuntimeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"generator didn't yield\"\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/chunk.py\u001B[0m in \u001B[0;36mchunk_log\u001B[0;34m(trace_label, chunk_tag, base)\u001B[0m\n\u001B[1;32m 927\u001B[0m \u001B[0;31m# avoids breaking the assertion below.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 928\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 929\u001B[0;31m \u001B[0;32massert\u001B[0m \u001B[0mbase\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mCHUNK_SIZERS\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 930\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 931\u001B[0m \u001B[0mtrace_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34mf\"{trace_label}.chunk_log\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mAssertionError\u001B[0m: " - ] + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" } - ], + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], "source": [], "metadata": { "collapsed": false, @@ -807,6 +969,21 @@ } } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "display(sample_size)\n", + "sample_list[2].groupby('person_id').pick_count.sum()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 64c4e6e2c31a3731734487a6b79eba5e114be6e8 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 17 Jun 2022 08:57:18 +1000 Subject: [PATCH 044/135] fixes alternative index as chosen value --- .../frozen_impl_interaction_sample_dev.ipynb | 91 +++++++------------ 1 file changed, 32 insertions(+), 59 deletions(-) diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index ad34deebeb..17c559e364 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -358,7 +358,7 @@ }, { "cell_type": "code", - "execution_count": 218, + "execution_count": 234, "outputs": [], "source": [ "from activitysim.core.logit import inverse_ev1_cdf\n", @@ -385,8 +385,10 @@ " # remove from sample\n", " probs = probs[~zero_probs]\n", " choosers = choosers[~zero_probs]\n", - " # TODO [janzill Jun2022]: probably need to remove these from the utilities as well for consistency?\n", - " # might need this in other places too?\n", + " # TODO [janzill Jun2022]: do we want this for consistency?\n", + " # might need this in other places too?\n", + " if utilities is not None:\n", + " utilities = utilities[~zero_probs]\n", "\n", " if choose_individual_max_utility:\n", " assert isinstance(utilities, pd.DataFrame)\n", @@ -400,10 +402,10 @@ " rands = rands.reshape(choice_dimension)\n", " #print(f\"before inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", " rands = inverse_ev1_cdf(rands)\n", - " print(f\"after inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", + " #print(f\"after inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", " utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented\n", " utilities = np.repeat(utilities[:,:,None], sample_size, axis=2)\n", - " print(f\"after utils reshape: {utilities.shape}\", flush=True)\n", + " #print(f\"after utils reshape: {utilities.shape}\", flush=True)\n", " utilities += rands\n", " # this gives us len(choosers), sample_size dimensions, with values the chosen alternative\n", " choices_array = np.argmax(utilities, axis=1)\n", @@ -412,20 +414,19 @@ " choosers_index_rep = np.tile(np.arange(0,choices_array.shape[0]), sample_size)\n", " #np.repeat(np.arange(0,choices_array.shape[0]), sample_size)\n", " choices_flattened = choices_array.flatten(order='F')\n", - " print(f\"choices flattened shape = {choices_flattened.shape}\")\n", + " #print(f\"choices flattened shape = {choices_flattened.shape}\")\n", "\n", " print(choosers_index_rep.shape, flush=True)\n", " print(probs.shape, flush=True)\n", " print(probs.head(3), flush=True)\n", - "\n", - " print(f\"fake rand shape = {np.zeros_like(choosers_index_rep).shape}\", flush=True)\n", " probs_look_up = probs.to_numpy()[choosers_index_rep, choices_flattened]\n", + " #print(f\"probs_look_up shape = {probs_look_up.shape}\", flush=True)\n", "\n", - " print(f\"probs_look_up shape = {probs_look_up.shape}\", flush=True)\n", - "\n", + " # choices_flattened are 0-based index into alternatives, need to map to alternative values given by\n", + " # alternatives.index.values (they are in this order by construction)\n", " # explode to one row per chooser.index, alt_zone_id\n", " choices_df = pd.DataFrame({\n", - " alt_col_name: choices_flattened,\n", + " alt_col_name: alternatives.index.values[choices_flattened],\n", " #'rand': rands.flatten(order='F'),\n", " 'rand': np.zeros_like(choosers_index_rep), # TODO: zero out for now\n", " 'prob': probs_look_up.flatten(order='F'),\n", @@ -502,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 219, + "execution_count": 237, "outputs": [ { "name": "stderr", @@ -518,10 +519,7 @@ "running university, 3\n", "dropping 19 of 25 rows where size_term is zero\n", "Running school_location with 17 persons\n", - "after inverse_ev1 rands shape = (17, 6, 6)\n", - "after utils reshape: (17, 6, 6)\n", "(17, 6)\n", - "choices flattened shape = (102,)\n", "(102,)\n", "(17, 6)\n", " 0 1 2 3 4 5\n", @@ -529,15 +527,10 @@ "325623 0.001874 0.005184 0.002695 0.299427 0.508256 0.182563\n", "386007 0.001584 0.724879 0.145533 0.080880 0.041571 0.005553\n", "1774265 0.019349 0.146800 0.038209 0.486232 0.273819 0.035591\n", - "fake rand shape = (102,)\n", - "probs_look_up shape = (102,)\n", "running highschool, 2\n", "dropping 23 of 25 rows where size_term is zero\n", "Running school_location with 5 persons\n", - "after inverse_ev1 rands shape = (5, 2, 2)\n", - "after utils reshape: (5, 2, 2)\n", "(5, 2)\n", - "choices flattened shape = (10,)\n", "(10,)\n", "(5, 2)\n", " 0 1\n", @@ -545,15 +538,10 @@ "386062 0.176063 0.823937\n", "595685 0.057237 0.942763\n", "2877285 0.126995 0.873005\n", - "fake rand shape = (10,)\n", - "probs_look_up shape = (10,)\n", "running gradeschool, 1\n", "dropping 0 of 25 rows where size_term is zero\n", "Running school_location with 17 persons\n", - "after inverse_ev1 rands shape = (17, 25, 10)\n", - "after utils reshape: (17, 25, 10)\n", "(17, 10)\n", - "choices flattened shape = (170,)\n", "(170,)\n", "(17, 25)\n", " 0 1 2 3 4 5 \\\n", @@ -585,10 +573,8 @@ "386008 0.025936 \n", "418442 0.028217 \n", "595686 0.040062 \n", - "fake rand shape = (170,)\n", - "probs_look_up shape = (170,)\n", - "CPU times: user 312 ms, sys: 46.9 ms, total: 359 ms\n", - "Wall time: 358 ms\n" + "CPU times: user 281 ms, sys: 31.2 ms, total: 312 ms\n", + "Wall time: 334 ms\n" ] } ], @@ -813,14 +799,14 @@ }, { "cell_type": "code", - "execution_count": 221, + "execution_count": 238, "outputs": [ { "data": { - "text/plain": " alt_dest rand prob person_id\n0 8 0 0.336708 386008\n51 6 0 0.056174 386008\n85 8 0 0.336708 386008\n102 8 0 0.336708 386008\n34 7 0 0.157982 386008\n.. ... ... ... ...\n50 15 0 0.182639 4171620\n33 5 0 0.032124 4171620\n16 8 0 0.111409 4171620\n152 9 0 0.097048 4171620\n169 6 0 0.062739 4171620\n\n[170 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
0800.336708386008
51600.056174386008
85800.336708386008
102800.336708386008
34700.157982386008
...............
501500.1826394171620
33500.0321244171620
16800.1114094171620
152900.0970484171620
169600.0627394171620
\n

170 rows × 4 columns

\n
" + "text/plain": " alt_dest rand prob person_id\n0 9 0 0.336708 386008\n51 7 0 0.056174 386008\n85 9 0 0.336708 386008\n102 9 0 0.336708 386008\n34 8 0 0.157982 386008\n.. ... ... ... ...\n50 16 0 0.182639 4171620\n33 6 0 0.032124 4171620\n16 9 0 0.111409 4171620\n152 10 0 0.097048 4171620\n169 7 0 0.062739 4171620\n\n[170 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
0900.336708386008
51700.056174386008
85900.336708386008
102900.336708386008
34800.157982386008
...............
501600.1826394171620
33600.0321244171620
16900.1114094171620
1521000.0970484171620
169700.0627394171620
\n

170 rows × 4 columns

\n
" }, - "execution_count": 221, + "execution_count": 238, "metadata": {}, "output_type": "execute_result" } @@ -899,33 +885,20 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" + "execution_count": 231, + "outputs": [ + { + "data": { + "text/plain": "array([1, 2, 3, 4, 5])" + }, + "execution_count": 231, + "metadata": {}, + "output_type": "execute_result" } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + ], + "source": [ + "dest_size_terms.index.values[[0,1,2,3,4]]" + ], "metadata": { "collapsed": false, "pycharm": { From e3e4326bdc8c86e426bb8e67a01a3b656e375f68 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 17 Jun 2022 09:14:45 +1000 Subject: [PATCH 045/135] interaction sample frozen rand indiv util --- activitysim/core/interaction_sample.py | 200 +++++++++++++++---------- 1 file changed, 124 insertions(+), 76 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index d792401dc2..4278b962c7 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -12,6 +12,7 @@ from . import tracing from . import chunk from .simulate import set_skim_wrapper_targets +from .logit import inverse_ev1_cdf from . import interaction_simulate @@ -27,7 +28,9 @@ def make_sample_choices( alternatives, sample_size, alternative_count, alt_col_name, allow_zero_probs, - trace_label): + trace_label, + utilities=None, + choose_individual_max_utility=False): """ Parameters @@ -62,96 +65,134 @@ def make_sample_choices( # remove from sample probs = probs[~zero_probs] choosers = choosers[~zero_probs] + # TODO [janzill Jun2022]: do we want this for consistency? + # might need this in other places too? + if utilities is not None: + utilities = utilities[~zero_probs] + + if choose_individual_max_utility: + assert isinstance(utilities, pd.DataFrame) + assert utilities.shape == (len(choosers), alternative_count) + choice_dimension = (len(choosers), alternative_count, sample_size) + + rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count*sample_size) + rands = rands.reshape(choice_dimension) + rands = inverse_ev1_cdf(rands) + chunk.log_df(trace_label, 'rands', rands) + + utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented + utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) + utilities += rands + + del rands + chunk.log_df(trace_label, 'rands', None) + + # this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative + choices_array = np.argmax(utilities, axis=1) + + choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) + choices_flattened = choices_array.flatten(order='F') + + # choices_flattened are 0-based index into alternatives, need to map to alternative values given by + # alternatives.index.values (they are in this order by construction) + # explode to one row per chooser.index, alt_zone_id + choices_df = pd.DataFrame({ + alt_col_name: alternatives.index.values[choices_flattened], + 'rand': np.zeros_like(choosers_index_rep), # TODO [janzill June2022]: zero out for now + 'prob': probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F'), + # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction + choosers.index.name: np.tile(choosers.index.values, sample_size) + }) + else: + cum_probs_array = probs.values.cumsum(axis=1) + chunk.log_df(trace_label, 'cum_probs_array', cum_probs_array) - cum_probs_array = probs.values.cumsum(axis=1) - chunk.log_df(trace_label, 'cum_probs_array', cum_probs_array) - - # alt probs in convenient layout to return prob of chose alternative - # (same layout as cum_probs_arr) - alt_probs_array = probs.values.flatten() - chunk.log_df(trace_label, 'alt_probs_array', alt_probs_array) + # alt probs in convenient layout to return prob of chose alternative + # (same layout as cum_probs_arr) + alt_probs_array = probs.values.flatten() + chunk.log_df(trace_label, 'alt_probs_array', alt_probs_array) - # get sample_size rands for each chooser - rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size) + # get sample_size rands for each chooser + rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size) - # transform as we iterate over alternatives - # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr - # i.e rands[i] is a 2-D array of one alt choice rand for each chooser - rands = rands.T.reshape(sample_size, -1, 1) - chunk.log_df(trace_label, 'rands', rands) + # transform as we iterate over alternatives + # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr + # i.e rands[i] is a 2-D array of one alt choice rand for each chooser + rands = rands.T.reshape(sample_size, -1, 1) + chunk.log_df(trace_label, 'rands', rands) - # the alternative value chosen - # WHY SHOULD CHOICES COL HAVE TO BE TYPE INT??? - # choices_array = np.empty([sample_size, len(choosers)]).astype(int) - choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype) - # chunk log these later after we populate them... + # the alternative value chosen + # WHY SHOULD CHOICES COL HAVE TO BE TYPE INT??? + # choices_array = np.empty([sample_size, len(choosers)]).astype(int) + choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype) + # chunk log these later after we populate them... - # the probability of the chosen alternative - choice_probs_array = np.empty([sample_size, len(choosers)]) - # chunk log these later after we populate them... + # the probability of the chosen alternative + choice_probs_array = np.empty([sample_size, len(choosers)]) + # chunk log these later after we populate them... - alts = np.tile(alternatives.index.values, len(choosers)) - chunk.log_df(trace_label, 'alts', alts) + alts = np.tile(alternatives.index.values, len(choosers)) + chunk.log_df(trace_label, 'alts', alts) - # FIXME - do this all at once rather than iterate? - for i in range(sample_size): + # FIXME - do this all at once rather than iterate? + for i in range(sample_size): - # FIXME - do this in numpy, not pandas? + # FIXME - do this in numpy, not pandas? - # rands for this alt in broadcastable shape - r = rands[i] + # rands for this alt in broadcastable shape + r = rands[i] - # position of first occurrence of positive value - positions = np.argmax(cum_probs_array > r, axis=1) + # position of first occurrence of positive value + positions = np.argmax(cum_probs_array > r, axis=1) - # FIXME - leave positions as numpy array, not pandas series? - # positions is series with the chosen alternative represented as a column index in probs - # which is an integer between zero and num alternatives in the alternative sample - positions = pd.Series(positions, index=probs.index) + # FIXME - leave positions as numpy array, not pandas series? + # positions is series with the chosen alternative represented as a column index in probs + # which is an integer between zero and num alternatives in the alternative sample + positions = pd.Series(positions, index=probs.index) - # need to get from an integer offset into the alternative sample to the alternative index - # that is, we want the index value of the row that is offset by rows into the - # tranche of this choosers alternatives created by cross join of alternatives and choosers + # need to get from an integer offset into the alternative sample to the alternative index + # that is, we want the index value of the row that is offset by rows into the + # tranche of this choosers alternatives created by cross join of alternatives and choosers - # offsets is the offset into model_design df of first row of chooser alternatives - offsets = np.arange(len(positions)) * alternative_count + # offsets is the offset into model_design df of first row of chooser alternatives + offsets = np.arange(len(positions)) * alternative_count - # choices and choice_probs have one element per chooser and is in same order as choosers - choices_array[i] = np.take(alts, positions + offsets) - choice_probs_array[i] = np.take(alt_probs_array, positions + offsets) + # choices and choice_probs have one element per chooser and is in same order as choosers + choices_array[i] = np.take(alts, positions + offsets) + choice_probs_array[i] = np.take(alt_probs_array, positions + offsets) - del positions - del offsets + del positions + del offsets - chunk.log_df(trace_label, 'choices_array', choices_array) - chunk.log_df(trace_label, 'choice_probs_array', choice_probs_array) + chunk.log_df(trace_label, 'choices_array', choices_array) + chunk.log_df(trace_label, 'choice_probs_array', choice_probs_array) - del alts - chunk.log_df(trace_label, 'alts', None) - del cum_probs_array - chunk.log_df(trace_label, 'cum_probs_array', None) - del alt_probs_array - chunk.log_df(trace_label, 'alt_probs_array', None) + del alts + chunk.log_df(trace_label, 'alts', None) + del cum_probs_array + chunk.log_df(trace_label, 'cum_probs_array', None) + del alt_probs_array + chunk.log_df(trace_label, 'alt_probs_array', None) - # explode to one row per chooser.index, alt_zone_id - choices_df = pd.DataFrame( - {alt_col_name: choices_array.flatten(order='F'), - 'rand': rands.flatten(order='F'), - 'prob': choice_probs_array.flatten(order='F'), - choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size) - }) + # explode to one row per chooser.index, alt_zone_id + choices_df = pd.DataFrame( + {alt_col_name: choices_array.flatten(order='F'), + 'rand': rands.flatten(order='F'), + 'prob': choice_probs_array.flatten(order='F'), + choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size) + }) - chunk.log_df(trace_label, 'choices_df', choices_df) + chunk.log_df(trace_label, 'choices_df', choices_df) - del choices_array - chunk.log_df(trace_label, 'choices_array', None) - del rands - chunk.log_df(trace_label, 'rands', None) - del choice_probs_array - chunk.log_df(trace_label, 'choice_probs_array', None) + del choices_array + chunk.log_df(trace_label, 'choices_array', None) + del rands + chunk.log_df(trace_label, 'rands', None) + del choice_probs_array + chunk.log_df(trace_label, 'choice_probs_array', None) - # handing this off to caller - chunk.log_df(trace_label, 'choices_df', None) + # handing this off to caller + chunk.log_df(trace_label, 'choices_df', None) return choices_df @@ -163,7 +204,8 @@ def _interaction_sample( log_alt_losers=False, skims=None, locals_d=None, - trace_label=None): + trace_label=None, + choose_individual_max_utility=False): """ Run a MNL simulation in the situation in which alternatives must be merged with choosers because there are interaction terms or @@ -314,8 +356,8 @@ def _interaction_sample( trace_label=trace_label, trace_choosers=choosers) chunk.log_df(trace_label, 'probs', probs) - del utilities - chunk.log_df(trace_label, 'utilities', None) + #del utilities + #chunk.log_df(trace_label, 'utilities', None) if have_trace_targets: tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), @@ -342,10 +384,15 @@ def _interaction_sample( choosers, probs, alternatives, sample_size, alternative_count, alt_col_name, allow_zero_probs=allow_zero_probs, - trace_label=trace_label) + trace_label=trace_label, + utilities=utilities, + choose_individual_max_utility=choose_individual_max_utility) chunk.log_df(trace_label, 'choices_df', choices_df) + del utilities + chunk.log_df(trace_label, 'utilities', None) + del probs chunk.log_df(trace_label, 'probs', None) @@ -395,7 +442,7 @@ def interaction_sample( allow_zero_probs=False, log_alt_losers=False, skims=None, locals_d=None, chunk_size=0, chunk_tag=None, - trace_label=None): + trace_label=None, choose_individual_max_utility=False): """ Run a simulation in the situation in which alternatives must @@ -476,7 +523,8 @@ def interaction_sample( log_alt_losers=log_alt_losers, skims=skims, locals_d=locals_d, - trace_label=chunk_trace_label) + trace_label=chunk_trace_label, + choose_individual_max_utility=choose_individual_max_utility) if choices.shape[0] > 0: # might not be any if allow_zero_probs From 00208e7bd3bccb20cec71c9df9cd7f1bb91f3a9b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 17 Jun 2022 10:24:49 +1000 Subject: [PATCH 046/135] adds interaction_sample at the module level, not individual model --- activitysim/abm/models/location_choice.py | 44 ++++++++++++------- activitysim/abm/models/trip_destination.py | 44 ++++++++++++------- .../abm/models/util/tour_destination.py | 38 +++++++++++----- 3 files changed, 85 insertions(+), 41 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 4968a8dcce..7be64b848d 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -109,7 +109,8 @@ def _location_sample( model_settings, alt_dest_col_name, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): """ select a sample of alternative locations. @@ -161,7 +162,9 @@ def _location_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility + ) return choices @@ -174,7 +177,8 @@ def location_sample( estimator, model_settings, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): # FIXME - MEMORY HACK - only include columns actually used in spec chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS'] @@ -198,7 +202,8 @@ def location_sample( model_settings, alt_dest_col_name, chunk_size, chunk_tag, - trace_label) + trace_label, + choose_individual_max_utility) return choices @@ -264,7 +269,8 @@ def location_presample( estimator, model_settings, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): trace_label = tracing.extend_trace_label(trace_label, 'presample') @@ -302,7 +308,8 @@ def location_presample( model_settings, DEST_TAZ, chunk_size, chunk_tag, - trace_label) + trace_label, + choose_individual_max_utility) # print(f"taz_sample\n{taz_sample}") # dest_TAZ prob pick_count @@ -329,7 +336,8 @@ def run_location_sample( estimator, model_settings, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): """ select a sample of alternative locations. @@ -372,7 +380,8 @@ def run_location_sample( model_settings, chunk_size, chunk_tag=f'{chunk_tag}.presample', - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) else: @@ -385,7 +394,8 @@ def run_location_sample( model_settings, chunk_size, chunk_tag=f'{chunk_tag}.sample', - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) return choices @@ -463,7 +473,7 @@ def run_location_simulate( estimator, model_settings, chunk_size, chunk_tag, - trace_label): + trace_label, choose_individual_max_utility): """ run location model on location_sample annotated with mode_choice logsum to select a dest zone from sample alternatives @@ -529,7 +539,7 @@ def run_location_simulate( trace_label=trace_label, trace_choice_name=model_settings['DEST_CHOICE_COLUMN_NAME'], estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + choose_individual_max_utility=choose_individual_max_utility ) if not want_logsums: @@ -551,7 +561,8 @@ def run_location_choice( estimator, model_settings, chunk_size, chunk_tag, - trace_hh_id, trace_label + trace_hh_id, trace_label, + choose_individual_max_utility, ): """ Run the three-part location choice algorithm to generate a location choice for each chooser @@ -614,7 +625,8 @@ def run_location_choice( model_settings, chunk_size, chunk_tag, # run_location_sample will add appropriate suffix for sample or presample - trace_label=tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name)) + trace_label=tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name), + choose_individual_max_utility=choose_individual_max_utility) # - location_logsums location_sample_df = \ @@ -639,7 +651,8 @@ def run_location_choice( estimator, model_settings, chunk_size, chunk_tag=f'{chunk_tag}.simulate', - trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name)) + trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name), + choose_individual_max_utility=choose_individual_max_utility) if estimator: if trace_hh_id: @@ -783,7 +796,8 @@ def iterate_location_choice( model_settings=model_settings, chunk_size=chunk_size, chunk_tag=chunk_tag, trace_hh_id=trace_hh_id, - trace_label=tracing.extend_trace_label(trace_label, 'i%s' % iteration)) + trace_label=tracing.extend_trace_label(trace_label, 'i%s' % iteration), + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) # choices_df is a pandas DataFrame with columns 'choice' and (optionally) 'logsum' if choices_df is None: diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index a5724c3540..31a99fa67b 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -58,7 +58,8 @@ def _destination_sample( estimator, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): """ Note: trips with no viable destination receive no sample rows @@ -109,8 +110,9 @@ def _destination_sample( skims=skims, locals_d=locals_dict, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label - ) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility + ) return choices @@ -124,7 +126,8 @@ def destination_sample( skim_hotel, estimator, chunk_size, - trace_label): + trace_label, + choose_individual_max_utility): chunk_tag = 'trip_destination.sample' @@ -142,7 +145,9 @@ def destination_sample( estimator, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility + ) return choices @@ -362,7 +367,8 @@ def destination_presample( network_los, estimator, chunk_size, trace_hh_id, - trace_label): + trace_label, + choose_individual_max_utility): trace_label = tracing.extend_trace_label(trace_label, 'presample') chunk_tag = 'trip_destination.presample' # distinguish from trip_destination.sample @@ -400,7 +406,8 @@ def destination_presample( estimator, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) # choose a MAZ for each DEST_TAZ choice, choice probability based on MAZ size_term fraction of TAZ total maz_sample = choose_MAZ_for_TAZ(taz_sample, size_term_matrix, trips, network_los, alt_dest_col_name, trace_label) @@ -419,7 +426,8 @@ def trip_destination_sample( skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label): + trace_label, + choose_individual_max_utility): """ Returns @@ -463,7 +471,8 @@ def trip_destination_sample( network_los, estimator, chunk_size, trace_hh_id, - trace_label) + trace_label, + choose_individual_max_utility) else: choices = destination_sample( @@ -475,7 +484,8 @@ def trip_destination_sample( skim_hotel, estimator, chunk_size, - trace_label) + trace_label, + choose_individual_max_utility) return choices @@ -654,7 +664,8 @@ def trip_destination_simulate( skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label): + trace_label, + choose_individual_max_utility): """ Chose destination from destination_sample (with od_logsum and dp_logsum columns added) @@ -700,7 +711,7 @@ def trip_destination_simulate( trace_label=trace_label, trace_choice_name='trip_dest', estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + choose_individual_max_utility=choose_individual_max_utility ) if not want_logsums: @@ -750,7 +761,8 @@ def choose_trip_destination( skim_hotel=skim_hotel, estimator=estimator, chunk_size=chunk_size, trace_hh_id=trace_hh_id, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) dropped_trips = ~trips.index.isin(destination_sample.index.unique()) if dropped_trips.any(): @@ -788,7 +800,8 @@ def choose_trip_destination( skim_hotel=skim_hotel, estimator=estimator, chunk_size=chunk_size, trace_hh_id=trace_hh_id, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) dropped_trips = ~trips.index.isin(destinations.index) if dropped_trips.any(): @@ -1041,7 +1054,8 @@ def run_trip_destination( size_term_matrix, skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label=tracing.extend_trace_label(nth_trace_label, primary_purpose)) + trace_label=tracing.extend_trace_label(nth_trace_label, primary_purpose), + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) choices_list.append(choices) if want_sample_table: diff --git a/activitysim/abm/models/util/tour_destination.py b/activitysim/abm/models/util/tour_destination.py index bafa6c5eed..5d184a9cb6 100644 --- a/activitysim/abm/models/util/tour_destination.py +++ b/activitysim/abm/models/util/tour_destination.py @@ -77,7 +77,8 @@ def _destination_sample( alt_dest_col_name, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): model_spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC', segment_name=spec_segment_name, estimator=estimator) @@ -110,7 +111,8 @@ def _destination_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) # remember person_id in chosen alts so we can merge with persons in subsequent steps # (broadcasts person_id onto all alternatives sharing the same tour_id index value) @@ -126,7 +128,8 @@ def destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label): + chunk_size, trace_label, + choose_individual_max_utility): chunk_tag = 'tour_destination.sample' @@ -153,7 +156,8 @@ def destination_sample( model_settings, alt_dest_col_name, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) return choices @@ -393,7 +397,8 @@ def destination_presample( network_los, destination_size_terms, estimator, - chunk_size, trace_label): + chunk_size, trace_label, + choose_individual_max_utility): trace_label = tracing.extend_trace_label(trace_label, 'presample') chunk_tag = 'tour_destination.presample' @@ -425,7 +430,9 @@ def destination_presample( model_settings, DEST_TAZ, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility + ) # choose a MAZ for each DEST_TAZ choice, choice probability based on MAZ size_term fraction of TAZ total maz_choices = choose_MAZ_for_TAZ(taz_sample, MAZ_size_terms, trace_label) @@ -444,7 +451,8 @@ def run_destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label): + chunk_size, trace_label, + choose_individual_max_utility): # FIXME - MEMORY HACK - only include columns actually used in spec (omit them pre-merge) chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS'] @@ -475,7 +483,9 @@ def run_destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label) + chunk_size, trace_label, + choose_individual_max_utility + ) else: choices = destination_sample( @@ -485,7 +495,9 @@ def run_destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label) + chunk_size, trace_label, + choose_individual_max_utility + ) # remember person_id in chosen alts so we can merge with persons in subsequent steps # (broadcasts person_id onto all alternatives sharing the same tour_id index value) @@ -693,7 +705,9 @@ def run_tour_destination( segment_destination_size_terms, estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(segment_trace_label, 'sample')) + trace_label=tracing.extend_trace_label(segment_trace_label, 'sample'), + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) # - destination_logsums tour_purpose = segment_name # tour_purpose is segment_name @@ -721,7 +735,9 @@ def run_tour_destination( destination_size_terms=segment_destination_size_terms, estimator=estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(segment_trace_label, 'simulate')) + trace_label=tracing.extend_trace_label(segment_trace_label, 'simulate'), + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) choices_list.append(choices) From 005b1311daa1ccf3bbdae6195d5d4d33db4bfe26 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 17 Jun 2022 10:29:01 +1000 Subject: [PATCH 047/135] adds interaction_sample at the module level, not individual model --- activitysim/abm/models/tour_od_choice.py | 3 +- activitysim/abm/models/util/tour_od.py | 38 ++++++++++++++++-------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/activitysim/abm/models/tour_od_choice.py b/activitysim/abm/models/tour_od_choice.py index 6816eed385..41705358e0 100644 --- a/activitysim/abm/models/tour_od_choice.py +++ b/activitysim/abm/models/tour_od_choice.py @@ -93,7 +93,8 @@ def tour_od_choice( model_settings, network_los, estimator, - chunk_size, trace_hh_id, trace_label) + chunk_size, trace_hh_id, trace_label, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if estimator: assert estimator.want_unsampled_alternatives diff --git a/activitysim/abm/models/util/tour_od.py b/activitysim/abm/models/util/tour_od.py index 1b47e02cdf..731aebe04a 100644 --- a/activitysim/abm/models/util/tour_od.py +++ b/activitysim/abm/models/util/tour_od.py @@ -115,7 +115,8 @@ def _od_sample( alt_od_col_name, chunk_size, chunk_tag, - trace_label): + trace_label, + choose_individual_max_utility): model_spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC', @@ -172,7 +173,8 @@ def _od_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label, + choose_individual_max_utility=choose_individual_max_utility) return choices @@ -477,7 +479,8 @@ def od_presample( destination_size_terms, estimator, chunk_size, - trace_label): + trace_label, + choose_individual_max_utility): trace_label = tracing.extend_trace_label(trace_label, 'presample') chunk_tag = 'tour_od.presample' @@ -507,7 +510,8 @@ def od_presample( alt_od_col_name, chunk_size, chunk_tag, - trace_label) + trace_label, + choose_individual_max_utility) orig_MAZ_dest_TAZ_sample[ORIG_MAZ] = orig_MAZ_dest_TAZ_sample[alt_od_col_name].str.split('_').str[0].astype(int) orig_MAZ_dest_TAZ_sample[DEST_TAZ] = orig_MAZ_dest_TAZ_sample[alt_od_col_name].str.split('_').str[1].astype(int) @@ -582,7 +586,8 @@ def run_od_sample( destination_size_terms, estimator, chunk_size, - trace_label): + trace_label, + choose_individual_max_utility): model_spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC', segment_name=spec_segment_name, estimator=estimator) @@ -616,7 +621,9 @@ def run_od_sample( destination_size_terms, estimator, chunk_size, - trace_label) + trace_label, + choose_individual_max_utility + ) else: choices = od_sample( @@ -626,7 +633,9 @@ def run_od_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label) + chunk_size, trace_label, + choose_individual_max_utility + ) return choices @@ -782,7 +791,8 @@ def run_od_simulate( destination_size_terms, estimator, chunk_size, - trace_label): + trace_label, + choose_individual_max_utility): """ run simulate OD choices on tour_od_sample annotated with mode_choice logsum to select a tour OD from sample alternatives @@ -854,7 +864,7 @@ def run_od_simulate( trace_label=trace_label, trace_choice_name='origin_destination', estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + choose_individual_max_utility=choose_individual_max_utility) if not want_logsums: choices = choices.to_frame('choice') @@ -872,7 +882,8 @@ def run_tour_od( model_settings, network_los, estimator, - chunk_size, trace_hh_id, trace_label): + chunk_size, trace_hh_id, trace_label, + choose_individual_max_utility): size_term_calculator = SizeTermCalculator(model_settings['SIZE_TERM_SELECTOR']) preprocessor_settings = model_settings.get('preprocessor', None) @@ -923,7 +934,9 @@ def run_tour_od( estimator, chunk_size=chunk_size, trace_label=tracing.extend_trace_label( - trace_label, 'sample.%s' % segment_name)) + trace_label, 'sample.%s' % segment_name), + choose_individual_max_utility=choose_individual_max_utility + ) if model_settings['ORIG_FILTER'] == 'original_MAZ > 0': pass @@ -962,7 +975,8 @@ def run_tour_od( destination_size_terms=segment_destination_size_terms, estimator=estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name)) + trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name), + choose_individual_max_utility=choose_individual_max_utility) choices_list.append(choices) if estimator: From bbce6aa028ca1f6cdfbaaea7312fc9bd5ad76ce7 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 13:52:33 +1000 Subject: [PATCH 048/135] nested index instead of name --- notebooks/fru_utils.py | 16 +++-- notebooks/validate_frozen_impl.ipynb | 88 +++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index 645db8f174..d12b1cc7ee 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -143,9 +143,11 @@ def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_sa choices.rename(columns={'logsum': logsum_column_name, 'choice': mode_column_name}, inplace=True) - if not choose_individual_max_utility: - alts = spec.columns - choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) + + # THIS SHOULD NOW BE CONSISTENT + #if not choose_individual_max_utility: + alts = spec.columns + choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) finalise = True if finalise: @@ -304,9 +306,11 @@ def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=T choices.rename(columns={'logsum': logsum_column_name, 'choice': mode_column_name}, inplace=True) - if not choose_individual_max_utility: - alts = spec.columns - choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) + + # SHOULD NOW BE CONSISTENT + #if not choose_individual_max_utility: + alts = spec.columns + choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) ################ choices_list.append(choices) choices_df_asim = pd.concat(choices_list) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 62f8405acd..e87bea3e42 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "execution": { @@ -26,14 +26,23 @@ }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "execution": { @@ -52,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 231, + "execution_count": 5, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -86,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -105,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -125,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -144,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -169,7 +178,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "outputs": [], "source": [ "## need to do this before loading checkpoint tables\n", @@ -185,6 +194,61 @@ } } }, + { + "cell_type": "code", + "execution_count": 11, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Couldn't find checkpoint 'trip_scheduling' in checkpoints\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 init\n", + "1 initialize_landuse\n", + "2 initialize_households\n", + "3 compute_accessibility\n", + "4 school_location\n", + "Name: checkpoint_name, dtype: object\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "Couldn't find checkpoint 'trip_scheduling' in checkpoints", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mIndexError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mload_checkpoint\u001B[0;34m(checkpoint_name)\u001B[0m\n\u001B[1;32m 353\u001B[0m \u001B[0;31m# truncate rows after target checkpoint\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 354\u001B[0;31m \u001B[0mi\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcheckpoints\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mcheckpoints\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mCHECKPOINT_NAME\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0mcheckpoint_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 355\u001B[0m \u001B[0mcheckpoints\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcheckpoints\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mloc\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0mi\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001B[0m in \u001B[0;36m__getitem__\u001B[0;34m(self, key)\u001B[0m\n\u001B[1;32m 4603\u001B[0m \u001B[0mkey\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcom\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcast_scalar_indexer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mwarn_float\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 4604\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mgetitem\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 4605\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mIndexError\u001B[0m: index 0 is out of bounds for axis 0 with size 0", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001B[0;31mRuntimeError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/notebooks/fru_utils.py\u001B[0m in \u001B[0;36mrun_trip_mode_choice\u001B[0;34m(do_these_purposes, choose_individual_max_utility)\u001B[0m\n\u001B[1;32m 190\u001B[0m \u001B[0mchunk_size\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m0\u001B[0m \u001B[0;31m# test_mtc means no chunking\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 191\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 192\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mopen_pipeline\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 193\u001B[0m \u001B[0;31m# preload any bulky injectables (e.g. skims) not in pipeline\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 194\u001B[0m \u001B[0minject\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mget_injectable\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'preload_injectables'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mopen_pipeline\u001B[0;34m(resume_after)\u001B[0m\n\u001B[1;32m 513\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdebug\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"open_pipeline - open existing pipeline\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 514\u001B[0m \u001B[0mopen_pipeline_store\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moverwrite\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 515\u001B[0;31m \u001B[0mload_checkpoint\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 516\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[0;31m# open new, empty pipeline\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mload_checkpoint\u001B[0;34m(checkpoint_name)\u001B[0m\n\u001B[1;32m 362\u001B[0m \u001B[0mprint\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcheckpoints\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mCHECKPOINT_NAME\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 363\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0merror\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 364\u001B[0;31m \u001B[0;32mraise\u001B[0m \u001B[0mRuntimeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 365\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 366\u001B[0m \u001B[0;31m# convert pandas dataframe back to array of checkpoint dicts\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mRuntimeError\u001B[0m: Couldn't find checkpoint 'trip_scheduling' in checkpoints" + ] + } + ], + "source": [ + "%%time\n", + "trips_df_ru = run_trip_mode_choice()\n", + "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": 9, @@ -254,11 +318,7 @@ ] } ], - "source": [ - "%%time\n", - "trips_df_ru = run_trip_mode_choice()\n", - "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" - ] + "source": [] }, { "cell_type": "code", From 7fbba93275cca185207023ea465a3ebb7caa109a Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 14:03:39 +1000 Subject: [PATCH 049/135] fies frozen_rand setting call --- .../abm/models/util/tour_destination.py | 5 +- notebooks/validate_frozen_impl.ipynb | 415 +++++++++++++++++- 2 files changed, 397 insertions(+), 23 deletions(-) diff --git a/activitysim/abm/models/util/tour_destination.py b/activitysim/abm/models/util/tour_destination.py index 5d184a9cb6..7e3303c530 100644 --- a/activitysim/abm/models/util/tour_destination.py +++ b/activitysim/abm/models/util/tour_destination.py @@ -578,7 +578,8 @@ def run_destination_simulate( network_los, destination_size_terms, estimator, - chunk_size, trace_label): + chunk_size, trace_label, + choose_individual_max_utility): """ run destination_simulate on tour_destination_sample annotated with mode_choice logsum to select a destination from sample alternatives @@ -645,7 +646,7 @@ def run_destination_simulate( trace_label=trace_label, trace_choice_name='destination', estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + choose_individual_max_utility=choose_individual_max_utility) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index e87bea3e42..23eba842b3 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "execution": { @@ -26,23 +26,14 @@ }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "execution": { @@ -61,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "execution": { @@ -95,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "execution": { @@ -114,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -134,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -153,7 +144,22 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 15, + "outputs": [], + "source": [ + "from activitysim.core import pipeline\n", + "pipeline.close_pipeline()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 14, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -165,12 +171,379 @@ }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Configured logging using basicConfig\n", + "INFO - Configured logging using basicConfig\n", + "INFO - Read logging configuration from: configs/logging.yaml\n", + "INFO - SETTING configs_dir: ['configs']\n", + "INFO - SETTING settings_file_name: settings.yaml\n", + "INFO - SETTING data_dir: ['data']\n", + "INFO - SETTING output_dir: output\n", + "INFO - SETTING households_sample_size: 100\n", + "INFO - SETTING chunk_size: 0\n", + "INFO - SETTING chunk_method: hybrid_uss\n", + "INFO - SETTING chunk_training_mode: disabled\n", + "INFO - SETTING multiprocess: None\n", + "INFO - SETTING num_processes: None\n", + "INFO - SETTING resume_after: None\n", + "INFO - SETTING trace_hh_id: [982875]\n", + "INFO - ENV MKL_NUM_THREADS: None\n", + "INFO - ENV OMP_NUM_THREADS: None\n", + "INFO - ENV OPENBLAS_NUM_THREADS: None\n", + "INFO - NUMPY blas_opt_info libraries: ['openblas', 'openblas']\n", + "INFO - NUMPY lapack_opt_info libraries: ['openblas', 'openblas']\n", + "INFO - run single process simulation\n", + "INFO - Time to execute open_pipeline : 0.017 seconds (0.0 minutes)\n", + "INFO - preload_injectables\n", + "INFO - Time to execute preload_injectables : 0.089 seconds (0.0 minutes)\n", + "INFO - #run_model running step initialize_landuse\n", + "Running step 'initialize_landuse'\n", + "INFO - Reading CSV file data/land_use.csv\n", + "INFO - loaded land_use (25, 24)\n", + "INFO - initialize_landuse.annotate_tables - annotating land_use SPEC annotate_landuse\n", + "INFO - Network_LOS using skim_dict_factory: NumpyArraySkimFactory\n", + "INFO - allocate_skim_buffer shared False taz shape (826, 25, 25) total size: 2_065_000 (2.1 MB)\n", + "INFO - _read_skims_from_omx data/skims.omx\n", + "INFO - _read_skims_from_omx loaded 826 skims from data/skims.omx\n", + "INFO - writing skim cache taz (826, 25, 25) to output/cache/cached_taz.mmap\n", + "INFO - load_skims_to_buffer taz shape (826, 25, 25)\n", + "INFO - get_skim_data taz SkimData shape (826, 25, 25)\n", + "INFO - SkimDict init taz\n", + "INFO - SkimDict.build_3d_skim_block_offset_table registered 167 3d keys\n", + "Time to execute step 'initialize_landuse': 2.22 s\n", + "Total time to execute iteration 1 with iteration value None: 2.22 s\n", + "INFO - #run_model running step initialize_households\n", + "Running step 'initialize_households'\n", + "WARNING - setting trace_hh_id is wrong type, should be an int, but was \n", + "INFO - Reading CSV file data/households.csv\n", + "INFO - full household list contains 5000 households\n", + "INFO - sampling 100 of 5000 households\n", + "INFO - loaded households (100, 7)\n", + "INFO - Reading CSV file data/persons.csv\n", + "INFO - loaded persons (167, 7)\n", + "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons\n", + "INFO - initialize_households.annotate_tables - annotating households SPEC annotate_households\n", + "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons_after_hh\n", + "Time to execute step 'initialize_households': 0.41 s\n", + "Total time to execute iteration 1 with iteration value None: 0.41 s\n", + "INFO - #run_model running step compute_accessibility\n", + "Running step 'compute_accessibility'\n", + "INFO - Running compute_accessibility with 25 orig zones 25 dest zones\n", + "INFO - compute_accessibility Running adaptive_chunked_choosers with 25 choosers\n", + "INFO - Running chunk 1 of 1 with 25 of 25 choosers\n", + "INFO - Running compute_accessibility with 25 orig zones 25 dest zones\n", + "INFO - compute_accessibility computed accessibilities (25, 10)\n", + "Time to execute step 'compute_accessibility': 0.08 s\n", + "Total time to execute iteration 1 with iteration value None: 0.08 s\n", + "INFO - #run_model running step school_location\n", + "Running step 'school_location'\n", + "INFO - Running school_location.i1.sample.university with 17 persons\n", + "INFO - school_location.i1.sample.university.interaction_sample Running adaptive_chunked_choosers with 17 choosers\n", + "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", + "INFO - Running eval_interaction_utilities on 102 rows\n", + "INFO - Running school_location.i1.logsums.university with 45 rows\n", + "INFO - school_location.i1.logsums.university.compute_logsums Running adaptive_chunked_choosers with 45 choosers\n", + "INFO - Running chunk 1 of 1 with 45 of 45 choosers\n", + "INFO - Running school_location.i1.simulate.university with 17 persons\n", + "INFO - school_location.i1.simulate.university.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 17 choosers and 45 alternatives\n", + "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", + "INFO - Running eval_interaction_utilities on 45 rows\n", + "INFO - Running school_location.i1.sample.highschool with 5 persons\n", + "INFO - school_location.i1.sample.highschool.interaction_sample Running adaptive_chunked_choosers with 5 choosers\n", + "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", + "INFO - Running eval_interaction_utilities on 10 rows\n", + "INFO - Running school_location.i1.logsums.highschool with 5 rows\n", + "INFO - school_location.i1.logsums.highschool.compute_logsums Running adaptive_chunked_choosers with 5 choosers\n", + "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", + "INFO - Running school_location.i1.simulate.highschool with 5 persons\n", + "INFO - school_location.i1.simulate.highschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 5 choosers and 5 alternatives\n", + "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", + "INFO - Running eval_interaction_utilities on 5 rows\n", + "INFO - Running school_location.i1.sample.gradeschool with 17 persons\n", + "INFO - school_location.i1.sample.gradeschool.interaction_sample Running adaptive_chunked_choosers with 17 choosers\n", + "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", + "INFO - Running eval_interaction_utilities on 425 rows\n", + "INFO - Running school_location.i1.logsums.gradeschool with 165 rows\n", + "INFO - school_location.i1.logsums.gradeschool.compute_logsums Running adaptive_chunked_choosers with 165 choosers\n", + "INFO - Running chunk 1 of 1 with 165 of 165 choosers\n", + "INFO - Running school_location.i1.simulate.gradeschool with 17 persons\n", + "INFO - school_location.i1.simulate.gradeschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 17 choosers and 165 alternatives\n", + "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", + "INFO - Running eval_interaction_utilities on 165 rows\n", + "INFO - write_trace_files iteration 1\n", + "INFO - school_location_logsum top 10 value counts:\n", + "10.852186 1\n", + "12.155904 1\n", + "20.491487 1\n", + "10.513007 1\n", + "20.774530 1\n", + "20.404239 1\n", + "20.284262 1\n", + "10.804509 1\n", + "10.180578 1\n", + "20.918083 1\n", + "Name: logsum, dtype: int64\n", + "Time to execute step 'school_location': 3.48 s\n", + "Total time to execute iteration 1 with iteration value None: 3.48 s\n", + "INFO - #run_model running step workplace_location\n", + "Running step 'workplace_location'\n", + "INFO - Running workplace_location.i1.sample.work_low with 37 persons\n", + "INFO - workplace_location.i1.sample.work_low.interaction_sample Running adaptive_chunked_choosers with 37 choosers\n", + "INFO - Running chunk 1 of 1 with 37 of 37 choosers\n", + "INFO - Running eval_interaction_utilities on 925 rows\n", + "INFO - Running workplace_location.i1.logsums.work_low with 511 rows\n", + "INFO - workplace_location.i1.logsums.work_low.compute_logsums Running adaptive_chunked_choosers with 511 choosers\n", + "INFO - Running chunk 1 of 1 with 511 of 511 choosers\n", + "INFO - Running workplace_location.i1.simulate.work_low with 37 persons\n", + "INFO - workplace_location.i1.simulate.work_low.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 37 choosers and 511 alternatives\n", + "INFO - Running chunk 1 of 1 with 37 of 37 choosers\n", + "INFO - Running eval_interaction_utilities on 511 rows\n", + "INFO - Running workplace_location.i1.sample.work_med with 26 persons\n", + "INFO - workplace_location.i1.sample.work_med.interaction_sample Running adaptive_chunked_choosers with 26 choosers\n", + "INFO - Running chunk 1 of 1 with 26 of 26 choosers\n", + "INFO - Running eval_interaction_utilities on 650 rows\n", + "INFO - Running workplace_location.i1.logsums.work_med with 356 rows\n", + "INFO - workplace_location.i1.logsums.work_med.compute_logsums Running adaptive_chunked_choosers with 356 choosers\n", + "INFO - Running chunk 1 of 1 with 356 of 356 choosers\n", + "INFO - Running workplace_location.i1.simulate.work_med with 26 persons\n", + "INFO - workplace_location.i1.simulate.work_med.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 26 choosers and 356 alternatives\n", + "INFO - Running chunk 1 of 1 with 26 of 26 choosers\n", + "INFO - Running eval_interaction_utilities on 356 rows\n", + "INFO - Running workplace_location.i1.sample.work_high with 16 persons\n", + "INFO - workplace_location.i1.sample.work_high.interaction_sample Running adaptive_chunked_choosers with 16 choosers\n", + "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", + "INFO - Running eval_interaction_utilities on 400 rows\n", + "INFO - Running workplace_location.i1.logsums.work_high with 224 rows\n", + "INFO - workplace_location.i1.logsums.work_high.compute_logsums Running adaptive_chunked_choosers with 224 choosers\n", + "INFO - Running chunk 1 of 1 with 224 of 224 choosers\n", + "INFO - Running workplace_location.i1.simulate.work_high with 16 persons\n", + "INFO - workplace_location.i1.simulate.work_high.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 16 choosers and 224 alternatives\n", + "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", + "INFO - Running eval_interaction_utilities on 224 rows\n", + "INFO - Running workplace_location.i1.sample.work_veryhigh with 18 persons\n", + "INFO - workplace_location.i1.sample.work_veryhigh.interaction_sample Running adaptive_chunked_choosers with 18 choosers\n", + "INFO - Running chunk 1 of 1 with 18 of 18 choosers\n", + "INFO - Running eval_interaction_utilities on 450 rows\n", + "INFO - Running workplace_location.i1.logsums.work_veryhigh with 259 rows\n", + "INFO - workplace_location.i1.logsums.work_veryhigh.compute_logsums Running adaptive_chunked_choosers with 259 choosers\n", + "INFO - Running chunk 1 of 1 with 259 of 259 choosers\n", + "INFO - Running workplace_location.i1.simulate.work_veryhigh with 18 persons\n", + "INFO - workplace_location.i1.simulate.work_veryhigh.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 18 choosers and 259 alternatives\n", + "INFO - Running chunk 1 of 1 with 18 of 18 choosers\n", + "INFO - Running eval_interaction_utilities on 259 rows\n", + "INFO - write_trace_files iteration 1\n", + "INFO - workplace_location_logsum top 10 value counts:\n", + "15.462302 1\n", + "14.407126 1\n", + "15.624373 1\n", + "15.654193 1\n", + "15.696199 1\n", + "15.701733 1\n", + "15.527734 1\n", + "15.547906 1\n", + "13.788314 1\n", + "13.903170 1\n", + "Name: logsum, dtype: int64\n", + "Time to execute step 'workplace_location': 5.28 s\n", + "Total time to execute iteration 1 with iteration value None: 5.28 s\n", + "INFO - #run_model running step auto_ownership_simulate\n", + "Running step 'auto_ownership_simulate'\n", + "INFO - Running auto_ownership_simulate with 100 households\n", + "INFO - auto_ownership_simulate.simple_simulate Running adaptive_chunked_choosers with 100 choosers\n", + "INFO - Running chunk 1 of 1 with 100 of 100 choosers\n", + "INFO - auto_ownership top 10 value counts:\n", + "0 67\n", + "1 32\n", + "2 1\n", + "Name: auto_ownership, dtype: int64\n", + "Time to execute step 'auto_ownership_simulate': 0.15 s\n", + "Total time to execute iteration 1 with iteration value None: 0.15 s\n", + "INFO - #run_model running step free_parking\n", + "Running step 'free_parking'\n", + "INFO - Running free_parking with 97 persons\n", + "INFO - free_parking.simple_simulate Running adaptive_chunked_choosers with 97 choosers\n", + "INFO - Running chunk 1 of 1 with 97 of 97 choosers\n", + "INFO - free_parking top 10 value counts:\n", + "False 163\n", + "True 4\n", + "Name: free_parking_at_work, dtype: int64\n", + "Time to execute step 'free_parking': 0.11 s\n", + "Total time to execute iteration 1 with iteration value None: 0.11 s\n", + "INFO - #run_model running step cdap_simulate\n", + "Running step 'cdap_simulate'\n", + "INFO - Pre-building cdap specs\n", + "INFO - Time to execute build_cdap_spec hh_size 2 : 0.115 seconds (0.0 minutes)\n", + "INFO - Time to execute build_cdap_spec hh_size 3 : 0.495 seconds (0.0 minutes)\n", + "INFO - Time to execute build_cdap_spec hh_size 4 : 1.316 seconds (0.0 minutes)\n", + "INFO - Time to execute build_cdap_spec hh_size 5 : 2.535 seconds (0.0 minutes)\n", + "INFO - Running cdap_simulate with 167 persons\n", + "INFO - cdap.cdap Running chunk 1 of 1 with 100 of 100 choosers\n", + "INFO - cdap_activity top 10 value counts:\n", + "M 89\n", + "N 50\n", + "H 28\n", + "Name: cdap_activity, dtype: int64\n", + "INFO - cdap crosstabs:\n", + "cdap_activity H M N All\n", + "ptype \n", + "1 3 43 6 52\n", + "2 8 19 7 34\n", + "3 1 11 5 17\n", + "4 4 0 17 21\n", + "5 8 0 13 21\n", + "6 4 1 0 5\n", + "7 0 9 2 11\n", + "8 0 6 0 6\n", + "All 28 89 50 167\n", + "Time to execute step 'cdap_simulate': 7.89 s\n", + "Total time to execute iteration 1 with iteration value None: 7.89 s\n", + "INFO - #run_model running step mandatory_tour_frequency\n", + "Running step 'mandatory_tour_frequency'\n", + "INFO - Running mandatory_tour_frequency with 89 persons\n", + "INFO - mandatory_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 89 choosers\n", + "INFO - Running chunk 1 of 1 with 89 of 89 choosers\n", + "INFO - mandatory_tour_frequency top 10 value counts:\n", + " 78\n", + "work1 68\n", + "school1 21\n", + "Name: mandatory_tour_frequency, dtype: int64\n", + "Time to execute step 'mandatory_tour_frequency': 0.68 s\n", + "Total time to execute iteration 1 with iteration value None: 0.68 s\n", + "INFO - #run_model running step mandatory_tour_scheduling\n", + "Running step 'mandatory_tour_scheduling'\n", + "DEBUG - @inject timetable\n", + "INFO - Running mandatory_tour_scheduling with 89 tours\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 68 tour choices\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work Running adaptive_chunked_choosers with 68 choosers\n", + "INFO - Running chunk 1 of 1 with 68 of 68 choosers\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 68 tour choices\n", + "INFO - tdd_alt_segments specified for representative logsums\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 12920 to 1020 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums compute_logsums for 1020 choosers 1020 alts\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums Running adaptive_chunked_choosers with 1020 choosers\n", + "INFO - Running chunk 1 of 1 with 1020 of 1020 choosers\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 68 choosers and 12920 alternatives\n", + "INFO - Running chunk 1 of 1 with 68 of 68 choosers\n", + "INFO - Running eval_interaction_utilities on 12920 rows\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 16 tour choices\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school Running adaptive_chunked_choosers with 16 choosers\n", + "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 16 tour choices\n", + "INFO - tdd_alt_segments specified for representative logsums\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 3040 to 240 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums compute_logsums for 240 choosers 240 alts\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums Running adaptive_chunked_choosers with 240 choosers\n", + "INFO - Running chunk 1 of 1 with 240 of 240 choosers\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 16 choosers and 3040 alternatives\n", + "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", + "INFO - Running eval_interaction_utilities on 3040 rows\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 5 tour choices\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ Running adaptive_chunked_choosers with 5 choosers\n", + "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 5 tour choices\n", + "INFO - tdd_alt_segments specified for representative logsums\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 950 to 75 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums compute_logsums for 75 choosers 75 alts\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums Running adaptive_chunked_choosers with 75 choosers\n", + "INFO - Running chunk 1 of 1 with 75 of 75 choosers\n", + "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 5 choosers and 950 alternatives\n", + "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", + "INFO - Running eval_interaction_utilities on 950 rows\n", + "Time to execute step 'mandatory_tour_scheduling': 3.83 s\n", + "Total time to execute iteration 1 with iteration value None: 3.83 s\n", + "INFO - #run_model running step joint_tour_frequency\n", + "Running step 'joint_tour_frequency'\n", + "INFO - Running joint_tour_frequency with 36 multi-person households\n", + "DEBUG - @inject timetable\n", + "INFO - joint_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 36 choosers\n", + "INFO - Running chunk 1 of 1 with 36 of 36 choosers\n", + "INFO - joint_tour_frequency top 10 value counts:\n", + "0_tours 96\n", + "1_Disc 2\n", + "1_Shop 1\n", + "1_Eat 1\n", + "Name: joint_tour_frequency, dtype: int64\n", + "Time to execute step 'joint_tour_frequency': 0.38 s\n", + "Total time to execute iteration 1 with iteration value None: 0.38 s\n", + "INFO - #run_model running step joint_tour_composition\n", + "Running step 'joint_tour_composition'\n", + "INFO - Running joint_tour_composition with 4 joint tours\n", + "DEBUG - @inject timetable\n", + "INFO - joint_tour_composition.simple_simulate Running adaptive_chunked_choosers with 4 choosers\n", + "INFO - Running chunk 1 of 1 with 4 of 4 choosers\n", + "INFO - joint_tour_composition top 10 value counts:\n", + "adults 2\n", + "children 1\n", + "mixed 1\n", + "Name: composition, dtype: int64\n", + "Time to execute step 'joint_tour_composition': 0.24 s\n", + "Total time to execute iteration 1 with iteration value None: 0.24 s\n", + "INFO - #run_model running step joint_tour_participation\n", + "Running step 'joint_tour_participation'\n", + "INFO - Running joint_tours_participation with 8 potential participants (candidates)\n", + "DEBUG - @inject timetable\n", + "INFO - joint_tour_participation Running chunk 1 of 1 with 4 of 4 choosers\n", + "INFO - joint_tour_participation.eval_mnl.participants_chooser 4 joint tours to satisfy.\n", + "INFO - joint_tour_participation.eval_mnl.participants_chooser 1 iterations to satisfy all joint tours.\n", + "Time to execute step 'joint_tour_participation': 0.46 s\n", + "Total time to execute iteration 1 with iteration value None: 0.46 s\n", + "INFO - #run_model running step joint_tour_destination\n", + "Running step 'joint_tour_destination'\n", + "INFO - running joint_tour_destination.shopping.sample with 1 tours\n", + "INFO - joint_tour_destination.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 1 choosers\n", + "INFO - Running chunk 1 of 1 with 1 of 1 choosers\n", + "INFO - Running eval_interaction_utilities on 25 rows\n", + "INFO - Running joint_tour_destination.shopping.logsums with 11 rows\n", + "INFO - joint_tour_destination.shopping.logsums.compute_logsums Running adaptive_chunked_choosers with 11 choosers\n", + "INFO - Running chunk 1 of 1 with 11 of 11 choosers\n", + "INFO - Time to execute all models until this error : 29.773 seconds (0.5 minutes)\n", + "ERROR - activitysim run encountered an unrecoverable error\n", + "Traceback (most recent call last):\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/cli/run.py\", line 261, in run\n", + " pipeline.run(models=config.setting('models'), resume_after=resume_after)\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\", line 617, in run\n", + " run_model(model)\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\", line 476, in run_model\n", + " orca.run([step_name])\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\", line 2168, in run\n", + " step()\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\", line 973, in __call__\n", + " return self._func(**kwargs)\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/joint_tour_destination.py\", line 68, in joint_tour_destination\n", + " choices_df, save_sample_df = tour_destination.run_tour_destination(\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/util/tour_destination.py\", line 727, in run_tour_destination\n", + " run_destination_simulate(\n", + "TypeError: run_destination_simulate() got an unexpected keyword argument 'choose_individual_max_utility'\n" + ] + }, + { + "ename": "TypeError", + "evalue": "run_destination_simulate() got an unexpected keyword argument 'choose_individual_max_utility'", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/tmp/ipykernel_2932/4273387002.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 2\u001B[0m \u001B[0mrun\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0madd_run_args\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mparser\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[0margs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mparser\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mparse_args\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m'-c'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'configs'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'-o'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'output'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'-d'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'data'\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m \u001B[0mrun\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;31m# 2mins full example run\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 5\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0minject\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mis_injectable\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'preload_injectables'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 6\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mactivitysim\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mabm\u001B[0m \u001B[0;31m# register abm steps and other abm-specific injectables\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/cli/run.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(args)\u001B[0m\n\u001B[1;32m 259\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'run single process simulation'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 260\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 261\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodels\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetting\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'models'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mresume_after\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 262\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 263\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetting\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'cleanup_pipeline_after_run'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(models, resume_after)\u001B[0m\n\u001B[1;32m 615\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mmodel\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mmodels\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 616\u001B[0m \u001B[0mt1\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mprint_elapsed_time\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 617\u001B[0;31m \u001B[0mrun_model\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodel\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 618\u001B[0m \u001B[0mmem\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrace_memory_info\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"pipeline.run after {model}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 619\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mrun_model\u001B[0;34m(model_name)\u001B[0m\n\u001B[1;32m 474\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"#run_model running step {step_name}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 475\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 476\u001B[0;31m \u001B[0morca\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mstep_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 477\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 478\u001B[0m \u001B[0mt0\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mprint_elapsed_time\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"#run_model completed step '%s'\"\u001B[0m \u001B[0;34m%\u001B[0m \u001B[0mmodel_name\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mt0\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdebug\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(steps, iter_vars, data_out, out_interval, out_base_tables, out_run_tables, compress, out_base_local, out_run_local)\u001B[0m\n\u001B[1;32m 2166\u001B[0m \u001B[0mstep\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mget_step\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mstep_name\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 2167\u001B[0m \u001B[0mt2\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 2168\u001B[0;31m \u001B[0mstep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 2169\u001B[0m print(\"Time to execute step '{}': {:.2f} s\".format(\n\u001B[1;32m 2170\u001B[0m step_name, time.time() - t2))\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 971\u001B[0m kwargs = _collect_variables(names=self._argspec.args,\n\u001B[1;32m 972\u001B[0m expressions=self._argspec.defaults)\n\u001B[0;32m--> 973\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_func\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 974\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 975\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0m_tables_used\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/joint_tour_destination.py\u001B[0m in \u001B[0;36mjoint_tour_destination\u001B[0;34m(tours, persons_merged, households_merged, network_los, chunk_size, trace_hh_id)\u001B[0m\n\u001B[1;32m 66\u001B[0m \u001B[0mestimator\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mwrite_model_settings\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodel_settings\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmodel_settings_file_name\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 67\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 68\u001B[0;31m choices_df, save_sample_df = tour_destination.run_tour_destination(\n\u001B[0m\u001B[1;32m 69\u001B[0m \u001B[0mtours\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 70\u001B[0m \u001B[0mpersons_merged\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/util/tour_destination.py\u001B[0m in \u001B[0;36mrun_tour_destination\u001B[0;34m(tours, persons_merged, want_logsums, want_sample_table, model_settings, network_los, estimator, chunk_size, trace_hh_id, trace_label)\u001B[0m\n\u001B[1;32m 725\u001B[0m \u001B[0mspec_segment_name\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0msegment_name\u001B[0m \u001B[0;31m# spec_segment_name is segment_name\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 726\u001B[0m \u001B[0mchoices\u001B[0m \u001B[0;34m=\u001B[0m\u001B[0;31m \u001B[0m\u001B[0;31m\\\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 727\u001B[0;31m run_destination_simulate(\n\u001B[0m\u001B[1;32m 728\u001B[0m \u001B[0mspec_segment_name\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 729\u001B[0m \u001B[0mchoosers\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mTypeError\u001B[0m: run_destination_simulate() got an unexpected keyword argument 'choose_individual_max_utility'" + ] + } + ], "source": [ "parser = argparse.ArgumentParser()\n", "run.add_run_args(parser)\n", "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", - "#run.run(args) # 2mins full example run\n", + "run.run(args) # 2mins full example run\n", "if not inject.is_injectable('preload_injectables'):\n", " from activitysim import abm # register abm steps and other abm-specific injectables\n", "run.handle_standard_args(args) # possibly update injectables" @@ -178,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "outputs": [], "source": [ "## need to do this before loading checkpoint tables\n", @@ -196,7 +569,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "outputs": [ { "name": "stderr", From 8c1d1ed2b2d71c0e3673b115d121df04157ed6a1 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 14:30:45 +1000 Subject: [PATCH 050/135] pass arguments in --- activitysim/abm/models/trip_destination.py | 3 +- notebooks/validate_frozen_impl.ipynb | 334 ++------------------- 2 files changed, 21 insertions(+), 316 deletions(-) diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index 31a99fa67b..1042197b83 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -745,7 +745,8 @@ def choose_trip_destination( size_term_matrix, skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label): + trace_label, + choose_individual_max_utility): logger.info("choose_trip_destination %s with %d trips", trace_label, trips.shape[0]) diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb index 23eba842b3..cf89790669 100644 --- a/notebooks/validate_frozen_impl.ipynb +++ b/notebooks/validate_frozen_impl.ipynb @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 19, "outputs": [], "source": [ "from activitysim.core import pipeline\n", @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 18, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -197,310 +197,11 @@ "INFO - NUMPY blas_opt_info libraries: ['openblas', 'openblas']\n", "INFO - NUMPY lapack_opt_info libraries: ['openblas', 'openblas']\n", "INFO - run single process simulation\n", - "INFO - Time to execute open_pipeline : 0.017 seconds (0.0 minutes)\n", - "INFO - preload_injectables\n", - "INFO - Time to execute preload_injectables : 0.089 seconds (0.0 minutes)\n", + "INFO - Time to execute open_pipeline : 0.163 seconds (0.0 minutes)\n", + "INFO - Time to execute preload_injectables : 0.08 seconds (0.0 minutes)\n", "INFO - #run_model running step initialize_landuse\n", "Running step 'initialize_landuse'\n", - "INFO - Reading CSV file data/land_use.csv\n", - "INFO - loaded land_use (25, 24)\n", - "INFO - initialize_landuse.annotate_tables - annotating land_use SPEC annotate_landuse\n", - "INFO - Network_LOS using skim_dict_factory: NumpyArraySkimFactory\n", - "INFO - allocate_skim_buffer shared False taz shape (826, 25, 25) total size: 2_065_000 (2.1 MB)\n", - "INFO - _read_skims_from_omx data/skims.omx\n", - "INFO - _read_skims_from_omx loaded 826 skims from data/skims.omx\n", - "INFO - writing skim cache taz (826, 25, 25) to output/cache/cached_taz.mmap\n", - "INFO - load_skims_to_buffer taz shape (826, 25, 25)\n", - "INFO - get_skim_data taz SkimData shape (826, 25, 25)\n", - "INFO - SkimDict init taz\n", - "INFO - SkimDict.build_3d_skim_block_offset_table registered 167 3d keys\n", - "Time to execute step 'initialize_landuse': 2.22 s\n", - "Total time to execute iteration 1 with iteration value None: 2.22 s\n", - "INFO - #run_model running step initialize_households\n", - "Running step 'initialize_households'\n", - "WARNING - setting trace_hh_id is wrong type, should be an int, but was \n", - "INFO - Reading CSV file data/households.csv\n", - "INFO - full household list contains 5000 households\n", - "INFO - sampling 100 of 5000 households\n", - "INFO - loaded households (100, 7)\n", - "INFO - Reading CSV file data/persons.csv\n", - "INFO - loaded persons (167, 7)\n", - "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons\n", - "INFO - initialize_households.annotate_tables - annotating households SPEC annotate_households\n", - "INFO - initialize_households.annotate_tables - annotating persons SPEC annotate_persons_after_hh\n", - "Time to execute step 'initialize_households': 0.41 s\n", - "Total time to execute iteration 1 with iteration value None: 0.41 s\n", - "INFO - #run_model running step compute_accessibility\n", - "Running step 'compute_accessibility'\n", - "INFO - Running compute_accessibility with 25 orig zones 25 dest zones\n", - "INFO - compute_accessibility Running adaptive_chunked_choosers with 25 choosers\n", - "INFO - Running chunk 1 of 1 with 25 of 25 choosers\n", - "INFO - Running compute_accessibility with 25 orig zones 25 dest zones\n", - "INFO - compute_accessibility computed accessibilities (25, 10)\n", - "Time to execute step 'compute_accessibility': 0.08 s\n", - "Total time to execute iteration 1 with iteration value None: 0.08 s\n", - "INFO - #run_model running step school_location\n", - "Running step 'school_location'\n", - "INFO - Running school_location.i1.sample.university with 17 persons\n", - "INFO - school_location.i1.sample.university.interaction_sample Running adaptive_chunked_choosers with 17 choosers\n", - "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", - "INFO - Running eval_interaction_utilities on 102 rows\n", - "INFO - Running school_location.i1.logsums.university with 45 rows\n", - "INFO - school_location.i1.logsums.university.compute_logsums Running adaptive_chunked_choosers with 45 choosers\n", - "INFO - Running chunk 1 of 1 with 45 of 45 choosers\n", - "INFO - Running school_location.i1.simulate.university with 17 persons\n", - "INFO - school_location.i1.simulate.university.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 17 choosers and 45 alternatives\n", - "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", - "INFO - Running eval_interaction_utilities on 45 rows\n", - "INFO - Running school_location.i1.sample.highschool with 5 persons\n", - "INFO - school_location.i1.sample.highschool.interaction_sample Running adaptive_chunked_choosers with 5 choosers\n", - "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", - "INFO - Running eval_interaction_utilities on 10 rows\n", - "INFO - Running school_location.i1.logsums.highschool with 5 rows\n", - "INFO - school_location.i1.logsums.highschool.compute_logsums Running adaptive_chunked_choosers with 5 choosers\n", - "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", - "INFO - Running school_location.i1.simulate.highschool with 5 persons\n", - "INFO - school_location.i1.simulate.highschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 5 choosers and 5 alternatives\n", - "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", - "INFO - Running eval_interaction_utilities on 5 rows\n", - "INFO - Running school_location.i1.sample.gradeschool with 17 persons\n", - "INFO - school_location.i1.sample.gradeschool.interaction_sample Running adaptive_chunked_choosers with 17 choosers\n", - "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", - "INFO - Running eval_interaction_utilities on 425 rows\n", - "INFO - Running school_location.i1.logsums.gradeschool with 165 rows\n", - "INFO - school_location.i1.logsums.gradeschool.compute_logsums Running adaptive_chunked_choosers with 165 choosers\n", - "INFO - Running chunk 1 of 1 with 165 of 165 choosers\n", - "INFO - Running school_location.i1.simulate.gradeschool with 17 persons\n", - "INFO - school_location.i1.simulate.gradeschool.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 17 choosers and 165 alternatives\n", - "INFO - Running chunk 1 of 1 with 17 of 17 choosers\n", - "INFO - Running eval_interaction_utilities on 165 rows\n", - "INFO - write_trace_files iteration 1\n", - "INFO - school_location_logsum top 10 value counts:\n", - "10.852186 1\n", - "12.155904 1\n", - "20.491487 1\n", - "10.513007 1\n", - "20.774530 1\n", - "20.404239 1\n", - "20.284262 1\n", - "10.804509 1\n", - "10.180578 1\n", - "20.918083 1\n", - "Name: logsum, dtype: int64\n", - "Time to execute step 'school_location': 3.48 s\n", - "Total time to execute iteration 1 with iteration value None: 3.48 s\n", - "INFO - #run_model running step workplace_location\n", - "Running step 'workplace_location'\n", - "INFO - Running workplace_location.i1.sample.work_low with 37 persons\n", - "INFO - workplace_location.i1.sample.work_low.interaction_sample Running adaptive_chunked_choosers with 37 choosers\n", - "INFO - Running chunk 1 of 1 with 37 of 37 choosers\n", - "INFO - Running eval_interaction_utilities on 925 rows\n", - "INFO - Running workplace_location.i1.logsums.work_low with 511 rows\n", - "INFO - workplace_location.i1.logsums.work_low.compute_logsums Running adaptive_chunked_choosers with 511 choosers\n", - "INFO - Running chunk 1 of 1 with 511 of 511 choosers\n", - "INFO - Running workplace_location.i1.simulate.work_low with 37 persons\n", - "INFO - workplace_location.i1.simulate.work_low.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 37 choosers and 511 alternatives\n", - "INFO - Running chunk 1 of 1 with 37 of 37 choosers\n", - "INFO - Running eval_interaction_utilities on 511 rows\n", - "INFO - Running workplace_location.i1.sample.work_med with 26 persons\n", - "INFO - workplace_location.i1.sample.work_med.interaction_sample Running adaptive_chunked_choosers with 26 choosers\n", - "INFO - Running chunk 1 of 1 with 26 of 26 choosers\n", - "INFO - Running eval_interaction_utilities on 650 rows\n", - "INFO - Running workplace_location.i1.logsums.work_med with 356 rows\n", - "INFO - workplace_location.i1.logsums.work_med.compute_logsums Running adaptive_chunked_choosers with 356 choosers\n", - "INFO - Running chunk 1 of 1 with 356 of 356 choosers\n", - "INFO - Running workplace_location.i1.simulate.work_med with 26 persons\n", - "INFO - workplace_location.i1.simulate.work_med.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 26 choosers and 356 alternatives\n", - "INFO - Running chunk 1 of 1 with 26 of 26 choosers\n", - "INFO - Running eval_interaction_utilities on 356 rows\n", - "INFO - Running workplace_location.i1.sample.work_high with 16 persons\n", - "INFO - workplace_location.i1.sample.work_high.interaction_sample Running adaptive_chunked_choosers with 16 choosers\n", - "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", - "INFO - Running eval_interaction_utilities on 400 rows\n", - "INFO - Running workplace_location.i1.logsums.work_high with 224 rows\n", - "INFO - workplace_location.i1.logsums.work_high.compute_logsums Running adaptive_chunked_choosers with 224 choosers\n", - "INFO - Running chunk 1 of 1 with 224 of 224 choosers\n", - "INFO - Running workplace_location.i1.simulate.work_high with 16 persons\n", - "INFO - workplace_location.i1.simulate.work_high.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 16 choosers and 224 alternatives\n", - "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", - "INFO - Running eval_interaction_utilities on 224 rows\n", - "INFO - Running workplace_location.i1.sample.work_veryhigh with 18 persons\n", - "INFO - workplace_location.i1.sample.work_veryhigh.interaction_sample Running adaptive_chunked_choosers with 18 choosers\n", - "INFO - Running chunk 1 of 1 with 18 of 18 choosers\n", - "INFO - Running eval_interaction_utilities on 450 rows\n", - "INFO - Running workplace_location.i1.logsums.work_veryhigh with 259 rows\n", - "INFO - workplace_location.i1.logsums.work_veryhigh.compute_logsums Running adaptive_chunked_choosers with 259 choosers\n", - "INFO - Running chunk 1 of 1 with 259 of 259 choosers\n", - "INFO - Running workplace_location.i1.simulate.work_veryhigh with 18 persons\n", - "INFO - workplace_location.i1.simulate.work_veryhigh.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 18 choosers and 259 alternatives\n", - "INFO - Running chunk 1 of 1 with 18 of 18 choosers\n", - "INFO - Running eval_interaction_utilities on 259 rows\n", - "INFO - write_trace_files iteration 1\n", - "INFO - workplace_location_logsum top 10 value counts:\n", - "15.462302 1\n", - "14.407126 1\n", - "15.624373 1\n", - "15.654193 1\n", - "15.696199 1\n", - "15.701733 1\n", - "15.527734 1\n", - "15.547906 1\n", - "13.788314 1\n", - "13.903170 1\n", - "Name: logsum, dtype: int64\n", - "Time to execute step 'workplace_location': 5.28 s\n", - "Total time to execute iteration 1 with iteration value None: 5.28 s\n", - "INFO - #run_model running step auto_ownership_simulate\n", - "Running step 'auto_ownership_simulate'\n", - "INFO - Running auto_ownership_simulate with 100 households\n", - "INFO - auto_ownership_simulate.simple_simulate Running adaptive_chunked_choosers with 100 choosers\n", - "INFO - Running chunk 1 of 1 with 100 of 100 choosers\n", - "INFO - auto_ownership top 10 value counts:\n", - "0 67\n", - "1 32\n", - "2 1\n", - "Name: auto_ownership, dtype: int64\n", - "Time to execute step 'auto_ownership_simulate': 0.15 s\n", - "Total time to execute iteration 1 with iteration value None: 0.15 s\n", - "INFO - #run_model running step free_parking\n", - "Running step 'free_parking'\n", - "INFO - Running free_parking with 97 persons\n", - "INFO - free_parking.simple_simulate Running adaptive_chunked_choosers with 97 choosers\n", - "INFO - Running chunk 1 of 1 with 97 of 97 choosers\n", - "INFO - free_parking top 10 value counts:\n", - "False 163\n", - "True 4\n", - "Name: free_parking_at_work, dtype: int64\n", - "Time to execute step 'free_parking': 0.11 s\n", - "Total time to execute iteration 1 with iteration value None: 0.11 s\n", - "INFO - #run_model running step cdap_simulate\n", - "Running step 'cdap_simulate'\n", - "INFO - Pre-building cdap specs\n", - "INFO - Time to execute build_cdap_spec hh_size 2 : 0.115 seconds (0.0 minutes)\n", - "INFO - Time to execute build_cdap_spec hh_size 3 : 0.495 seconds (0.0 minutes)\n", - "INFO - Time to execute build_cdap_spec hh_size 4 : 1.316 seconds (0.0 minutes)\n", - "INFO - Time to execute build_cdap_spec hh_size 5 : 2.535 seconds (0.0 minutes)\n", - "INFO - Running cdap_simulate with 167 persons\n", - "INFO - cdap.cdap Running chunk 1 of 1 with 100 of 100 choosers\n", - "INFO - cdap_activity top 10 value counts:\n", - "M 89\n", - "N 50\n", - "H 28\n", - "Name: cdap_activity, dtype: int64\n", - "INFO - cdap crosstabs:\n", - "cdap_activity H M N All\n", - "ptype \n", - "1 3 43 6 52\n", - "2 8 19 7 34\n", - "3 1 11 5 17\n", - "4 4 0 17 21\n", - "5 8 0 13 21\n", - "6 4 1 0 5\n", - "7 0 9 2 11\n", - "8 0 6 0 6\n", - "All 28 89 50 167\n", - "Time to execute step 'cdap_simulate': 7.89 s\n", - "Total time to execute iteration 1 with iteration value None: 7.89 s\n", - "INFO - #run_model running step mandatory_tour_frequency\n", - "Running step 'mandatory_tour_frequency'\n", - "INFO - Running mandatory_tour_frequency with 89 persons\n", - "INFO - mandatory_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 89 choosers\n", - "INFO - Running chunk 1 of 1 with 89 of 89 choosers\n", - "INFO - mandatory_tour_frequency top 10 value counts:\n", - " 78\n", - "work1 68\n", - "school1 21\n", - "Name: mandatory_tour_frequency, dtype: int64\n", - "Time to execute step 'mandatory_tour_frequency': 0.68 s\n", - "Total time to execute iteration 1 with iteration value None: 0.68 s\n", - "INFO - #run_model running step mandatory_tour_scheduling\n", - "Running step 'mandatory_tour_scheduling'\n", - "DEBUG - @inject timetable\n", - "INFO - Running mandatory_tour_scheduling with 89 tours\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 68 tour choices\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work Running adaptive_chunked_choosers with 68 choosers\n", - "INFO - Running chunk 1 of 1 with 68 of 68 choosers\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work schedule_tours running 68 tour choices\n", - "INFO - tdd_alt_segments specified for representative logsums\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 12920 to 1020 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums compute_logsums for 1020 choosers 1020 alts\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.compute_logsums.logsums Running adaptive_chunked_choosers with 1020 choosers\n", - "INFO - Running chunk 1 of 1 with 1020 of 1020 choosers\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.work.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 68 choosers and 12920 alternatives\n", - "INFO - Running chunk 1 of 1 with 68 of 68 choosers\n", - "INFO - Running eval_interaction_utilities on 12920 rows\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 16 tour choices\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school Running adaptive_chunked_choosers with 16 choosers\n", - "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school schedule_tours running 16 tour choices\n", - "INFO - tdd_alt_segments specified for representative logsums\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 3040 to 240 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums compute_logsums for 240 choosers 240 alts\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.compute_logsums.logsums Running adaptive_chunked_choosers with 240 choosers\n", - "INFO - Running chunk 1 of 1 with 240 of 240 choosers\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.school.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 16 choosers and 3040 alternatives\n", - "INFO - Running chunk 1 of 1 with 16 of 16 choosers\n", - "INFO - Running eval_interaction_utilities on 3040 rows\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 5 tour choices\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ Running adaptive_chunked_choosers with 5 choosers\n", - "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ schedule_tours running 5 tour choices\n", - "INFO - tdd_alt_segments specified for representative logsums\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums compute_logsums deduped_alt_tdds reduced number of rows by 92.11% from 950 to 75 compared to USE_BRUTE_FORCE_TO_COMPUTE_LOGSUMS\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums compute_logsums for 75 choosers 75 alts\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.compute_logsums.logsums Running adaptive_chunked_choosers with 75 choosers\n", - "INFO - Running chunk 1 of 1 with 75 of 75 choosers\n", - "INFO - mandatory_tour_scheduling.vectorize_tour_scheduling.tour_1.univ.interaction_sample_simulate Running adaptive_chunked_choosers_and_alts with 5 choosers and 950 alternatives\n", - "INFO - Running chunk 1 of 1 with 5 of 5 choosers\n", - "INFO - Running eval_interaction_utilities on 950 rows\n", - "Time to execute step 'mandatory_tour_scheduling': 3.83 s\n", - "Total time to execute iteration 1 with iteration value None: 3.83 s\n", - "INFO - #run_model running step joint_tour_frequency\n", - "Running step 'joint_tour_frequency'\n", - "INFO - Running joint_tour_frequency with 36 multi-person households\n", - "DEBUG - @inject timetable\n", - "INFO - joint_tour_frequency.simple_simulate Running adaptive_chunked_choosers with 36 choosers\n", - "INFO - Running chunk 1 of 1 with 36 of 36 choosers\n", - "INFO - joint_tour_frequency top 10 value counts:\n", - "0_tours 96\n", - "1_Disc 2\n", - "1_Shop 1\n", - "1_Eat 1\n", - "Name: joint_tour_frequency, dtype: int64\n", - "Time to execute step 'joint_tour_frequency': 0.38 s\n", - "Total time to execute iteration 1 with iteration value None: 0.38 s\n", - "INFO - #run_model running step joint_tour_composition\n", - "Running step 'joint_tour_composition'\n", - "INFO - Running joint_tour_composition with 4 joint tours\n", - "DEBUG - @inject timetable\n", - "INFO - joint_tour_composition.simple_simulate Running adaptive_chunked_choosers with 4 choosers\n", - "INFO - Running chunk 1 of 1 with 4 of 4 choosers\n", - "INFO - joint_tour_composition top 10 value counts:\n", - "adults 2\n", - "children 1\n", - "mixed 1\n", - "Name: composition, dtype: int64\n", - "Time to execute step 'joint_tour_composition': 0.24 s\n", - "Total time to execute iteration 1 with iteration value None: 0.24 s\n", - "INFO - #run_model running step joint_tour_participation\n", - "Running step 'joint_tour_participation'\n", - "INFO - Running joint_tours_participation with 8 potential participants (candidates)\n", - "DEBUG - @inject timetable\n", - "INFO - joint_tour_participation Running chunk 1 of 1 with 4 of 4 choosers\n", - "INFO - joint_tour_participation.eval_mnl.participants_chooser 4 joint tours to satisfy.\n", - "INFO - joint_tour_participation.eval_mnl.participants_chooser 1 iterations to satisfy all joint tours.\n", - "Time to execute step 'joint_tour_participation': 0.46 s\n", - "Total time to execute iteration 1 with iteration value None: 0.46 s\n", - "INFO - #run_model running step joint_tour_destination\n", - "Running step 'joint_tour_destination'\n", - "INFO - running joint_tour_destination.shopping.sample with 1 tours\n", - "INFO - joint_tour_destination.shopping.sample.interaction_sample Running adaptive_chunked_choosers with 1 choosers\n", - "INFO - Running chunk 1 of 1 with 1 of 1 choosers\n", - "INFO - Running eval_interaction_utilities on 25 rows\n", - "INFO - Running joint_tour_destination.shopping.logsums with 11 rows\n", - "INFO - joint_tour_destination.shopping.logsums.compute_logsums Running adaptive_chunked_choosers with 11 choosers\n", - "INFO - Running chunk 1 of 1 with 11 of 11 choosers\n", - "INFO - Time to execute all models until this error : 29.773 seconds (0.5 minutes)\n", + "INFO - Time to execute all models until this error : 0.393 seconds (0.0 minutes)\n", "ERROR - activitysim run encountered an unrecoverable error\n", "Traceback (most recent call last):\n", " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/cli/run.py\", line 261, in run\n", @@ -513,29 +214,32 @@ " step()\n", " File \"/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\", line 973, in __call__\n", " return self._func(**kwargs)\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/joint_tour_destination.py\", line 68, in joint_tour_destination\n", - " choices_df, save_sample_df = tour_destination.run_tour_destination(\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/util/tour_destination.py\", line 727, in run_tour_destination\n", - " run_destination_simulate(\n", - "TypeError: run_destination_simulate() got an unexpected keyword argument 'choose_individual_max_utility'\n" + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/initialize.py\", line 100, in initialize_landuse\n", + " with chunk.chunk_log(trace_label, base=True):\n", + " File \"/home/jan/.pyenv/versions/3.8.2/lib/python3.8/contextlib.py\", line 113, in __enter__\n", + " return next(self.gen)\n", + " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/chunk.py\", line 929, in chunk_log\n", + " assert base == (len(CHUNK_SIZERS) == 0)\n", + "AssertionError\n" ] }, { - "ename": "TypeError", - "evalue": "run_destination_simulate() got an unexpected keyword argument 'choose_individual_max_utility'", + "ename": "AssertionError", + "evalue": "", "output_type": "error", "traceback": [ "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + "\u001B[0;31mAssertionError\u001B[0m Traceback (most recent call last)", "\u001B[0;32m/tmp/ipykernel_2932/4273387002.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 2\u001B[0m \u001B[0mrun\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0madd_run_args\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mparser\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[0margs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mparser\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mparse_args\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m'-c'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'configs'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'-o'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'output'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'-d'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'data'\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m \u001B[0mrun\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;31m# 2mins full example run\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 5\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0minject\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mis_injectable\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'preload_injectables'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 6\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mactivitysim\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mabm\u001B[0m \u001B[0;31m# register abm steps and other abm-specific injectables\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/cli/run.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(args)\u001B[0m\n\u001B[1;32m 259\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'run single process simulation'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 260\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 261\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodels\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetting\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'models'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mresume_after\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 262\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 263\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetting\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'cleanup_pipeline_after_run'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(models, resume_after)\u001B[0m\n\u001B[1;32m 615\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mmodel\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mmodels\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 616\u001B[0m \u001B[0mt1\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mprint_elapsed_time\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 617\u001B[0;31m \u001B[0mrun_model\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodel\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 618\u001B[0m \u001B[0mmem\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrace_memory_info\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"pipeline.run after {model}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 619\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mrun_model\u001B[0;34m(model_name)\u001B[0m\n\u001B[1;32m 474\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"#run_model running step {step_name}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 475\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 476\u001B[0;31m \u001B[0morca\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mstep_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 477\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 478\u001B[0m \u001B[0mt0\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mprint_elapsed_time\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"#run_model completed step '%s'\"\u001B[0m \u001B[0;34m%\u001B[0m \u001B[0mmodel_name\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mt0\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdebug\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(steps, iter_vars, data_out, out_interval, out_base_tables, out_run_tables, compress, out_base_local, out_run_local)\u001B[0m\n\u001B[1;32m 2166\u001B[0m \u001B[0mstep\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mget_step\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mstep_name\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 2167\u001B[0m \u001B[0mt2\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 2168\u001B[0;31m \u001B[0mstep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 2169\u001B[0m print(\"Time to execute step '{}': {:.2f} s\".format(\n\u001B[1;32m 2170\u001B[0m step_name, time.time() - t2))\n", "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 971\u001B[0m kwargs = _collect_variables(names=self._argspec.args,\n\u001B[1;32m 972\u001B[0m expressions=self._argspec.defaults)\n\u001B[0;32m--> 973\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_func\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 974\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 975\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0m_tables_used\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/joint_tour_destination.py\u001B[0m in \u001B[0;36mjoint_tour_destination\u001B[0;34m(tours, persons_merged, households_merged, network_los, chunk_size, trace_hh_id)\u001B[0m\n\u001B[1;32m 66\u001B[0m \u001B[0mestimator\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mwrite_model_settings\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodel_settings\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmodel_settings_file_name\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 67\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 68\u001B[0;31m choices_df, save_sample_df = tour_destination.run_tour_destination(\n\u001B[0m\u001B[1;32m 69\u001B[0m \u001B[0mtours\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 70\u001B[0m \u001B[0mpersons_merged\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/util/tour_destination.py\u001B[0m in \u001B[0;36mrun_tour_destination\u001B[0;34m(tours, persons_merged, want_logsums, want_sample_table, model_settings, network_los, estimator, chunk_size, trace_hh_id, trace_label)\u001B[0m\n\u001B[1;32m 725\u001B[0m \u001B[0mspec_segment_name\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0msegment_name\u001B[0m \u001B[0;31m# spec_segment_name is segment_name\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 726\u001B[0m \u001B[0mchoices\u001B[0m \u001B[0;34m=\u001B[0m\u001B[0;31m \u001B[0m\u001B[0;31m\\\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 727\u001B[0;31m run_destination_simulate(\n\u001B[0m\u001B[1;32m 728\u001B[0m \u001B[0mspec_segment_name\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 729\u001B[0m \u001B[0mchoosers\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mTypeError\u001B[0m: run_destination_simulate() got an unexpected keyword argument 'choose_individual_max_utility'" + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/initialize.py\u001B[0m in \u001B[0;36minitialize_landuse\u001B[0;34m()\u001B[0m\n\u001B[1;32m 98\u001B[0m \u001B[0mtrace_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m'initialize_landuse'\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 99\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 100\u001B[0;31m \u001B[0;32mwith\u001B[0m \u001B[0mchunk\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchunk_log\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrace_label\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbase\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 101\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 102\u001B[0m \u001B[0mmodel_settings\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mread_model_settings\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'initialize_landuse.yaml'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmandatory\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/.pyenv/versions/3.8.2/lib/python3.8/contextlib.py\u001B[0m in \u001B[0;36m__enter__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 111\u001B[0m \u001B[0;32mdel\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkwds\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfunc\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 112\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 113\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mnext\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgen\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 114\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mStopIteration\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 115\u001B[0m \u001B[0;32mraise\u001B[0m \u001B[0mRuntimeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"generator didn't yield\"\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/chunk.py\u001B[0m in \u001B[0;36mchunk_log\u001B[0;34m(trace_label, chunk_tag, base)\u001B[0m\n\u001B[1;32m 927\u001B[0m \u001B[0;31m# avoids breaking the assertion below.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 928\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 929\u001B[0;31m \u001B[0;32massert\u001B[0m \u001B[0mbase\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mCHUNK_SIZERS\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 930\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 931\u001B[0m \u001B[0mtrace_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34mf\"{trace_label}.chunk_log\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mAssertionError\u001B[0m: " ] } ], From 9685aca3ef75019d4ee75acdbf3383bc4b0e9397 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 15:06:50 +1000 Subject: [PATCH 051/135] make place of call consistent --- activitysim/abm/models/trip_mode_choice.py | 5 ++++- activitysim/abm/models/util/mode.py | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/activitysim/abm/models/trip_mode_choice.py b/activitysim/abm/models/trip_mode_choice.py index 59e9fc5e5e..9c90e9d8ff 100644 --- a/activitysim/abm/models/trip_mode_choice.py +++ b/activitysim/abm/models/trip_mode_choice.py @@ -194,7 +194,8 @@ def trip_mode_choice( logsum_column_name=logsum_column_name, trace_label=segment_trace_label, trace_choice_name='trip_mode_choice', - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) if trace_hh_id: # trace the coefficients @@ -244,6 +245,8 @@ def trip_mode_choice( tracing.print_summary('trip_mode_choice choices', trips_df[mode_column_name], value_counts=True) + print(mode_column_name) + assert not trips_df[mode_column_name].isnull().any() pipeline.replace_table("trips", trips_df) diff --git a/activitysim/abm/models/util/mode.py b/activitysim/abm/models/util/mode.py index c13b9cc66b..ec3f39dd8f 100644 --- a/activitysim/abm/models/util/mode.py +++ b/activitysim/abm/models/util/mode.py @@ -25,7 +25,8 @@ def mode_choice_simulate( trace_label, trace_choice_name, trace_column_names=None, - estimator=None): + estimator=None, + choose_individual_max_utility=False): """ common method for both tour_mode_choice and trip_mode_choice @@ -61,7 +62,7 @@ def mode_choice_simulate( trace_choice_name=trace_choice_name, estimator=estimator, trace_column_names=trace_column_names, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + choose_individual_max_utility=choose_individual_max_utility) # for consistency, always return dataframe, whether or not logsums were requested if isinstance(choices, pd.Series): @@ -145,6 +146,8 @@ def run_tour_mode_choice_simulate( trace_label=trace_label, trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, - estimator=estimator) + estimator=estimator, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) return choices From 19d1222b07f962680ce5838b0350a4744d9516c9 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 15:22:54 +1000 Subject: [PATCH 052/135] remove debug print statement --- activitysim/abm/models/trip_mode_choice.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/activitysim/abm/models/trip_mode_choice.py b/activitysim/abm/models/trip_mode_choice.py index 9c90e9d8ff..623c4b34df 100644 --- a/activitysim/abm/models/trip_mode_choice.py +++ b/activitysim/abm/models/trip_mode_choice.py @@ -245,8 +245,6 @@ def trip_mode_choice( tracing.print_summary('trip_mode_choice choices', trips_df[mode_column_name], value_counts=True) - print(mode_column_name) - assert not trips_df[mode_column_name].isnull().any() pipeline.replace_table("trips", trips_df) From 61a314f84ffc66600f7f50ecbafe1d6b1a5630ca Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 15:30:07 +1000 Subject: [PATCH 053/135] move reporting in make choices --- activitysim/core/logit.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 516add8aa9..b8a2eca3f1 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -297,12 +297,6 @@ def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_ """ trace_label = tracing.extend_trace_label(trace_label, 'make_choices') - if choose_individual_max_utility: - choices = make_choices_ru_frozen(utilities, nest_spec, trace_label) - # TODO: rands - rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) - return choices, rands - # probs should sum to 1 across each row BAD_PROB_THRESHOLD = 0.001 bad_probs = \ @@ -316,6 +310,12 @@ def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_ msg="probabilities do not add up to 1", trace_choosers=trace_choosers) + if choose_individual_max_utility: + choices = make_choices_ru_frozen(utilities, nest_spec, trace_label) + # TODO: rands - log all zeros for now + rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) + return choices, rands + rands = pipeline.get_rn_generator().random_for_df(probs) probs_arr = probs.values.cumsum(axis=1) - rands From 8c167658bfc7cc9650271c5b1bb86a5a6f83c2d9 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 15:56:29 +1000 Subject: [PATCH 054/135] make choice indexing for nested logit consistent with mnl and cumsum MC --- activitysim/core/logit.py | 18 ++++++++++++------ notebooks/fru_utils.py | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index b8a2eca3f1..ad24b464be 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -228,7 +228,7 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter # alternatives and set the corresponding entry to 1 for each row, set all other alternatives at this level to zero. # Once the tree is walked (all alternatives have been processed), take the product of the alternatives in each # leaf's alternative list. Then pick the only alternative with entry 1, all others must be 0. -def make_choices_ru_frozen_nl(nested_utilities, nest_spec): +def make_choices_ru_frozen_nl(nested_utilities, alt_order_array, nest_spec): """ walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" nest_utils_for_choice = add_ev1_random(nested_utilities) @@ -243,8 +243,11 @@ def make_choices_ru_frozen_nl(nested_utilities, nest_spec): assert not choices.isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=nest_utils_for_choice.index) - # TODO [janzill Jun2022]: REMOVE HACK, make this numpy and positional indexes from the beginning - choices = choices.map({v: k for k,v in enumerate(nest_utils_for_choice.columns)}) + # In order for choice indexing to be consistent with MNL and cumsum MC choices, we need to index in the order + # alternatives were originally created before adding nest nodes that are not elemental alternatives + choices = choices.map({v: k for k, v in enumerate(alt_order_array)}) + ## the following is wrong, the order might be changed + #choices = choices.map({v: k for k, v in enumerate(nest_utils_for_choice.columns)}) return choices @@ -259,12 +262,13 @@ def make_choices_ru_frozen_mnl(utilities): choices = pd.Series(choices, index=utilities_incl_unobs.index) return choices -def make_choices_ru_frozen(utilities, nest_spec=None, trace_label=None): +def make_choices_ru_frozen(utilities, alt_order_array, nest_spec=None, trace_label=None): trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen_mnl') if nest_spec is None: + assert (alt_order_array == utilities.columns.values).all() choices = make_choices_ru_frozen_mnl(utilities) else: - choices = make_choices_ru_frozen_nl(utilities, nest_spec) + choices = make_choices_ru_frozen_nl(utilities, alt_order_array, nest_spec) return choices @@ -311,7 +315,9 @@ def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_ trace_choosers=trace_choosers) if choose_individual_max_utility: - choices = make_choices_ru_frozen(utilities, nest_spec, trace_label) + # TODO: index of choices for nested utilities is different than unnested - this needs to be consistent for + # turning indexes into alternative names; use probs.columns for now. + choices = make_choices_ru_frozen(utilities, probs.columns.values, nest_spec, trace_label) # TODO: rands - log all zeros for now rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) return choices, rands diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py index d12b1cc7ee..77a4e920c2 100644 --- a/notebooks/fru_utils.py +++ b/notebooks/fru_utils.py @@ -554,7 +554,6 @@ def get_stuff(do_these_purposes=None): nested_probs_list.append(nested_probs) base_probs_list.append(base_probs) - choices_df_asim = pd.concat(choices_list) # update trips table with choices (and potionally logssums) @@ -579,4 +578,5 @@ def get_stuff(do_these_purposes=None): print("Done") - return trips_df, raw_util_list, nest_list, nu_list, nest_spec_list, nested_probs_list, base_probs_list \ No newline at end of file + return trips_df, raw_util_list, nest_list, nu_list, nest_spec_list, nested_probs_list, base_probs_list, \ + dict(list(zip(list(range(len(alts))), alts))) \ No newline at end of file From 087ec0ebfcd25c4dc4425b87161993dcb2deb638 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 16:20:34 +1000 Subject: [PATCH 055/135] use fru in models that use logit.make_choices w/o simulate wrapper --- activitysim/abm/models/joint_tour_participation.py | 4 +++- activitysim/abm/models/non_mandatory_tour_frequency.py | 4 +++- activitysim/abm/models/trip_departure_choice.py | 3 ++- activitysim/abm/models/trip_purpose.py | 4 +++- activitysim/abm/models/util/cdap.py | 10 ++++++++-- .../abm/models/util/probabilistic_scheduling.py | 5 ++++- activitysim/core/pathbuilder.py | 5 ++++- 7 files changed, 27 insertions(+), 8 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 23d13ac9e5..898d87bc48 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -175,7 +175,9 @@ def participants_chooser(probs, choosers, spec, trace_label): print(unsatisfied_candidates.head(20)) assert False - choices, rands = logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers) + choices, rands = logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers, + choose_individual_max_utility=config.setting( + "freeze_unobserved_utilities", False)) participate = (choices == PARTICIPATE_CHOICE) # satisfaction indexed by tour_id diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index 0cf58e4618..fb45c13513 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -111,7 +111,9 @@ def extend_tour_counts(persons, tour_counts, alternatives, trace_hh_id, trace_la choices, rands = logit.make_choices( choosers[PROBABILITY_COLUMNS], trace_label=tour_type_trace_label, - trace_choosers=choosers) + trace_choosers=choosers, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) # - extend tour_count (0-based prob alternative choice equals magnitude of extension) if choices.any(): diff --git a/activitysim/abm/models/trip_departure_choice.py b/activitysim/abm/models/trip_departure_choice.py index 80eb9b912a..cfff604ffe 100644 --- a/activitysim/abm/models/trip_departure_choice.py +++ b/activitysim/abm/models/trip_departure_choice.py @@ -286,7 +286,8 @@ def choose_tour_leg_pattern(trip_segment, # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample positions, rands = \ - logit.make_choices(probs, trace_label=trace_label, trace_choosers=trip_segment) + logit.make_choices(probs, trace_label=trace_label, trace_choosers=trip_segment, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) diff --git a/activitysim/abm/models/trip_purpose.py b/activitysim/abm/models/trip_purpose.py index f0787ccf55..7838ac0149 100644 --- a/activitysim/abm/models/trip_purpose.py +++ b/activitysim/abm/models/trip_purpose.py @@ -108,7 +108,9 @@ def choose_intermediate_trip_purpose( choices, rands = logit.make_choices( choosers[purpose_cols], - trace_label=trace_label, trace_choosers=choosers) + trace_label=trace_label, trace_choosers=choosers, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, columns=[None, 'trip_purpose']) diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index fdc34961ae..1146f9732c 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -640,7 +640,10 @@ def household_activity_choices(indiv_utils, interaction_coefficients, hhsize, # select an activity pattern alternative for each household based on probability # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs - idx_choices, rands = logit.make_choices(probs, trace_label=trace_label) + idx_choices, rands = logit.make_choices(probs, trace_label=trace_label, + choose_individual_max_utility=config.setting( + "freeze_unobserved_utilities", False) + ) # convert choice expressed as index into alternative name from util column label choices = pd.Series(utils.columns[idx_choices].values, index=utils.index) @@ -762,7 +765,10 @@ def extra_hh_member_choices(persons, cdap_fixed_relative_proportions, locals_d, # select an activity pattern alternative for each person based on probability # idx_choices is a series (indexed on _persons_index_ ) with the chosen alternative represented # as the integer (0 based) index of the chosen column from probs - idx_choices, rands = logit.make_choices(probs, trace_label=trace_label) + idx_choices, rands = logit.make_choices(probs, trace_label=trace_label, + choose_individual_max_utility=config.setting( + "freeze_unobserved_utilities", False) + ) # convert choice from column index to activity name choices = pd.Series(probs.columns[idx_choices].values, index=probs.index) diff --git a/activitysim/abm/models/util/probabilistic_scheduling.py b/activitysim/abm/models/util/probabilistic_scheduling.py index 73500f58e8..94675bccd3 100644 --- a/activitysim/abm/models/util/probabilistic_scheduling.py +++ b/activitysim/abm/models/util/probabilistic_scheduling.py @@ -248,7 +248,10 @@ def make_scheduling_choices( if trace_hh_id and tracing.has_trace_targets(choosers_df): tracing.trace_df(chooser_probs, '%s.chooser_probs' % trace_label) - raw_choices, rands = logit.make_choices(chooser_probs, trace_label=trace_label, trace_choosers=choosers) + raw_choices, rands = logit.make_choices(chooser_probs, trace_label=trace_label, trace_choosers=choosers, + choose_individual_max_utility=config.setting( + "freeze_unobserved_utilities", False) + ) chunk.log_df(trace_label, "choices", raw_choices) chunk.log_df(trace_label, "rands", rands) diff --git a/activitysim/core/pathbuilder.py b/activitysim/core/pathbuilder.py index d4cee77504..2cc4862d42 100644 --- a/activitysim/core/pathbuilder.py +++ b/activitysim/core/pathbuilder.py @@ -744,7 +744,10 @@ def build_virtual_path(self, recipe, path_type, orig, dest, tod, demographic_seg self.trace_df(probs, trace_label, 'probs') else: - choices, rands = logit.make_choices(probs, allow_bad_probs=True, trace_label=trace_label) + choices, rands = logit.make_choices(probs, allow_bad_probs=True, trace_label=trace_label, + choose_individual_max_utility=config.setting( + "freeze_unobserved_utilities", False) + ) chunk.log_df(trace_label, "rands", rands) del rands From c26c80c08985b547bd8adb8c89473c7218398470 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 19 Jun 2022 16:58:25 +1000 Subject: [PATCH 056/135] no fru for probabilitic lookup tables --- activitysim/abm/models/joint_tour_participation.py | 9 +++++---- .../abm/models/non_mandatory_tour_frequency.py | 5 +++-- activitysim/abm/models/trip_departure_choice.py | 8 ++++---- activitysim/abm/models/trip_purpose.py | 5 +++-- activitysim/abm/models/util/cdap.py | 13 +++++-------- .../abm/models/util/probabilistic_scheduling.py | 5 +---- activitysim/core/pathbuilder.py | 8 ++++---- activitysim/core/simulate.py | 4 +++- 8 files changed, 28 insertions(+), 29 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 898d87bc48..24f7295042 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -108,7 +108,7 @@ def get_tour_satisfaction(candidates, participate): return satisfaction -def participants_chooser(probs, choosers, spec, trace_label): +def participants_chooser(probs, choosers, spec, trace_label, utilities): """ custom alternative to logit.make_choices for simulate.simple_simulate @@ -175,9 +175,10 @@ def participants_chooser(probs, choosers, spec, trace_label): print(unsatisfied_candidates.head(20)) assert False - choices, rands = logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers, - choose_individual_max_utility=config.setting( - "freeze_unobserved_utilities", False)) + choices, rands = logit.make_choices( + probs, utilities=utilities, trace_label=trace_label, trace_choosers=choosers, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) participate = (choices == PARTICIPATE_CHOICE) # satisfaction indexed by tour_id diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index fb45c13513..5816d3349d 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -111,8 +111,9 @@ def extend_tour_counts(persons, tour_counts, alternatives, trace_hh_id, trace_la choices, rands = logit.make_choices( choosers[PROBABILITY_COLUMNS], trace_label=tour_type_trace_label, - trace_choosers=choosers, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + trace_choosers=choosers + # [janzill Jun2022]: don't change for probs tables + #choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) ) # - extend tour_count (0-based prob alternative choice equals magnitude of extension) diff --git a/activitysim/abm/models/trip_departure_choice.py b/activitysim/abm/models/trip_departure_choice.py index cfff604ffe..37b3b48223 100644 --- a/activitysim/abm/models/trip_departure_choice.py +++ b/activitysim/abm/models/trip_departure_choice.py @@ -275,9 +275,6 @@ def choose_tour_leg_pattern(trip_segment, chunk.log_df(trace_label, 'probs', probs) - del utilities_df - chunk.log_df(trace_label, 'utilities_df', None) - if have_trace_targets: tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), column_labels=['alternative', 'probability']) @@ -286,12 +283,15 @@ def choose_tour_leg_pattern(trip_segment, # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample positions, rands = \ - logit.make_choices(probs, trace_label=trace_label, trace_choosers=trip_segment, + logit.make_choices(probs, utilities=utilities_df, trace_label=trace_label, trace_choosers=trip_segment, choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) + del utilities_df + chunk.log_df(trace_label, 'utilities_df', None) + del probs chunk.log_df(trace_label, 'probs', None) diff --git a/activitysim/abm/models/trip_purpose.py b/activitysim/abm/models/trip_purpose.py index 7838ac0149..1058e0685f 100644 --- a/activitysim/abm/models/trip_purpose.py +++ b/activitysim/abm/models/trip_purpose.py @@ -108,8 +108,9 @@ def choose_intermediate_trip_purpose( choices, rands = logit.make_choices( choosers[purpose_cols], - trace_label=trace_label, trace_choosers=choosers, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + trace_label=trace_label, trace_choosers=choosers + # [janzill Jun2022]: don't change for probs tables + #choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) ) if have_trace_targets: diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index 1146f9732c..3af60c183d 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -640,10 +640,10 @@ def household_activity_choices(indiv_utils, interaction_coefficients, hhsize, # select an activity pattern alternative for each household based on probability # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs - idx_choices, rands = logit.make_choices(probs, trace_label=trace_label, - choose_individual_max_utility=config.setting( - "freeze_unobserved_utilities", False) - ) + idx_choices, rands = logit.make_choices( + probs, utilities=utils, trace_label=trace_label, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) # convert choice expressed as index into alternative name from util column label choices = pd.Series(utils.columns[idx_choices].values, index=utils.index) @@ -765,10 +765,7 @@ def extra_hh_member_choices(persons, cdap_fixed_relative_proportions, locals_d, # select an activity pattern alternative for each person based on probability # idx_choices is a series (indexed on _persons_index_ ) with the chosen alternative represented # as the integer (0 based) index of the chosen column from probs - idx_choices, rands = logit.make_choices(probs, trace_label=trace_label, - choose_individual_max_utility=config.setting( - "freeze_unobserved_utilities", False) - ) + idx_choices, rands = logit.make_choices(probs, trace_label=trace_label) # no fru for prob tables # convert choice from column index to activity name choices = pd.Series(probs.columns[idx_choices].values, index=probs.index) diff --git a/activitysim/abm/models/util/probabilistic_scheduling.py b/activitysim/abm/models/util/probabilistic_scheduling.py index 94675bccd3..cc5869ba57 100644 --- a/activitysim/abm/models/util/probabilistic_scheduling.py +++ b/activitysim/abm/models/util/probabilistic_scheduling.py @@ -248,10 +248,7 @@ def make_scheduling_choices( if trace_hh_id and tracing.has_trace_targets(choosers_df): tracing.trace_df(chooser_probs, '%s.chooser_probs' % trace_label) - raw_choices, rands = logit.make_choices(chooser_probs, trace_label=trace_label, trace_choosers=choosers, - choose_individual_max_utility=config.setting( - "freeze_unobserved_utilities", False) - ) + raw_choices, rands = logit.make_choices(chooser_probs, trace_label=trace_label, trace_choosers=choosers) # no fru chunk.log_df(trace_label, "choices", raw_choices) chunk.log_df(trace_label, "rands", rands) diff --git a/activitysim/core/pathbuilder.py b/activitysim/core/pathbuilder.py index 2cc4862d42..28d0707575 100644 --- a/activitysim/core/pathbuilder.py +++ b/activitysim/core/pathbuilder.py @@ -744,10 +744,10 @@ def build_virtual_path(self, recipe, path_type, orig, dest, tod, demographic_seg self.trace_df(probs, trace_label, 'probs') else: - choices, rands = logit.make_choices(probs, allow_bad_probs=True, trace_label=trace_label, - choose_individual_max_utility=config.setting( - "freeze_unobserved_utilities", False) - ) + choices, rands = logit.make_choices( + probs, utilities=utilities_df, allow_bad_probs=True, trace_label=trace_label, + choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + ) chunk.log_df(trace_label, "rands", rands) del rands diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 0162d20616..039182decd 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -965,8 +965,10 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, column_labels=['alternative', 'probability']) if custom_chooser: + # TODO [janzill Jun2022]: is this used somewhere outside Asim such that adding utilities would be a breaking + # change? choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, - trace_label=trace_label) + trace_label=trace_label, utilities=utilities) else: choices, rands = logit.make_choices(probs, utilities=utilities, trace_label=trace_label, choose_individual_max_utility=choose_individual_max_utility) From d14fac389e298e4334fdecd41a14b7642856fc01 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 14:34:40 +1000 Subject: [PATCH 057/135] memory saving work - delete dfs wherever possible --- .../abm/models/joint_tour_participation.py | 12 +- activitysim/core/logit.py | 38 +-- activitysim/core/simulate.py | 220 ++++++++++-------- 3 files changed, 146 insertions(+), 124 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 24f7295042..af3a32e543 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -108,7 +108,7 @@ def get_tour_satisfaction(candidates, participate): return satisfaction -def participants_chooser(probs, choosers, spec, trace_label, utilities): +def participants_chooser(probs, choosers, spec, trace_label): """ custom alternative to logit.make_choices for simulate.simple_simulate @@ -175,10 +175,7 @@ def participants_chooser(probs, choosers, spec, trace_label, utilities): print(unsatisfied_candidates.head(20)) assert False - choices, rands = logit.make_choices( - probs, utilities=utilities, trace_label=trace_label, trace_choosers=choosers, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) - ) + choices, rands = logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers) participate = (choices == PARTICIPATE_CHOICE) # satisfaction indexed by tour_id @@ -317,8 +314,7 @@ def joint_tour_participation( trace_label=trace_label, trace_choice_name='participation', custom_chooser=participants_chooser, - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) # choice is boolean (participate or not) choice_col = model_settings.get('participation_choice', 'participate') @@ -383,4 +379,4 @@ def joint_tour_participation( label="joint_tour_participation.participants") tracing.trace_df(joint_tours, - label="joint_tour_participation.joint_tours") + label="joint_tour_participation.joint_tours") \ No newline at end of file diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index ad24b464be..1fe6065fd4 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -272,36 +272,52 @@ def make_choices_ru_frozen(utilities, alt_order_array, nest_spec=None, trace_lab return choices -def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_choosers=None, allow_bad_probs=False, - choose_individual_max_utility=False): +# TODO: memory usage +def make_choices_utility_based( + utilities, + # for nested: need mapping of index to alternative name to "fake" indexes if I want to keep with current + # structure, OR need to make returning names optional. sharrow impl will make our life so much easier + name_mapping=None, + nest_spec=None, + trace_label=None, + trace_choosers=None, + allow_bad_probs=False, +): + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_utility_based') + + # TODO: index of choices for nested utilities is different than unnested - this needs to be consistent for + # turning indexes into alternative names to keep code changes to minimum for now + choices = make_choices_ru_frozen(utilities, name_mapping, nest_spec, trace_label) + # TODO: rands - log all zeros for now + rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) + return choices, rands + + +def make_choices(probs, trace_label=None, trace_choosers=None, allow_bad_probs=False): """ Make choices for each chooser from among a set of alternatives. - Parameters ---------- probs : pandas.DataFrame Rows for choosers and columns for the alternatives from which they are choosing. Values are expected to be valid probabilities across each row, e.g. they should sum to 1. - trace_choosers : pandas.dataframe the choosers df (for interaction_simulate) to facilitate the reporting of hh_id by report_bad_choices because it can't deduce hh_id from the interaction_dataset which is indexed on index values from alternatives df - Returns ------- choices : pandas.Series Maps chooser IDs (from `probs` index) to a choice, where the choice is an index into the columns of `probs`. - rands : pandas.Series The random numbers used to make the choices (for debugging, tracing) - """ trace_label = tracing.extend_trace_label(trace_label, 'make_choices') # probs should sum to 1 across each row + BAD_PROB_THRESHOLD = 0.001 bad_probs = \ probs.sum(axis=1).sub(np.ones(len(probs.index))).abs() \ @@ -314,14 +330,6 @@ def make_choices(probs, utilities=None, nest_spec=None, trace_label=None, trace_ msg="probabilities do not add up to 1", trace_choosers=trace_choosers) - if choose_individual_max_utility: - # TODO: index of choices for nested utilities is different than unnested - this needs to be consistent for - # turning indexes into alternative names; use probs.columns for now. - choices = make_choices_ru_frozen(utilities, probs.columns.values, nest_spec, trace_label) - # TODO: rands - log all zeros for now - rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) - return choices, rands - rands = pipeline.get_rn_generator().random_for_df(probs) probs_arr = probs.values.cumsum(axis=1) - rands diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 039182decd..434c7216f1 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -887,8 +887,7 @@ def compute_base_probabilities(nested_probabilities, nests, spec): def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, log_alt_losers=False, want_logsums=False, trace_label=None, - trace_choice_name=None, trace_column_names=None, - choose_individual_max_utility=False): + trace_choice_name=None, trace_column_names=None): """ Run a simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -952,32 +951,37 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, tracing.trace_df(utilities, '%s.utilities' % trace_label, column_labels=['alternative', 'utility']) - #if choose_individual_max_utility: - # choices = logit.make_choices_ru_frozen_mnl(utilities, trace_label=trace_label) - # rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up - #else: - probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) - chunk.log_df(trace_label, "probs", probs) + if config.setting("freeze_unobserved_utilities", False): + if custom_chooser: + # TODO JAN: ADD HERE - need to figure out what to do with custom chooser though. leave out for now, + # we only need to fix up joit_tour_participation for now + raise "Not implemented yet" + + choices = logit.make_choices_utility_based(utilities, trace_label=trace_label) + rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up + + del utilities + chunk.log_df(trace_label, 'utilities', None) - if have_trace_targets: - # report these now in case make_choices throws error on bad_choices - tracing.trace_df(probs, '%s.probs' % trace_label, - column_labels=['alternative', 'probability']) - - if custom_chooser: - # TODO [janzill Jun2022]: is this used somewhere outside Asim such that adding utilities would be a breaking - # change? - choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, - trace_label=trace_label, utilities=utilities) else: - choices, rands = logit.make_choices(probs, utilities=utilities, trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) + chunk.log_df(trace_label, "probs", probs) - del utilities - chunk.log_df(trace_label, 'utilities', None) + if have_trace_targets: + # report these now in case make_choices throws error on bad_choices + tracing.trace_df(probs, '%s.probs' % trace_label, + column_labels=['alternative', 'probability']) + + del utilities + chunk.log_df(trace_label, 'utilities', None) + + if custom_chooser: + choices, rands = custom_chooser(probs=probs, choosers=choosers, spec=spec, trace_label=trace_label) + else: + choices, rands = logit.make_choices(probs, trace_label=trace_label) - del probs - chunk.log_df(trace_label, 'probs', None) + del probs + chunk.log_df(trace_label, 'probs', None) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, @@ -1189,75 +1193,106 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label, column_labels=['alternative', 'utility']) - # exponentiated utilities of leaves and nests - nested_exp_utilities = compute_nested_exp_utilities(raw_utilities, nest_spec) - chunk.log_df(trace_label, "nested_exp_utilities", nested_exp_utilities) - # TODO [janzill Jun2022: combine with nested_exp_utilities somehow?] - # utilities of leaves and nests - nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) - chunk.log_df(trace_label, "nested_utilities", nested_utilities) + if config.setting("freeze_unobserved_utilities", False): + if custom_chooser: + # TODO JAN: ADD HERE - need to figure out what to do with custom chooser though. leave out for now, + # we only need to fix up joit_tour_participation for now + raise "Not implemented yet" - del raw_utilities - chunk.log_df(trace_label, 'raw_utilities', None) + # TODO [janzill Jun2022]: combine with nested_exp_utilities? + # utilities of leaves and nests + nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) + chunk.log_df(trace_label, "nested_utilities", nested_utilities) - if have_trace_targets: - tracing.trace_df(nested_exp_utilities, '%s.nested_exp_utilities' % trace_label, - column_labels=['alternative', 'utility']) + # TODO [janzill Jun2022]: this can be done from utils directly, but use existing methodology for prototype + if want_logsums: + # logsum of nest root + # exponentiated utilities of leaves and nests + nested_exp_utilities = compute_nested_exp_utilities(raw_utilities, nest_spec) + chunk.log_df(trace_label, "nested_exp_utilities", nested_exp_utilities) + logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) + chunk.log_df(trace_label, "logsums", logsums) + + # TODO: index of choices for nested utilities is different than unnested - this needs to be consistent for + # turning indexes into alternative names to keep code changes to minimum for now + name_mapping = raw_utilities.columns.values + + del raw_utilities + chunk.log_df(trace_label, 'raw_utilities', None) + + choices = logit.make_choices_utility_based( + nested_utilities, + name_mapping=name_mapping, + nest_spec=nest_spec, + trace_label=trace_label + ) + rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up + + del(nested_utilities) + chunk.log_df(trace_label, "nested_utilities", None) - # probabilities of alternatives relative to siblings sharing the same nest - nested_probabilities = \ - compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label) - chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) + else: + # exponentiated utilities of leaves and nests + nested_exp_utilities = compute_nested_exp_utilities(raw_utilities, nest_spec) + chunk.log_df(trace_label, "nested_exp_utilities", nested_exp_utilities) - if want_logsums: - # logsum of nest root - logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) - chunk.log_df(trace_label, "logsums", logsums) + del raw_utilities + chunk.log_df(trace_label, 'raw_utilities', None) - del nested_exp_utilities - chunk.log_df(trace_label, 'nested_exp_utilities', None) + if have_trace_targets: + tracing.trace_df(nested_exp_utilities, '%s.nested_exp_utilities' % trace_label, + column_labels=['alternative', 'utility']) - if have_trace_targets: - tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, - column_labels=['alternative', 'probability']) + # probabilities of alternatives relative to siblings sharing the same nest + nested_probabilities = \ + compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label) + chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) + + if want_logsums: + # logsum of nest root + logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) + chunk.log_df(trace_label, "logsums", logsums) - # global (flattened) leaf probabilities based on relative nest coefficients (in spec order) - base_probabilities = compute_base_probabilities(nested_probabilities, nest_spec, spec) - chunk.log_df(trace_label, "base_probabilities", base_probabilities) + del nested_exp_utilities + chunk.log_df(trace_label, 'nested_exp_utilities', None) - del nested_probabilities - chunk.log_df(trace_label, 'nested_probabilities', None) + if have_trace_targets: + tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, + column_labels=['alternative', 'probability']) - if have_trace_targets: - tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label, - column_labels=['alternative', 'probability']) + # global (flattened) leaf probabilities based on relative nest coefficients (in spec order) + base_probabilities = compute_base_probabilities(nested_probabilities, nest_spec, spec) + chunk.log_df(trace_label, "base_probabilities", base_probabilities) - # note base_probabilities could all be zero since we allowed all probs for nests to be zero - # check here to print a clear message but make_choices will raise error if probs don't sum to 1 - BAD_PROB_THRESHOLD = 0.001 - no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD + del nested_probabilities + chunk.log_df(trace_label, 'nested_probabilities', None) - if no_choices.any(): + if have_trace_targets: + tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label, + column_labels=['alternative', 'probability']) - logit.report_bad_choices( - no_choices, base_probabilities, - trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'), - trace_choosers=choosers, - msg="base_probabilities do not sum to one") + # note base_probabilities could all be zero since we allowed all probs for nests to be zero + # check here to print a clear message but make_choices will raise error if probs don't sum to 1 + BAD_PROB_THRESHOLD = 0.001 + no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD - if custom_chooser: - choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, - trace_label=trace_label) - else: - choices, rands = logit.make_choices(base_probabilities, utilities=nested_utilities, nest_spec=nest_spec, - trace_label=trace_label, choose_individual_max_utility=choose_individual_max_utility) + if no_choices.any(): - del(nested_utilities) - chunk.log_df(trace_label, "nested_utilities", None) + logit.report_bad_choices( + no_choices, base_probabilities, + trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'), + trace_choosers=choosers, + msg="base_probabilities do not sum to one") - del base_probabilities - chunk.log_df(trace_label, 'base_probabilities', None) + if custom_chooser: + choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, + trace_label=trace_label) + else: + choices, rands = logit.make_choices(base_probabilities,nest_spec=nest_spec, trace_label=trace_label) + + del base_probabilities + chunk.log_df(trace_label, 'base_probabilities', None) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, @@ -1280,9 +1315,7 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, log_alt_losers=False, want_logsums=False, estimator=None, - trace_label=None, trace_choice_name=None, trace_column_names=None, - choose_individual_max_utility=False - ): + trace_label=None, trace_choice_name=None, trace_column_names=None): """ Run an MNL or NL simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -1337,25 +1370,14 @@ def _simple_simulate(choosers, spec, nest_spec, skims=None, locals_d=None, want_logsums=want_logsums, estimator=estimator, trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, - choose_individual_max_utility=choose_individual_max_utility) + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) else: - ##make_choices now carries choose_individual_max_utility -> incorporate there - #if choose_individual_max_utility: - # choices = eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, - # log_alt_losers=log_alt_losers, - # want_logsums=want_logsums, - # estimator=estimator, - # trace_label=trace_label, - # trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) - #else: choices = eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, log_alt_losers=log_alt_losers, want_logsums=want_logsums, estimator=estimator, trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, - choose_individual_max_utility=choose_individual_max_utility) + trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) return choices @@ -1378,8 +1400,7 @@ def simple_simulate(choosers, spec, nest_spec, log_alt_losers=False, want_logsums=False, estimator=None, - trace_label=None, trace_choice_name=None, trace_column_names=None, - choose_individual_max_utility=False): + trace_label=None, trace_choice_name=None, trace_column_names=None): """ Run an MNL or NL simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -1405,8 +1426,7 @@ def simple_simulate(choosers, spec, nest_spec, estimator=estimator, trace_label=chunk_trace_label, trace_choice_name=trace_choice_name, - trace_column_names=trace_column_names, - choose_individual_max_utility=choose_individual_max_utility) + trace_column_names=trace_column_names) result_list.append(choices) @@ -1427,8 +1447,7 @@ def simple_simulate_by_chunk_id(choosers, spec, nest_spec, want_logsums=False, estimator=None, trace_label=None, - trace_choice_name=None, - choose_individual_max_utility=False): + trace_choice_name=None): """ chunk_by_chunk_id wrapper for simple_simulate """ @@ -1446,8 +1465,7 @@ def simple_simulate_by_chunk_id(choosers, spec, nest_spec, want_logsums=want_logsums, estimator=estimator, trace_label=chunk_trace_label, - trace_choice_name=trace_choice_name, - choose_individual_max_utility=choose_individual_max_utility) + trace_choice_name=trace_choice_name) result_list.append(choices) From f890799f8ae869acec3b82e18bd3737e6f3142f2 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 14:36:24 +1000 Subject: [PATCH 058/135] fake rand tracing --- activitysim/core/simulate.py | 146 +---------------------------------- 1 file changed, 2 insertions(+), 144 deletions(-) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 434c7216f1..183bea60cf 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -957,8 +957,7 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, # we only need to fix up joit_tour_participation for now raise "Not implemented yet" - choices = logit.make_choices_utility_based(utilities, trace_label=trace_label) - rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up + choices, rands = logit.make_choices_utility_based(utilities, trace_label=trace_label) del utilities chunk.log_df(trace_label, 'utilities', None) @@ -991,146 +990,6 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, return choices -# -# def eval_nl_fixed_ru(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, -# log_alt_losers=False, -# want_logsums=False, trace_label=None, -# trace_choice_name=None, trace_column_names=None): -# """ -# Run a nested-logit simulation for when the model spec does not involve alternative -# specific data, e.g. there are no interactions with alternative -# properties and no need to sample from alternatives. -# -# Parameters -# ---------- -# choosers : pandas.DataFrame -# spec : pandas.DataFrame -# A table of variable specifications and coefficient values. -# Variable expressions should be in the table index and the table -# should have a column for each alternative. -# nest_spec: -# dictionary specifying nesting structure and nesting coefficients -# (from the model spec yaml file) -# locals_d : Dict or None -# This is a dictionary of local variables that will be the environment -# for an evaluation of an expression that begins with @ -# custom_chooser : function(probs, choosers, spec, trace_label) returns choices, rands -# custom alternative to logit.make_choices -# estimator : Estimator object -# called to report intermediate table results (used for estimation) -# trace_label: str -# This is the label to be used for trace log file entries and dump file names -# when household tracing enabled. No tracing occurs if label is empty or None. -# trace_choice_name: str -# This is the column label to be used in trace file csv dump of choices -# trace_column_names: str or list of str -# chooser columns to include when tracing expression_values -# -# Returns -# ------- -# choices : pandas.Series -# Index will be that of `choosers`, values will match the columns -# of `spec`. -# """ -# -# trace_label = tracing.extend_trace_label(trace_label, 'eval_nl_fixed_ru') -# assert trace_label -# have_trace_targets = tracing.has_trace_targets(choosers) -# -# logit.validate_nest_spec(nest_spec, trace_label) -# -# if have_trace_targets: -# tracing.trace_df(choosers, '%s.choosers' % trace_label) -# -# raw_utilities = eval_utilities(spec, choosers, locals_d, -# log_alt_losers=log_alt_losers, -# trace_label=trace_label, have_trace_targets=have_trace_targets, -# estimator=estimator, trace_column_names=trace_column_names) -# chunk.log_df(trace_label, "raw_utilities", raw_utilities) -# -# if have_trace_targets: -# tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label, -# column_labels=['alternative', 'utility']) -# -# # utilities of leaves and nests -# nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) -# chunk.log_df(trace_label, "nested_utilities", nested_utilities) -# -# del raw_utilities -# chunk.log_df(trace_label, 'raw_utilities', None) -# -# if have_trace_targets: -# tracing.trace_df(nested_utilities, '%s.nested_utilities' % trace_label, -# column_labels=['alternative', 'utility']) -# -# -# ########### comparison only -# # probabilities of alternatives relative to siblings sharing the same nest -# nested_probabilities = \ -# compute_nested_probabilities(np.exp(nested_utilities), nest_spec, trace_label=trace_label) -# chunk.log_df(trace_label, "nested_probabilities", nested_probabilities) -# -# if have_trace_targets: -# tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label, -# column_labels=['alternative', 'probability']) -# -# base_probabilities = compute_base_probabilities(nested_probabilities, nest_spec, spec) -# chunk.log_df(trace_label, "base_probabilities", base_probabilities) -# -# del nested_probabilities -# chunk.log_df(trace_label, "nested_probabilities", None) -# -# if have_trace_targets: -# tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label, -# column_labels=['alternative', 'probability']) -# -# del(base_probabilities) -# chunk.log_df(trace_label, "base_probabilities", None) -# ########### end comparison only -# -# # TODO - check this is correct -# if want_logsums: -# # logsum of nest root -# logsums = pd.Series(nested_utilities.root, index=choosers.index) -# chunk.log_df(trace_label, "logsums", logsums) -# -# # TODO: add checks on utilities? -# # # note base_probabilities could all be zero since we allowed all probs for nests to be zero -# # # check here to print a clear message but make_choices will raise error if probs don't sum to 1 -# # BAD_PROB_THRESHOLD = 0.001 -# # no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD -# # -# # if no_choices.any(): -# # -# # logit.report_bad_choices( -# # no_choices, base_probabilities, -# # trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'), -# # trace_choosers=choosers, -# # msg="base_probabilities do not sum to one") -# -# # TODO: add custom_chooser -# # if custom_chooser: -# # choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, -# # trace_label=trace_label) -# # else: -# -# choices = logit.make_choices_ru_frozen(nested_utilities, nest_spec, trace_label=trace_label) -# -# if have_trace_targets: -# tracing.trace_df(choices, '%s.choices' % trace_label, -# columns=[None, trace_choice_name]) -# #tracing.trace_df(rands, '%s.rands' % trace_label, -# # columns=[None, 'rand']) -# if want_logsums: -# tracing.trace_df(logsums, '%s.logsums' % trace_label, -# columns=[None, 'logsum']) -# -# if want_logsums: -# choices = choices.to_frame('choice') -# choices['logsum'] = logsums -# -# return choices - def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, log_alt_losers=False, @@ -1221,13 +1080,12 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, del raw_utilities chunk.log_df(trace_label, 'raw_utilities', None) - choices = logit.make_choices_utility_based( + choices, rands = logit.make_choices_utility_based( nested_utilities, name_mapping=name_mapping, nest_spec=nest_spec, trace_label=trace_label ) - rands = pd.Series([0, 0, 0]) # TODO: fix me, bring back up del(nested_utilities) chunk.log_df(trace_label, "nested_utilities", None) From 043a0bdf1b29cd4c46db9b40e9a30b5b7bb7a84d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 14:53:12 +1000 Subject: [PATCH 059/135] working through all make_choices calls to implement memory saving hack until sharrow arrives --- .../models/non_mandatory_tour_frequency.py | 2 - .../abm/models/trip_departure_choice.py | 49 +++++++++++-------- activitysim/abm/models/trip_purpose.py | 7 +-- activitysim/abm/models/util/cdap.py | 21 ++++---- .../models/util/probabilistic_scheduling.py | 2 +- activitysim/core/simulate.py | 2 +- 6 files changed, 43 insertions(+), 40 deletions(-) diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index 5816d3349d..e0dd4b92c1 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -112,8 +112,6 @@ def extend_tour_counts(persons, tour_counts, alternatives, trace_hh_id, trace_la choosers[PROBABILITY_COLUMNS], trace_label=tour_type_trace_label, trace_choosers=choosers - # [janzill Jun2022]: don't change for probs tables - #choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) ) # - extend tour_count (0-based prob alternative choice equals magnitude of extension) diff --git a/activitysim/abm/models/trip_departure_choice.py b/activitysim/abm/models/trip_departure_choice.py index 37b3b48223..f19f4241c3 100644 --- a/activitysim/abm/models/trip_departure_choice.py +++ b/activitysim/abm/models/trip_departure_choice.py @@ -268,32 +268,41 @@ def choose_tour_leg_pattern(trip_segment, tracing.trace_df(utilities_df, tracing.extend_trace_label(trace_label, 'utilities'), column_labels=['alternative', 'utility']) - # convert to probabilities (utilities exponentiated and normalized to probs) - # probs is same shape as utilities, one row per chooser and one column for alternative - probs = logit.utils_to_probs(utilities_df, - trace_label=trace_label, trace_choosers=trip_segment) + if config.setting("freeze_unobserved_utilities", False): + # make choices + # positions is series with the chosen alternative represented as a column index in probs + # which is an integer between zero and num alternatives in the alternative sample + positions, rands = logit.make_choices_utility_based( + utilities_df, trace_label=trace_label, trace_choosers=trip_segment + ) + + del utilities_df + chunk.log_df(trace_label, 'utilities_df', None) + else: + # convert to probabilities (utilities exponentiated and normalized to probs) + # probs is same shape as utilities, one row per chooser and one column for alternative + probs = logit.utils_to_probs(utilities_df, + trace_label=trace_label, trace_choosers=trip_segment) - chunk.log_df(trace_label, 'probs', probs) + chunk.log_df(trace_label, 'probs', probs) - if have_trace_targets: - tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), - column_labels=['alternative', 'probability']) + if have_trace_targets: + tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), + column_labels=['alternative', 'probability']) - # make choices - # positions is series with the chosen alternative represented as a column index in probs - # which is an integer between zero and num alternatives in the alternative sample - positions, rands = \ - logit.make_choices(probs, utilities=utilities_df, trace_label=trace_label, trace_choosers=trip_segment, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + del utilities_df + chunk.log_df(trace_label, 'utilities_df', None) - chunk.log_df(trace_label, 'positions', positions) - chunk.log_df(trace_label, 'rands', rands) + # make choices + # positions is series with the chosen alternative represented as a column index in probs + # which is an integer between zero and num alternatives in the alternative sample + positions, rands = logit.make_choices(probs, trace_label=trace_label, trace_choosers=trip_segment) - del utilities_df - chunk.log_df(trace_label, 'utilities_df', None) + del probs + chunk.log_df(trace_label, 'probs', None) - del probs - chunk.log_df(trace_label, 'probs', None) + chunk.log_df(trace_label, 'positions', positions) + chunk.log_df(trace_label, 'rands', rands) # shouldn't have chosen any of the dummy pad utilities assert positions.max() < max_sample_count diff --git a/activitysim/abm/models/trip_purpose.py b/activitysim/abm/models/trip_purpose.py index 1058e0685f..ab4dc28007 100644 --- a/activitysim/abm/models/trip_purpose.py +++ b/activitysim/abm/models/trip_purpose.py @@ -106,12 +106,7 @@ def choose_intermediate_trip_purpose( print(choosers[probs_cols]) estimator.write_table(choosers[probs_cols], 'probs', append=True) - choices, rands = logit.make_choices( - choosers[purpose_cols], - trace_label=trace_label, trace_choosers=choosers - # [janzill Jun2022]: don't change for probs tables - #choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) - ) + choices, rands = logit.make_choices(choosers[purpose_cols], trace_label=trace_label, trace_choosers=choosers) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, columns=[None, 'trip_purpose']) diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index 3af60c183d..d0bf565931 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -636,14 +636,14 @@ def household_activity_choices(indiv_utils, interaction_coefficients, hhsize, if len(utils.index) == 0: return pd.Series(dtype='float64') - probs = logit.utils_to_probs(utils, trace_label=trace_label) + if config.setting("freeze_unobserved_utilities", False): + idx_choices, rands = logit.make_choices(utils, trace_label=trace_label) + else: + probs = logit.utils_to_probs(utils, trace_label=trace_label) - # select an activity pattern alternative for each household based on probability - # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs - idx_choices, rands = logit.make_choices( - probs, utilities=utils, trace_label=trace_label, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) - ) + # select an activity pattern alternative for each household based on probability + # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs + idx_choices, rands = logit.make_choices(probs, trace_label=trace_label) # convert choice expressed as index into alternative name from util column label choices = pd.Series(utils.columns[idx_choices].values, index=utils.index) @@ -656,8 +656,9 @@ def household_activity_choices(indiv_utils, interaction_coefficients, hhsize, tracing.trace_df(utils, '%s.hhsize%d_utils' % (trace_label, hhsize), column_labels=['expression', 'household']) - tracing.trace_df(probs, '%s.hhsize%d_probs' % (trace_label, hhsize), - column_labels=['expression', 'household']) + if not config.setting("freeze_unobserved_utilities", False): + tracing.trace_df(probs, '%s.hhsize%d_probs' % (trace_label, hhsize), + column_labels=['expression', 'household']) tracing.trace_df(choices, '%s.hhsize%d_activity_choices' % (trace_label, hhsize), column_labels=['expression', 'household']) tracing.trace_df(rands, '%s.hhsize%d_rands' % (trace_label, hhsize), @@ -765,7 +766,7 @@ def extra_hh_member_choices(persons, cdap_fixed_relative_proportions, locals_d, # select an activity pattern alternative for each person based on probability # idx_choices is a series (indexed on _persons_index_ ) with the chosen alternative represented # as the integer (0 based) index of the chosen column from probs - idx_choices, rands = logit.make_choices(probs, trace_label=trace_label) # no fru for prob tables + idx_choices, rands = logit.make_choices(probs, trace_label=trace_label) # convert choice from column index to activity name choices = pd.Series(probs.columns[idx_choices].values, index=probs.index) diff --git a/activitysim/abm/models/util/probabilistic_scheduling.py b/activitysim/abm/models/util/probabilistic_scheduling.py index cc5869ba57..73500f58e8 100644 --- a/activitysim/abm/models/util/probabilistic_scheduling.py +++ b/activitysim/abm/models/util/probabilistic_scheduling.py @@ -248,7 +248,7 @@ def make_scheduling_choices( if trace_hh_id and tracing.has_trace_targets(choosers_df): tracing.trace_df(chooser_probs, '%s.chooser_probs' % trace_label) - raw_choices, rands = logit.make_choices(chooser_probs, trace_label=trace_label, trace_choosers=choosers) # no fru + raw_choices, rands = logit.make_choices(chooser_probs, trace_label=trace_label, trace_choosers=choosers) chunk.log_df(trace_label, "choices", raw_choices) chunk.log_df(trace_label, "rands", rands) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 183bea60cf..5ce21c359d 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -954,7 +954,7 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, if config.setting("freeze_unobserved_utilities", False): if custom_chooser: # TODO JAN: ADD HERE - need to figure out what to do with custom chooser though. leave out for now, - # we only need to fix up joit_tour_participation for now + # we only need to fix up joint_tour_participation for now raise "Not implemented yet" choices, rands = logit.make_choices_utility_based(utilities, trace_label=trace_label) From f9b1e2e09f6d6094aeaf02eb21009afa57ade4e6 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 15:59:30 +1000 Subject: [PATCH 060/135] working through all make_choices calls to implement memory saving hack until sharrow arrives --- .../core/interaction_sample_simulate.py | 69 +++++++++++-------- activitysim/core/interaction_simulate.py | 47 ++++++++----- activitysim/core/pathbuilder.py | 5 +- 3 files changed, 71 insertions(+), 50 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 573ac89b8d..ad81696213 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -8,6 +8,7 @@ from . import logit from . import tracing from . import chunk +from . import config from .simulate import set_skim_wrapper_targets from . import interaction_simulate @@ -22,7 +23,7 @@ def _interaction_sample_simulate( want_logsums, skims, locals_d, trace_label, trace_choice_name, - estimator, choose_individual_max_utility): + estimator): """ Run a MNL simulation in the situation in which alternatives must @@ -205,35 +206,49 @@ def _interaction_sample_simulate( tracing.trace_df(utilities_df, tracing.extend_trace_label(trace_label, 'utilities'), column_labels=['alternative', 'utility']) - # convert to probabilities (utilities exponentiated and normalized to probs) - # probs is same shape as utilities, one row per chooser and one column for alternative - probs = logit.utils_to_probs(utilities_df, allow_zero_probs=allow_zero_probs, - trace_label=trace_label, trace_choosers=choosers) - chunk.log_df(trace_label, 'probs', probs) + if config.setting("freeze_unobserved_utilities", False): + # positions is series with the chosen alternative represented as a column index in utilities_df + # which is an integer between zero and num alternatives in the alternative sample + positions, rands = logit.make_choices_utility_based( + utilities_df, trace_label=trace_label, trace_choosers=choosers + ) - if want_logsums: - logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) - chunk.log_df(trace_label, 'logsums', logsums) + if want_logsums: + logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) + chunk.log_df(trace_label, 'logsums', logsums) - if have_trace_targets: - tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), - column_labels=['alternative', 'probability']) + del utilities_df + chunk.log_df(trace_label, 'utilities_df', None) + + else: + # convert to probabilities (utilities exponentiated and normalized to probs) + # probs is same shape as utilities, one row per chooser and one column for alternative + probs = logit.utils_to_probs(utilities_df, allow_zero_probs=allow_zero_probs, + trace_label=trace_label, trace_choosers=choosers) + chunk.log_df(trace_label, 'probs', probs) + + if want_logsums: + logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) + chunk.log_df(trace_label, 'logsums', logsums) + + del utilities_df + chunk.log_df(trace_label, 'utilities_df', None) - if allow_zero_probs: - zero_probs = (probs.sum(axis=1) == 0) - if zero_probs.any(): - # FIXME this is kind of gnarly, but we force choice of first alt - probs.loc[zero_probs, 0] = 1.0 + if have_trace_targets: + tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), + column_labels=['alternative', 'probability']) - # make choices - # positions is series with the chosen alternative represented as a column index in probs - # which is an integer between zero and num alternatives in the alternative sample - positions, rands = \ - logit.make_choices(probs, utilities_df, trace_label=trace_label, trace_choosers=choosers, - choose_individual_max_utility=choose_individual_max_utility) + if allow_zero_probs: + zero_probs = (probs.sum(axis=1) == 0) + if zero_probs.any(): + # FIXME this is kind of gnarly, but we force choice of first alt + probs.loc[zero_probs, 0] = 1.0 - del utilities_df - chunk.log_df(trace_label, 'utilities_df', None) + # make choices + # positions is series with the chosen alternative represented as a column index in probs + # which is an integer between zero and num alternatives in the alternative sample + positions, rands = \ + logit.make_choices(probs, utilities_df, trace_label=trace_label, trace_choosers=choosers) chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) @@ -288,7 +303,7 @@ def interaction_sample_simulate( want_logsums=False, skims=None, locals_d=None, chunk_size=0, chunk_tag=None, trace_label=None, trace_choice_name=None, - estimator=None, choose_individual_max_utility=False): + estimator=None): """ Run a simulation in the situation in which alternatives must @@ -358,7 +373,7 @@ def interaction_sample_simulate( want_logsums, skims, locals_d, chunk_trace_label, trace_choice_name, - estimator, choose_individual_max_utility) + estimator) result_list.append(choices) diff --git a/activitysim/core/interaction_simulate.py b/activitysim/core/interaction_simulate.py index b0a74ab1e0..70067ebb95 100644 --- a/activitysim/core/interaction_simulate.py +++ b/activitysim/core/interaction_simulate.py @@ -240,8 +240,7 @@ def _interaction_simulate( skims=None, locals_d=None, sample_size=None, trace_label=None, trace_choice_name=None, log_alt_losers=False, - estimator=None, - choose_individual_max_utility=False): + estimator=None): """ Run a MNL simulation in the situation in which alternatives must be merged with choosers because there are interaction terms or @@ -375,24 +374,34 @@ def _interaction_simulate( tracing.dump_df(DUMP, utilities, trace_label, 'utilities') - # convert to probabilities (utilities exponentiated and normalized to probs) - # probs is same shape as utilities, one row per chooser and one column for alternative - probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) - chunk.log_df(trace_label, 'probs', probs) + if config.setting("freeze_unobserved_utilities", False): + positions, rands = logit.make_choices_utility_based( + utilities, trace_label=trace_label, trace_choosers=choosers + ) - if have_trace_targets: - tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), - column_labels=['alternative', 'probability']) - - # make choices - # positions is series with the chosen alternative represented as a column index in probs - # which is an integer between zero and num alternatives in the alternative sample - positions, rands = \ - logit.make_choices(probs, utilities=utilities, trace_label=trace_label, trace_choosers=choosers, - choose_individual_max_utility=choose_individual_max_utility) - - del utilities - chunk.log_df(trace_label, 'utilities', None) + del utilities + chunk.log_df(trace_label, 'utilities', None) + + else: + # convert to probabilities (utilities exponentiated and normalized to probs) + # probs is same shape as utilities, one row per chooser and one column for alternative + probs = logit.utils_to_probs(utilities, trace_label=trace_label, trace_choosers=choosers) + chunk.log_df(trace_label, 'probs', probs) + + if have_trace_targets: + tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), + column_labels=['alternative', 'probability']) + + del utilities + chunk.log_df(trace_label, 'utilities', None) + + # make choices + # positions is series with the chosen alternative represented as a column index in probs + # which is an integer between zero and num alternatives in the alternative sample + positions, rands = logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers) + + del probs + chunk.log_df(trace_label, 'probs', None) chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) diff --git a/activitysim/core/pathbuilder.py b/activitysim/core/pathbuilder.py index 28d0707575..d4cee77504 100644 --- a/activitysim/core/pathbuilder.py +++ b/activitysim/core/pathbuilder.py @@ -744,10 +744,7 @@ def build_virtual_path(self, recipe, path_type, orig, dest, tod, demographic_seg self.trace_df(probs, trace_label, 'probs') else: - choices, rands = logit.make_choices( - probs, utilities=utilities_df, allow_bad_probs=True, trace_label=trace_label, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) - ) + choices, rands = logit.make_choices(probs, allow_bad_probs=True, trace_label=trace_label) chunk.log_df(trace_label, "rands", rands) del rands From 4b5cad2681a3ef6a968bfcf0957961d575d8b6f7 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:02:52 +1000 Subject: [PATCH 061/135] working through all make_choices calls to implement memory saving hack until sharrow arrives --- activitysim/abm/models/location_choice.py | 3 +-- activitysim/abm/models/non_mandatory_tour_frequency.py | 3 +-- activitysim/abm/models/parking_location_choice.py | 4 ++-- activitysim/abm/models/trip_destination.py | 3 +-- activitysim/abm/models/trip_scheduling_choice.py | 3 +-- activitysim/abm/models/util/tour_destination.py | 4 ++-- activitysim/abm/models/util/tour_od.py | 4 ++-- activitysim/abm/models/util/vectorize_tour_scheduling.py | 3 +-- activitysim/core/interaction_simulate.py | 6 ++---- 9 files changed, 13 insertions(+), 20 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 7be64b848d..3670c089ef 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -538,8 +538,7 @@ def run_location_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name=model_settings['DEST_CHOICE_COLUMN_NAME'], - estimator=estimator, - choose_individual_max_utility=choose_individual_max_utility + estimator=estimator ) if not want_logsums: diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index e0dd4b92c1..2ae277296b 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -232,8 +232,7 @@ def non_mandatory_tour_frequency(persons, persons_merged, chunk_size=chunk_size, trace_label='non_mandatory_tour_frequency.%s' % segment_name, trace_choice_name='non_mandatory_tour_frequency', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) if estimator: estimator.write_choices(choices) diff --git a/activitysim/abm/models/parking_location_choice.py b/activitysim/abm/models/parking_location_choice.py index c81eb89691..8802981616 100644 --- a/activitysim/abm/models/parking_location_choice.py +++ b/activitysim/abm/models/parking_location_choice.py @@ -124,8 +124,8 @@ def parking_destination_simulate( locals_d=locals_dict, chunk_size=chunk_size, trace_label=trace_label, - trace_choice_name='parking_loc', - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + trace_choice_name='parking_loc' + ) # drop any failed zero_prob destinations if (parking_locations == NO_DESTINATION).any(): diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index 1042197b83..167c327466 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -710,8 +710,7 @@ def trip_destination_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name='trip_dest', - estimator=estimator, - choose_individual_max_utility=choose_individual_max_utility + estimator=estimator ) if not want_logsums: diff --git a/activitysim/abm/models/trip_scheduling_choice.py b/activitysim/abm/models/trip_scheduling_choice.py index b1e6f72626..c950b0726f 100644 --- a/activitysim/abm/models/trip_scheduling_choice.py +++ b/activitysim/abm/models/trip_scheduling_choice.py @@ -261,8 +261,7 @@ def run_trip_scheduling_choice(spec, tours, skims, locals_dict, locals_d=locals_dict, trace_label=chunk_trace_label, trace_choice_name='trip_schedule_stage_1', - estimator=None, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + estimator=None ) assert len(choices.index) == len(choosers.index) diff --git a/activitysim/abm/models/util/tour_destination.py b/activitysim/abm/models/util/tour_destination.py index 7e3303c530..72212aff9d 100644 --- a/activitysim/abm/models/util/tour_destination.py +++ b/activitysim/abm/models/util/tour_destination.py @@ -645,8 +645,8 @@ def run_destination_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name='destination', - estimator=estimator, - choose_individual_max_utility=choose_individual_max_utility) + estimator=estimator + ) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/activitysim/abm/models/util/tour_od.py b/activitysim/abm/models/util/tour_od.py index 731aebe04a..ea632554a3 100644 --- a/activitysim/abm/models/util/tour_od.py +++ b/activitysim/abm/models/util/tour_od.py @@ -863,8 +863,8 @@ def run_od_simulate( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='origin_destination', - estimator=estimator, - choose_individual_max_utility=choose_individual_max_utility) + estimator=estimator + ) if not want_logsums: choices = choices.to_frame('choice') diff --git a/activitysim/abm/models/util/vectorize_tour_scheduling.py b/activitysim/abm/models/util/vectorize_tour_scheduling.py index ca77ab6c08..199059e2af 100644 --- a/activitysim/abm/models/util/vectorize_tour_scheduling.py +++ b/activitysim/abm/models/util/vectorize_tour_scheduling.py @@ -600,8 +600,7 @@ def _schedule_tours( locals_d=locals_d, chunk_size=0, trace_label=tour_trace_label, - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + estimator=estimator ) chunk.log_df(tour_trace_label, 'choices', choices) diff --git a/activitysim/core/interaction_simulate.py b/activitysim/core/interaction_simulate.py index 70067ebb95..827ed3aab3 100644 --- a/activitysim/core/interaction_simulate.py +++ b/activitysim/core/interaction_simulate.py @@ -432,8 +432,7 @@ def interaction_simulate( log_alt_losers=False, skims=None, locals_d=None, sample_size=None, chunk_size=0, trace_label=None, trace_choice_name=None, - estimator=None, - choose_individual_max_utility=False): + estimator=None): """ Run a simulation in the situation in which alternatives must @@ -499,8 +498,7 @@ def interaction_simulate( trace_label=chunk_trace_label, trace_choice_name=trace_choice_name, log_alt_losers=log_alt_losers, - estimator=estimator, - choose_individual_max_utility=choose_individual_max_utility) + estimator=estimator) result_list.append(choices) From b48dd71bbf9924351dd27659eec925bd3e4c138b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:09:57 +1000 Subject: [PATCH 062/135] working through all make_choices calls to implement memory saving hack until sharrow arrives --- activitysim/abm/models/atwork_subtour_frequency.py | 3 +-- activitysim/abm/models/auto_ownership.py | 3 +-- activitysim/abm/models/free_parking.py | 3 +-- activitysim/abm/models/joint_tour_composition.py | 3 +-- activitysim/abm/models/joint_tour_frequency.py | 3 +-- activitysim/abm/models/mandatory_tour_frequency.py | 3 +-- activitysim/abm/models/stop_frequency.py | 3 +-- activitysim/abm/models/trip_mode_choice.py | 3 +-- activitysim/abm/models/util/mode.py | 9 +++------ .../example_semcog/extensions/telecommute_frequency.py | 3 +-- .../example_semcog/extensions/transit_pass_ownership.py | 3 +-- .../example_semcog/extensions/transit_pass_subsidy.py | 3 +-- .../examples/example_semcog/extensions/work_from_home.py | 3 +-- 13 files changed, 15 insertions(+), 30 deletions(-) diff --git a/activitysim/abm/models/atwork_subtour_frequency.py b/activitysim/abm/models/atwork_subtour_frequency.py index ed62db14cd..cb9d3ca4c7 100644 --- a/activitysim/abm/models/atwork_subtour_frequency.py +++ b/activitysim/abm/models/atwork_subtour_frequency.py @@ -88,8 +88,7 @@ def atwork_subtour_frequency(tours, chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='atwork_subtour_frequency', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/auto_ownership.py b/activitysim/abm/models/auto_ownership.py index a6a1e2106a..12f8fbd2bb 100644 --- a/activitysim/abm/models/auto_ownership.py +++ b/activitysim/abm/models/auto_ownership.py @@ -57,8 +57,7 @@ def auto_ownership_simulate(households, trace_label=trace_label, trace_choice_name='auto_ownership', log_alt_losers=log_alt_losers, - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) if estimator: estimator.write_choices(choices) diff --git a/activitysim/abm/models/free_parking.py b/activitysim/abm/models/free_parking.py index 9f37872f9a..b144ed0543 100644 --- a/activitysim/abm/models/free_parking.py +++ b/activitysim/abm/models/free_parking.py @@ -68,8 +68,7 @@ def free_parking( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='free_parking_at_work', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) free_parking_alt = model_settings['FREE_PARKING_ALT'] choices = (choices == free_parking_alt) diff --git a/activitysim/abm/models/joint_tour_composition.py b/activitysim/abm/models/joint_tour_composition.py index 3e6cbc10f4..f21b8f8677 100644 --- a/activitysim/abm/models/joint_tour_composition.py +++ b/activitysim/abm/models/joint_tour_composition.py @@ -96,8 +96,7 @@ def joint_tour_composition( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='composition', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/joint_tour_frequency.py b/activitysim/abm/models/joint_tour_frequency.py index 8cf63e332d..d8930d9834 100644 --- a/activitysim/abm/models/joint_tour_frequency.py +++ b/activitysim/abm/models/joint_tour_frequency.py @@ -87,8 +87,7 @@ def joint_tour_frequency( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='joint_tour_frequency', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/mandatory_tour_frequency.py b/activitysim/abm/models/mandatory_tour_frequency.py index de3c6f2a5a..bbe1df599e 100644 --- a/activitysim/abm/models/mandatory_tour_frequency.py +++ b/activitysim/abm/models/mandatory_tour_frequency.py @@ -96,8 +96,7 @@ def mandatory_tour_frequency(persons_merged, chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='mandatory_tour_frequency', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/stop_frequency.py b/activitysim/abm/models/stop_frequency.py index db967f8b77..14ddeb0669 100644 --- a/activitysim/abm/models/stop_frequency.py +++ b/activitysim/abm/models/stop_frequency.py @@ -143,8 +143,7 @@ def stop_frequency( chunk_size=chunk_size, trace_label=tracing.extend_trace_label(trace_label, segment_name), trace_choice_name='stops', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) # convert indexes to alternative names choices = pd.Series(segment_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/abm/models/trip_mode_choice.py b/activitysim/abm/models/trip_mode_choice.py index 623c4b34df..59e9fc5e5e 100644 --- a/activitysim/abm/models/trip_mode_choice.py +++ b/activitysim/abm/models/trip_mode_choice.py @@ -194,8 +194,7 @@ def trip_mode_choice( logsum_column_name=logsum_column_name, trace_label=segment_trace_label, trace_choice_name='trip_mode_choice', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) if trace_hh_id: # trace the coefficients diff --git a/activitysim/abm/models/util/mode.py b/activitysim/abm/models/util/mode.py index ec3f39dd8f..41937ff0e3 100644 --- a/activitysim/abm/models/util/mode.py +++ b/activitysim/abm/models/util/mode.py @@ -25,8 +25,7 @@ def mode_choice_simulate( trace_label, trace_choice_name, trace_column_names=None, - estimator=None, - choose_individual_max_utility=False): + estimator=None): """ common method for both tour_mode_choice and trip_mode_choice @@ -61,8 +60,7 @@ def mode_choice_simulate( trace_label=trace_label, trace_choice_name=trace_choice_name, estimator=estimator, - trace_column_names=trace_column_names, - choose_individual_max_utility=choose_individual_max_utility) + trace_column_names=trace_column_names) # for consistency, always return dataframe, whether or not logsums were requested if isinstance(choices, pd.Series): @@ -146,8 +144,7 @@ def run_tour_mode_choice_simulate( trace_label=trace_label, trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) + estimator=estimator ) return choices diff --git a/activitysim/examples/example_semcog/extensions/telecommute_frequency.py b/activitysim/examples/example_semcog/extensions/telecommute_frequency.py index 6246820e5d..bbe72e11f1 100755 --- a/activitysim/examples/example_semcog/extensions/telecommute_frequency.py +++ b/activitysim/examples/example_semcog/extensions/telecommute_frequency.py @@ -75,8 +75,7 @@ def telecommute_frequency( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='telecommute_frequency', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) choices = pd.Series(model_spec.columns[choices.values], index=choices.index) diff --git a/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py b/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py index 1f111e3dec..1b9d3c1fae 100644 --- a/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py +++ b/activitysim/examples/example_semcog/extensions/transit_pass_ownership.py @@ -69,8 +69,7 @@ def transit_pass_ownership( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='transit_pass_ownership', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) if estimator: estimator.write_choices(choices) diff --git a/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py b/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py index ae02208c83..ddbcfbef32 100644 --- a/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py +++ b/activitysim/examples/example_semcog/extensions/transit_pass_subsidy.py @@ -69,8 +69,7 @@ def transit_pass_subsidy( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='transit_pass_subsidy', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) if estimator: estimator.write_choices(choices) diff --git a/activitysim/examples/example_semcog/extensions/work_from_home.py b/activitysim/examples/example_semcog/extensions/work_from_home.py index 38182fcd39..f6e2728b23 100755 --- a/activitysim/examples/example_semcog/extensions/work_from_home.py +++ b/activitysim/examples/example_semcog/extensions/work_from_home.py @@ -87,8 +87,7 @@ def work_from_home( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='work_from_home', - estimator=estimator, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + estimator=estimator) if iterations_target_percent is not None: choices_for_filter = choices[choosers[iterations_chooser_filter]] From 17792f5edb81eb0eb64b3f866b99015b3a1b1f2b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:13:49 +1000 Subject: [PATCH 063/135] working through all make_choices calls to implement memory saving hack until sharrow arrives --- .../abm/models/joint_tour_participation.py | 2 +- activitysim/abm/models/location_choice.py | 41 +++++++------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index af3a32e543..c69b5093c5 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -379,4 +379,4 @@ def joint_tour_participation( label="joint_tour_participation.participants") tracing.trace_df(joint_tours, - label="joint_tour_participation.joint_tours") \ No newline at end of file + label="joint_tour_participation.joint_tours") diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 3670c089ef..1c893bccbc 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -109,8 +109,7 @@ def _location_sample( model_settings, alt_dest_col_name, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): """ select a sample of alternative locations. @@ -162,8 +161,7 @@ def _location_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility + trace_label=trace_label ) return choices @@ -177,8 +175,7 @@ def location_sample( estimator, model_settings, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): # FIXME - MEMORY HACK - only include columns actually used in spec chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS'] @@ -202,8 +199,7 @@ def location_sample( model_settings, alt_dest_col_name, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility) + trace_label) return choices @@ -269,8 +265,7 @@ def location_presample( estimator, model_settings, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): trace_label = tracing.extend_trace_label(trace_label, 'presample') @@ -308,8 +303,7 @@ def location_presample( model_settings, DEST_TAZ, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility) + trace_label) # print(f"taz_sample\n{taz_sample}") # dest_TAZ prob pick_count @@ -336,8 +330,7 @@ def run_location_sample( estimator, model_settings, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): """ select a sample of alternative locations. @@ -380,8 +373,7 @@ def run_location_sample( model_settings, chunk_size, chunk_tag=f'{chunk_tag}.presample', - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) else: @@ -394,8 +386,7 @@ def run_location_sample( model_settings, chunk_size, chunk_tag=f'{chunk_tag}.sample', - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) return choices @@ -473,7 +464,7 @@ def run_location_simulate( estimator, model_settings, chunk_size, chunk_tag, - trace_label, choose_individual_max_utility): + trace_label): """ run location model on location_sample annotated with mode_choice logsum to select a dest zone from sample alternatives @@ -560,8 +551,7 @@ def run_location_choice( estimator, model_settings, chunk_size, chunk_tag, - trace_hh_id, trace_label, - choose_individual_max_utility, + trace_hh_id, trace_label ): """ Run the three-part location choice algorithm to generate a location choice for each chooser @@ -624,8 +614,7 @@ def run_location_choice( model_settings, chunk_size, chunk_tag, # run_location_sample will add appropriate suffix for sample or presample - trace_label=tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name), - choose_individual_max_utility=choose_individual_max_utility) + trace_label=tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name)) # - location_logsums location_sample_df = \ @@ -650,8 +639,7 @@ def run_location_choice( estimator, model_settings, chunk_size, chunk_tag=f'{chunk_tag}.simulate', - trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name), - choose_individual_max_utility=choose_individual_max_utility) + trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name)) if estimator: if trace_hh_id: @@ -795,8 +783,7 @@ def iterate_location_choice( model_settings=model_settings, chunk_size=chunk_size, chunk_tag=chunk_tag, trace_hh_id=trace_hh_id, - trace_label=tracing.extend_trace_label(trace_label, 'i%s' % iteration), - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + trace_label=tracing.extend_trace_label(trace_label, 'i%s' % iteration)) # choices_df is a pandas DataFrame with columns 'choice' and (optionally) 'logsum' if choices_df is None: From affe2b6dda4c12ec231b67fcca4b27d12d30a078 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:21:32 +1000 Subject: [PATCH 064/135] remove usage of choose_individual_max_utility --- activitysim/abm/models/location_choice.py | 6 +-- .../models/non_mandatory_tour_frequency.py | 3 +- .../abm/models/parking_location_choice.py | 3 +- activitysim/abm/models/tour_od_choice.py | 3 +- activitysim/abm/models/util/tour_od.py | 37 ++++++------------- 5 files changed, 16 insertions(+), 36 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 1c893bccbc..5b3b7d1e47 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -161,8 +161,7 @@ def _location_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label - ) + trace_label=trace_label) return choices @@ -529,8 +528,7 @@ def run_location_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name=model_settings['DEST_CHOICE_COLUMN_NAME'], - estimator=estimator - ) + estimator=estimator) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index 2ae277296b..814bec97e6 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -111,8 +111,7 @@ def extend_tour_counts(persons, tour_counts, alternatives, trace_hh_id, trace_la choices, rands = logit.make_choices( choosers[PROBABILITY_COLUMNS], trace_label=tour_type_trace_label, - trace_choosers=choosers - ) + trace_choosers=choosers) # - extend tour_count (0-based prob alternative choice equals magnitude of extension) if choices.any(): diff --git a/activitysim/abm/models/parking_location_choice.py b/activitysim/abm/models/parking_location_choice.py index 8802981616..5c13969eed 100644 --- a/activitysim/abm/models/parking_location_choice.py +++ b/activitysim/abm/models/parking_location_choice.py @@ -124,8 +124,7 @@ def parking_destination_simulate( locals_d=locals_dict, chunk_size=chunk_size, trace_label=trace_label, - trace_choice_name='parking_loc' - ) + trace_choice_name='parking_loc') # drop any failed zero_prob destinations if (parking_locations == NO_DESTINATION).any(): diff --git a/activitysim/abm/models/tour_od_choice.py b/activitysim/abm/models/tour_od_choice.py index 41705358e0..6816eed385 100644 --- a/activitysim/abm/models/tour_od_choice.py +++ b/activitysim/abm/models/tour_od_choice.py @@ -93,8 +93,7 @@ def tour_od_choice( model_settings, network_los, estimator, - chunk_size, trace_hh_id, trace_label, - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + chunk_size, trace_hh_id, trace_label) if estimator: assert estimator.want_unsampled_alternatives diff --git a/activitysim/abm/models/util/tour_od.py b/activitysim/abm/models/util/tour_od.py index ea632554a3..bd017c4cae 100644 --- a/activitysim/abm/models/util/tour_od.py +++ b/activitysim/abm/models/util/tour_od.py @@ -115,8 +115,7 @@ def _od_sample( alt_od_col_name, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): model_spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC', @@ -173,8 +172,7 @@ def _od_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) return choices @@ -479,8 +477,7 @@ def od_presample( destination_size_terms, estimator, chunk_size, - trace_label, - choose_individual_max_utility): + trace_label): trace_label = tracing.extend_trace_label(trace_label, 'presample') chunk_tag = 'tour_od.presample' @@ -510,8 +507,7 @@ def od_presample( alt_od_col_name, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility) + trace_label) orig_MAZ_dest_TAZ_sample[ORIG_MAZ] = orig_MAZ_dest_TAZ_sample[alt_od_col_name].str.split('_').str[0].astype(int) orig_MAZ_dest_TAZ_sample[DEST_TAZ] = orig_MAZ_dest_TAZ_sample[alt_od_col_name].str.split('_').str[1].astype(int) @@ -586,8 +582,7 @@ def run_od_sample( destination_size_terms, estimator, chunk_size, - trace_label, - choose_individual_max_utility): + trace_label): model_spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC', segment_name=spec_segment_name, estimator=estimator) @@ -621,9 +616,7 @@ def run_od_sample( destination_size_terms, estimator, chunk_size, - trace_label, - choose_individual_max_utility - ) + trace_label) else: choices = od_sample( @@ -633,9 +626,7 @@ def run_od_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility - ) + chunk_size, trace_label) return choices @@ -791,8 +782,7 @@ def run_od_simulate( destination_size_terms, estimator, chunk_size, - trace_label, - choose_individual_max_utility): + trace_label): """ run simulate OD choices on tour_od_sample annotated with mode_choice logsum to select a tour OD from sample alternatives @@ -882,8 +872,7 @@ def run_tour_od( model_settings, network_los, estimator, - chunk_size, trace_hh_id, trace_label, - choose_individual_max_utility): + chunk_size, trace_hh_id, trace_label): size_term_calculator = SizeTermCalculator(model_settings['SIZE_TERM_SELECTOR']) preprocessor_settings = model_settings.get('preprocessor', None) @@ -933,10 +922,7 @@ def run_tour_od( segment_destination_size_terms, estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label( - trace_label, 'sample.%s' % segment_name), - choose_individual_max_utility=choose_individual_max_utility - ) + trace_label=tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name)) if model_settings['ORIG_FILTER'] == 'original_MAZ > 0': pass @@ -975,8 +961,7 @@ def run_tour_od( destination_size_terms=segment_destination_size_terms, estimator=estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name), - choose_individual_max_utility=choose_individual_max_utility) + trace_label=tracing.extend_trace_label(trace_label, 'simulate.%s' % segment_name)) choices_list.append(choices) if estimator: From 6b38f128566027460966ca082c64592f9da46e0f Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:26:48 +1000 Subject: [PATCH 065/135] remove usage of choose_individual_max_utility --- activitysim/abm/models/trip_destination.py | 44 +++++++--------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index 167c327466..1e0462efe4 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -58,8 +58,7 @@ def _destination_sample( estimator, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): """ Note: trips with no viable destination receive no sample rows @@ -110,9 +109,7 @@ def _destination_sample( skims=skims, locals_d=locals_dict, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility - ) + trace_label=trace_label) return choices @@ -126,8 +123,7 @@ def destination_sample( skim_hotel, estimator, chunk_size, - trace_label, - choose_individual_max_utility): + trace_label): chunk_tag = 'trip_destination.sample' @@ -145,9 +141,7 @@ def destination_sample( estimator, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility - ) + trace_label=trace_label) return choices @@ -367,8 +361,7 @@ def destination_presample( network_los, estimator, chunk_size, trace_hh_id, - trace_label, - choose_individual_max_utility): + trace_label): trace_label = tracing.extend_trace_label(trace_label, 'presample') chunk_tag = 'trip_destination.presample' # distinguish from trip_destination.sample @@ -406,8 +399,7 @@ def destination_presample( estimator, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) # choose a MAZ for each DEST_TAZ choice, choice probability based on MAZ size_term fraction of TAZ total maz_sample = choose_MAZ_for_TAZ(taz_sample, size_term_matrix, trips, network_los, alt_dest_col_name, trace_label) @@ -426,8 +418,7 @@ def trip_destination_sample( skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label, - choose_individual_max_utility): + trace_label): """ Returns @@ -471,8 +462,7 @@ def trip_destination_sample( network_los, estimator, chunk_size, trace_hh_id, - trace_label, - choose_individual_max_utility) + trace_label) else: choices = destination_sample( @@ -484,8 +474,7 @@ def trip_destination_sample( skim_hotel, estimator, chunk_size, - trace_label, - choose_individual_max_utility) + trace_label) return choices @@ -664,8 +653,7 @@ def trip_destination_simulate( skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label, - choose_individual_max_utility): + trace_label): """ Chose destination from destination_sample (with od_logsum and dp_logsum columns added) @@ -744,8 +732,7 @@ def choose_trip_destination( size_term_matrix, skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label, - choose_individual_max_utility): + trace_label): logger.info("choose_trip_destination %s with %d trips", trace_label, trips.shape[0]) @@ -761,8 +748,7 @@ def choose_trip_destination( skim_hotel=skim_hotel, estimator=estimator, chunk_size=chunk_size, trace_hh_id=trace_hh_id, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) dropped_trips = ~trips.index.isin(destination_sample.index.unique()) if dropped_trips.any(): @@ -800,8 +786,7 @@ def choose_trip_destination( skim_hotel=skim_hotel, estimator=estimator, chunk_size=chunk_size, trace_hh_id=trace_hh_id, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) dropped_trips = ~trips.index.isin(destinations.index) if dropped_trips.any(): @@ -1054,8 +1039,7 @@ def run_trip_destination( size_term_matrix, skim_hotel, estimator, chunk_size, trace_hh_id, - trace_label=tracing.extend_trace_label(nth_trace_label, primary_purpose), - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False)) + trace_label=tracing.extend_trace_label(nth_trace_label, primary_purpose)) choices_list.append(choices) if want_sample_table: From 0387c356e604aa75c388a8156bb3b3f9704159cf Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:27:33 +1000 Subject: [PATCH 066/135] remove usage of choose_individual_max_utility --- activitysim/abm/models/trip_purpose.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/trip_purpose.py b/activitysim/abm/models/trip_purpose.py index ab4dc28007..f0787ccf55 100644 --- a/activitysim/abm/models/trip_purpose.py +++ b/activitysim/abm/models/trip_purpose.py @@ -106,7 +106,9 @@ def choose_intermediate_trip_purpose( print(choosers[probs_cols]) estimator.write_table(choosers[probs_cols], 'probs', append=True) - choices, rands = logit.make_choices(choosers[purpose_cols], trace_label=trace_label, trace_choosers=choosers) + choices, rands = logit.make_choices( + choosers[purpose_cols], + trace_label=trace_label, trace_choosers=choosers) if have_trace_targets: tracing.trace_df(choices, '%s.choices' % trace_label, columns=[None, 'trip_purpose']) From 8ac91f19a056c478c7c380423ecffd342c17bc76 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:28:39 +1000 Subject: [PATCH 067/135] remove usage of choose_individual_max_utility --- activitysim/abm/models/trip_destination.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index 1e0462efe4..8988bda74a 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -109,7 +109,8 @@ def _destination_sample( skims=skims, locals_d=locals_dict, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label) + trace_label=trace_label + ) return choices @@ -698,8 +699,7 @@ def trip_destination_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name='trip_dest', - estimator=estimator - ) + estimator=estimator) if not want_logsums: # for consistency, always return a dataframe with canonical column name From eea84ca079c1acef4a521372717816b42c7ccba7 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:31:14 +1000 Subject: [PATCH 068/135] remove usage of choose_individual_max_utility --- activitysim/abm/models/util/mode.py | 3 +- .../abm/models/util/tour_destination.py | 41 ++++++------------- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/activitysim/abm/models/util/mode.py b/activitysim/abm/models/util/mode.py index 41937ff0e3..37afe6d47d 100644 --- a/activitysim/abm/models/util/mode.py +++ b/activitysim/abm/models/util/mode.py @@ -144,7 +144,6 @@ def run_tour_mode_choice_simulate( trace_label=trace_label, trace_choice_name=trace_choice_name, trace_column_names=trace_column_names, - estimator=estimator - ) + estimator=estimator) return choices diff --git a/activitysim/abm/models/util/tour_destination.py b/activitysim/abm/models/util/tour_destination.py index 72212aff9d..ab21de7352 100644 --- a/activitysim/abm/models/util/tour_destination.py +++ b/activitysim/abm/models/util/tour_destination.py @@ -77,8 +77,7 @@ def _destination_sample( alt_dest_col_name, chunk_size, chunk_tag, - trace_label, - choose_individual_max_utility): + trace_label): model_spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC', segment_name=spec_segment_name, estimator=estimator) @@ -111,8 +110,7 @@ def _destination_sample( locals_d=locals_d, chunk_size=chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) # remember person_id in chosen alts so we can merge with persons in subsequent steps # (broadcasts person_id onto all alternatives sharing the same tour_id index value) @@ -128,8 +126,7 @@ def destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility): + chunk_size, trace_label): chunk_tag = 'tour_destination.sample' @@ -156,8 +153,7 @@ def destination_sample( model_settings, alt_dest_col_name, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=trace_label) return choices @@ -397,8 +393,7 @@ def destination_presample( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility): + chunk_size, trace_label): trace_label = tracing.extend_trace_label(trace_label, 'presample') chunk_tag = 'tour_destination.presample' @@ -430,9 +425,7 @@ def destination_presample( model_settings, DEST_TAZ, chunk_size, chunk_tag=chunk_tag, - trace_label=trace_label, - choose_individual_max_utility=choose_individual_max_utility - ) + trace_label=trace_label) # choose a MAZ for each DEST_TAZ choice, choice probability based on MAZ size_term fraction of TAZ total maz_choices = choose_MAZ_for_TAZ(taz_sample, MAZ_size_terms, trace_label) @@ -451,8 +444,7 @@ def run_destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility): + chunk_size, trace_label): # FIXME - MEMORY HACK - only include columns actually used in spec (omit them pre-merge) chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS'] @@ -483,9 +475,7 @@ def run_destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility - ) + chunk_size, trace_label) else: choices = destination_sample( @@ -495,9 +485,7 @@ def run_destination_sample( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility - ) + chunk_size, trace_label) # remember person_id in chosen alts so we can merge with persons in subsequent steps # (broadcasts person_id onto all alternatives sharing the same tour_id index value) @@ -578,8 +566,7 @@ def run_destination_simulate( network_los, destination_size_terms, estimator, - chunk_size, trace_label, - choose_individual_max_utility): + chunk_size, trace_label): """ run destination_simulate on tour_destination_sample annotated with mode_choice logsum to select a destination from sample alternatives @@ -706,9 +693,7 @@ def run_tour_destination( segment_destination_size_terms, estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(segment_trace_label, 'sample'), - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) - ) + trace_label=tracing.extend_trace_label(segment_trace_label, 'sample')) # - destination_logsums tour_purpose = segment_name # tour_purpose is segment_name @@ -736,9 +721,7 @@ def run_tour_destination( destination_size_terms=segment_destination_size_terms, estimator=estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(segment_trace_label, 'simulate'), - choose_individual_max_utility=config.setting("freeze_unobserved_utilities", False) - ) + trace_label=tracing.extend_trace_label(segment_trace_label, 'simulate')) choices_list.append(choices) From cb807d13edf11ca199f864f8a6481eabaa336862 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 16:35:49 +1000 Subject: [PATCH 069/135] remove usage of choose_individual_max_utility --- activitysim/abm/models/trip_destination.py | 2 +- activitysim/abm/models/util/tour_destination.py | 3 +-- activitysim/abm/models/util/tour_od.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/activitysim/abm/models/trip_destination.py b/activitysim/abm/models/trip_destination.py index 8988bda74a..e4dbed4444 100644 --- a/activitysim/abm/models/trip_destination.py +++ b/activitysim/abm/models/trip_destination.py @@ -110,7 +110,7 @@ def _destination_sample( locals_d=locals_dict, chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label - ) + ) return choices diff --git a/activitysim/abm/models/util/tour_destination.py b/activitysim/abm/models/util/tour_destination.py index ab21de7352..7482aeb6db 100644 --- a/activitysim/abm/models/util/tour_destination.py +++ b/activitysim/abm/models/util/tour_destination.py @@ -632,8 +632,7 @@ def run_destination_simulate( chunk_size=chunk_size, chunk_tag=chunk_tag, trace_label=trace_label, trace_choice_name='destination', - estimator=estimator - ) + estimator=estimator) if not want_logsums: # for consistency, always return a dataframe with canonical column name diff --git a/activitysim/abm/models/util/tour_od.py b/activitysim/abm/models/util/tour_od.py index bd017c4cae..0db7e01d74 100644 --- a/activitysim/abm/models/util/tour_od.py +++ b/activitysim/abm/models/util/tour_od.py @@ -853,8 +853,7 @@ def run_od_simulate( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='origin_destination', - estimator=estimator - ) + estimator=estimator) if not want_logsums: choices = choices.to_frame('choice') @@ -922,7 +921,8 @@ def run_tour_od( segment_destination_size_terms, estimator, chunk_size=chunk_size, - trace_label=tracing.extend_trace_label(trace_label, 'sample.%s' % segment_name)) + trace_label=tracing.extend_trace_label( + trace_label, 'sample.%s' % segment_name)) if model_settings['ORIG_FILTER'] == 'original_MAZ > 0': pass From f0cc563bdb93bd5ca42cf5005c12474a9f952276 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 17:16:05 +1000 Subject: [PATCH 070/135] interaction_sample util based w/p probs --- activitysim/core/interaction_sample.py | 320 +++++++++++++------------ 1 file changed, 167 insertions(+), 153 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 4278b962c7..0b7fb99aea 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -11,6 +11,7 @@ from . import logit from . import tracing from . import chunk +from . import config from .simulate import set_skim_wrapper_targets from .logit import inverse_ev1_cdf @@ -23,14 +24,54 @@ DUMP = False +def make_sample_choices_utility_based( + choosers, utilities, + alternatives, + sample_size, alternative_count, alt_col_name, + allow_zero_probs, + trace_label): + + assert isinstance(utilities, pd.DataFrame) + assert utilities.shape == (len(choosers), alternative_count) + choice_dimension = (len(choosers), alternative_count, sample_size) + + rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count*sample_size) + rands = rands.reshape(choice_dimension) + rands = inverse_ev1_cdf(rands) + chunk.log_df(trace_label, 'rands', rands) + + utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented + utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) + utilities += rands + + del rands + chunk.log_df(trace_label, 'rands', None) + + # this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative + choices_array = np.argmax(utilities, axis=1) + + choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) + choices_flattened = choices_array.flatten(order='F') + + # choices_flattened are 0-based index into alternatives, need to map to alternative values given by + # alternatives.index.values (they are in this order by construction) + # explode to one row per chooser.index, alt_zone_id + choices_df = pd.DataFrame({ + alt_col_name: alternatives.index.values[choices_flattened], + 'rand': np.zeros_like(choosers_index_rep), # TODO [janzill June2022]: zero out for now + #'prob': probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F'), + # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction + choosers.index.name: np.tile(choosers.index.values, sample_size) + }) + return choices_df + + def make_sample_choices( choosers, probs, alternatives, sample_size, alternative_count, alt_col_name, allow_zero_probs, - trace_label, - utilities=None, - choose_individual_max_utility=False): + trace_label): """ Parameters @@ -50,7 +91,6 @@ def make_sample_choices( ------- """ - assert isinstance(probs, pd.DataFrame) assert probs.shape == (len(choosers), alternative_count) @@ -65,134 +105,96 @@ def make_sample_choices( # remove from sample probs = probs[~zero_probs] choosers = choosers[~zero_probs] - # TODO [janzill Jun2022]: do we want this for consistency? - # might need this in other places too? - if utilities is not None: - utilities = utilities[~zero_probs] - - if choose_individual_max_utility: - assert isinstance(utilities, pd.DataFrame) - assert utilities.shape == (len(choosers), alternative_count) - choice_dimension = (len(choosers), alternative_count, sample_size) - - rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count*sample_size) - rands = rands.reshape(choice_dimension) - rands = inverse_ev1_cdf(rands) - chunk.log_df(trace_label, 'rands', rands) - - utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented - utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) - utilities += rands - - del rands - chunk.log_df(trace_label, 'rands', None) - - # this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative - choices_array = np.argmax(utilities, axis=1) - - choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) - choices_flattened = choices_array.flatten(order='F') - - # choices_flattened are 0-based index into alternatives, need to map to alternative values given by - # alternatives.index.values (they are in this order by construction) - # explode to one row per chooser.index, alt_zone_id - choices_df = pd.DataFrame({ - alt_col_name: alternatives.index.values[choices_flattened], - 'rand': np.zeros_like(choosers_index_rep), # TODO [janzill June2022]: zero out for now - 'prob': probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F'), - # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction - choosers.index.name: np.tile(choosers.index.values, sample_size) - }) - else: - cum_probs_array = probs.values.cumsum(axis=1) - chunk.log_df(trace_label, 'cum_probs_array', cum_probs_array) - # alt probs in convenient layout to return prob of chose alternative - # (same layout as cum_probs_arr) - alt_probs_array = probs.values.flatten() - chunk.log_df(trace_label, 'alt_probs_array', alt_probs_array) + cum_probs_array = probs.values.cumsum(axis=1) + chunk.log_df(trace_label, 'cum_probs_array', cum_probs_array) - # get sample_size rands for each chooser - rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size) + # alt probs in convenient layout to return prob of chose alternative + # (same layout as cum_probs_arr) + alt_probs_array = probs.values.flatten() + chunk.log_df(trace_label, 'alt_probs_array', alt_probs_array) - # transform as we iterate over alternatives - # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr - # i.e rands[i] is a 2-D array of one alt choice rand for each chooser - rands = rands.T.reshape(sample_size, -1, 1) - chunk.log_df(trace_label, 'rands', rands) + # get sample_size rands for each chooser + rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size) - # the alternative value chosen - # WHY SHOULD CHOICES COL HAVE TO BE TYPE INT??? - # choices_array = np.empty([sample_size, len(choosers)]).astype(int) - choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype) - # chunk log these later after we populate them... + # transform as we iterate over alternatives + # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr + # i.e rands[i] is a 2-D array of one alt choice rand for each chooser + rands = rands.T.reshape(sample_size, -1, 1) + chunk.log_df(trace_label, 'rands', rands) - # the probability of the chosen alternative - choice_probs_array = np.empty([sample_size, len(choosers)]) - # chunk log these later after we populate them... + # the alternative value chosen + # WHY SHOULD CHOICES COL HAVE TO BE TYPE INT??? + # choices_array = np.empty([sample_size, len(choosers)]).astype(int) + choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype) + # chunk log these later after we populate them... - alts = np.tile(alternatives.index.values, len(choosers)) - chunk.log_df(trace_label, 'alts', alts) + # the probability of the chosen alternative + choice_probs_array = np.empty([sample_size, len(choosers)]) + # chunk log these later after we populate them... - # FIXME - do this all at once rather than iterate? - for i in range(sample_size): + alts = np.tile(alternatives.index.values, len(choosers)) + chunk.log_df(trace_label, 'alts', alts) - # FIXME - do this in numpy, not pandas? + # FIXME - do this all at once rather than iterate? + for i in range(sample_size): - # rands for this alt in broadcastable shape - r = rands[i] + # FIXME - do this in numpy, not pandas? - # position of first occurrence of positive value - positions = np.argmax(cum_probs_array > r, axis=1) + # rands for this alt in broadcastable shape + r = rands[i] - # FIXME - leave positions as numpy array, not pandas series? - # positions is series with the chosen alternative represented as a column index in probs - # which is an integer between zero and num alternatives in the alternative sample - positions = pd.Series(positions, index=probs.index) + # position of first occurrence of positive value + positions = np.argmax(cum_probs_array > r, axis=1) - # need to get from an integer offset into the alternative sample to the alternative index - # that is, we want the index value of the row that is offset by rows into the - # tranche of this choosers alternatives created by cross join of alternatives and choosers + # FIXME - leave positions as numpy array, not pandas series? + # positions is series with the chosen alternative represented as a column index in probs + # which is an integer between zero and num alternatives in the alternative sample + positions = pd.Series(positions, index=probs.index) - # offsets is the offset into model_design df of first row of chooser alternatives - offsets = np.arange(len(positions)) * alternative_count + # need to get from an integer offset into the alternative sample to the alternative index + # that is, we want the index value of the row that is offset by rows into the + # tranche of this choosers alternatives created by cross join of alternatives and choosers - # choices and choice_probs have one element per chooser and is in same order as choosers - choices_array[i] = np.take(alts, positions + offsets) - choice_probs_array[i] = np.take(alt_probs_array, positions + offsets) + # offsets is the offset into model_design df of first row of chooser alternatives + offsets = np.arange(len(positions)) * alternative_count - del positions - del offsets + # choices and choice_probs have one element per chooser and is in same order as choosers + choices_array[i] = np.take(alts, positions + offsets) + choice_probs_array[i] = np.take(alt_probs_array, positions + offsets) - chunk.log_df(trace_label, 'choices_array', choices_array) - chunk.log_df(trace_label, 'choice_probs_array', choice_probs_array) + del positions + del offsets - del alts - chunk.log_df(trace_label, 'alts', None) - del cum_probs_array - chunk.log_df(trace_label, 'cum_probs_array', None) - del alt_probs_array - chunk.log_df(trace_label, 'alt_probs_array', None) + chunk.log_df(trace_label, 'choices_array', choices_array) + chunk.log_df(trace_label, 'choice_probs_array', choice_probs_array) - # explode to one row per chooser.index, alt_zone_id - choices_df = pd.DataFrame( - {alt_col_name: choices_array.flatten(order='F'), - 'rand': rands.flatten(order='F'), - 'prob': choice_probs_array.flatten(order='F'), - choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size) - }) + del alts + chunk.log_df(trace_label, 'alts', None) + del cum_probs_array + chunk.log_df(trace_label, 'cum_probs_array', None) + del alt_probs_array + chunk.log_df(trace_label, 'alt_probs_array', None) - chunk.log_df(trace_label, 'choices_df', choices_df) + # explode to one row per chooser.index, alt_zone_id + choices_df = pd.DataFrame( + {alt_col_name: choices_array.flatten(order='F'), + 'rand': rands.flatten(order='F'), + 'prob': choice_probs_array.flatten(order='F'), + choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size) + }) + + chunk.log_df(trace_label, 'choices_df', choices_df) - del choices_array - chunk.log_df(trace_label, 'choices_array', None) - del rands - chunk.log_df(trace_label, 'rands', None) - del choice_probs_array - chunk.log_df(trace_label, 'choice_probs_array', None) + del choices_array + chunk.log_df(trace_label, 'choices_array', None) + del rands + chunk.log_df(trace_label, 'rands', None) + del choice_probs_array + chunk.log_df(trace_label, 'choice_probs_array', None) - # handing this off to caller - chunk.log_df(trace_label, 'choices_df', None) + # handing this off to caller + chunk.log_df(trace_label, 'choices_df', None) return choices_df @@ -204,8 +206,7 @@ def _interaction_sample( log_alt_losers=False, skims=None, locals_d=None, - trace_label=None, - choose_individual_max_utility=False): + trace_label=None): """ Run a MNL simulation in the situation in which alternatives must be merged with choosers because there are interaction terms or @@ -350,51 +351,65 @@ def _interaction_sample( tracing.dump_df(DUMP, utilities, trace_label, 'utilities') - # convert to probabilities (utilities exponentiated and normalized to probs) - # probs is same shape as utilities, one row per chooser and one column for alternative - probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs, - trace_label=trace_label, trace_choosers=choosers) - chunk.log_df(trace_label, 'probs', probs) - - #del utilities - #chunk.log_df(trace_label, 'utilities', None) - if have_trace_targets: - tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), - column_labels=['alternative', 'probability']) + # sample size 0 is for estimation mode - see below + if config.setting("freeze_unobserved_utilities", False) and (sample_size != 0): - if sample_size == 0: - # FIXME return full alternative set rather than sample - logger.info("Estimation mode for %s using unsampled alternatives" % (trace_label, )) + choices_df = make_sample_choices_utility_based( + choosers, utilities, alternatives, + sample_size, alternative_count, alt_col_name, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label) - index_name = probs.index.name - choices_df = \ - pd.melt(probs.reset_index(), id_vars=[index_name])\ - .sort_values(by=index_name, kind='mergesort')\ - .set_index(index_name)\ - .rename(columns={'value': 'prob'})\ - .drop(columns='variable') + chunk.log_df(trace_label, 'choices_df', choices_df) - choices_df['pick_count'] = 1 - choices_df.insert(0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index))) + del utilities + chunk.log_df(trace_label, 'utilities', None) - return choices_df else: - choices_df = make_sample_choices( - choosers, probs, alternatives, - sample_size, alternative_count, alt_col_name, - allow_zero_probs=allow_zero_probs, - trace_label=trace_label, - utilities=utilities, - choose_individual_max_utility=choose_individual_max_utility) + # convert to probabilities (utilities exponentiated and normalized to probs) + # probs is same shape as utilities, one row per chooser and one column for alternative + probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs, + trace_label=trace_label, trace_choosers=choosers) + chunk.log_df(trace_label, 'probs', probs) + + del utilities + chunk.log_df(trace_label, 'utilities', None) + + if have_trace_targets: + tracing.trace_df(probs, tracing.extend_trace_label(trace_label, 'probs'), + column_labels=['alternative', 'probability']) + + if sample_size == 0: + # FIXME return full alternative set rather than sample + logger.info("Estimation mode for %s using unsampled alternatives" % (trace_label, )) + + index_name = probs.index.name + choices_df = \ + pd.melt(probs.reset_index(), id_vars=[index_name])\ + .sort_values(by=index_name, kind='mergesort')\ + .set_index(index_name)\ + .rename(columns={'value': 'prob'})\ + .drop(columns='variable') + + choices_df['pick_count'] = 1 + choices_df.insert(0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index))) + + return choices_df + else: + choices_df = make_sample_choices( + choosers, probs, alternatives, + sample_size, alternative_count, alt_col_name, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label) - chunk.log_df(trace_label, 'choices_df', choices_df) + chunk.log_df(trace_label, 'choices_df', choices_df) - del utilities - chunk.log_df(trace_label, 'utilities', None) + # - NARROW + choices_df['prob'] = choices_df['prob'].astype(np.float32) - del probs - chunk.log_df(trace_label, 'probs', None) + del probs + chunk.log_df(trace_label, 'probs', None) # pick_count and pick_dup # pick_count is number of duplicate picks @@ -429,7 +444,7 @@ def _interaction_sample( chunk.log_df(trace_label, 'choices_df', choices_df) # - NARROW - choices_df['prob'] = choices_df['prob'].astype(np.float32) + # choices_df['prob'] = choices_df['prob'].astype(np.float32) assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty) choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32) @@ -442,7 +457,7 @@ def interaction_sample( allow_zero_probs=False, log_alt_losers=False, skims=None, locals_d=None, chunk_size=0, chunk_tag=None, - trace_label=None, choose_individual_max_utility=False): + trace_label=None): """ Run a simulation in the situation in which alternatives must @@ -523,8 +538,7 @@ def interaction_sample( log_alt_losers=log_alt_losers, skims=skims, locals_d=locals_d, - trace_label=chunk_trace_label, - choose_individual_max_utility=choose_individual_max_utility) + trace_label=chunk_trace_label) if choices.shape[0] > 0: # might not be any if allow_zero_probs From 417e90feb6b95eda77338334455299ead1ebc8e3 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 17:17:44 +1000 Subject: [PATCH 071/135] wrong method in cdap --- activitysim/abm/models/util/cdap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index d0bf565931..3cc2bc862e 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -637,7 +637,7 @@ def household_activity_choices(indiv_utils, interaction_coefficients, hhsize, return pd.Series(dtype='float64') if config.setting("freeze_unobserved_utilities", False): - idx_choices, rands = logit.make_choices(utils, trace_label=trace_label) + idx_choices, rands = logit.make_choices_utility_based(utils, trace_label=trace_label) else: probs = logit.utils_to_probs(utils, trace_label=trace_label) From 4f825ae9c4cbb6980e7c8632030fe9a786554780 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 17:22:29 +1000 Subject: [PATCH 072/135] style fixes --- activitysim/core/logit.py | 2 +- activitysim/core/simulate.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 1fe6065fd4..1f884fce8f 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -436,7 +436,7 @@ def __init__(self, name=None, level=0): self.coefficient = 0 def print(self): - print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s " % + print("Nest name: %s level: %s coefficient: %s product_of_coefficients: %s ancestors: %s" % (self.name, self.level, self.coefficient, self.product_of_coefficients, self.ancestors)) @property diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 5ce21c359d..10f27dcf7d 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -922,8 +922,6 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, This is the column label to be used in trace file csv dump of choices trace_column_names: str or list of str chooser columns to include when tracing expression_values - choose_individual_max_utility: bool - apply frozen randomness at the individual utility level Returns ------- @@ -994,8 +992,7 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, log_alt_losers=False, want_logsums=False, trace_label=None, - trace_choice_name=None, trace_column_names=None, - choose_individual_max_utility=False): + trace_choice_name=None, trace_column_names=None): """ Run a nested-logit simulation for when the model spec does not involve alternative specific data, e.g. there are no interactions with alternative @@ -1052,7 +1049,6 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label, column_labels=['alternative', 'utility']) - if config.setting("freeze_unobserved_utilities", False): if custom_chooser: # TODO JAN: ADD HERE - need to figure out what to do with custom chooser though. leave out for now, From 7a6cc29ee70a06aa752e490418401ab6e895a285 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 17:38:34 +1000 Subject: [PATCH 073/135] need probs in interaction_sample - try it the quick way --- activitysim/core/interaction_sample.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 0b7fb99aea..a85d8113ca 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -25,7 +25,7 @@ def make_sample_choices_utility_based( - choosers, utilities, + choosers, utilities, probs, alternatives, sample_size, alternative_count, alt_col_name, allow_zero_probs, @@ -59,7 +59,7 @@ def make_sample_choices_utility_based( choices_df = pd.DataFrame({ alt_col_name: alternatives.index.values[choices_flattened], 'rand': np.zeros_like(choosers_index_rep), # TODO [janzill June2022]: zero out for now - #'prob': probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F'), + 'prob': probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F'), # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction choosers.index.name: np.tile(choosers.index.values, sample_size) }) @@ -355,14 +355,23 @@ def _interaction_sample( # sample size 0 is for estimation mode - see below if config.setting("freeze_unobserved_utilities", False) and (sample_size != 0): + # TODO: calc probs afterwards and merge to keep memory usage down? + probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs, + trace_label=trace_label, trace_choosers=choosers) + probs = probs.astype(np.float32) + chunk.log_df(trace_label, 'probs', probs) + choices_df = make_sample_choices_utility_based( - choosers, utilities, alternatives, + choosers, utilities, probs, + alternatives, sample_size, alternative_count, alt_col_name, allow_zero_probs=allow_zero_probs, trace_label=trace_label) chunk.log_df(trace_label, 'choices_df', choices_df) + # Now we need to attach probs - memory preserving to do this after making choices + del utilities chunk.log_df(trace_label, 'utilities', None) @@ -405,9 +414,6 @@ def _interaction_sample( chunk.log_df(trace_label, 'choices_df', choices_df) - # - NARROW - choices_df['prob'] = choices_df['prob'].astype(np.float32) - del probs chunk.log_df(trace_label, 'probs', None) @@ -444,7 +450,7 @@ def _interaction_sample( chunk.log_df(trace_label, 'choices_df', choices_df) # - NARROW - # choices_df['prob'] = choices_df['prob'].astype(np.float32) + choices_df['prob'] = choices_df['prob'].astype(np.float32) assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty) choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32) From f0143e99f3bebbe6f29e6e5a07eb37f4df1eb497 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 17:54:36 +1000 Subject: [PATCH 074/135] remove false comparison --- activitysim/core/interaction_sample_simulate.py | 12 ++++-------- activitysim/core/logit.py | 1 - 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index ad81696213..0ef9506eaf 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -199,6 +199,10 @@ def _interaction_sample_simulate( index=choosers.index) chunk.log_df(trace_label, 'utilities_df', utilities_df) + if want_logsums: + logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) + chunk.log_df(trace_label, 'logsums', logsums) + del padded_utilities chunk.log_df(trace_label, 'padded_utilities', None) @@ -213,10 +217,6 @@ def _interaction_sample_simulate( utilities_df, trace_label=trace_label, trace_choosers=choosers ) - if want_logsums: - logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) - chunk.log_df(trace_label, 'logsums', logsums) - del utilities_df chunk.log_df(trace_label, 'utilities_df', None) @@ -227,10 +227,6 @@ def _interaction_sample_simulate( trace_label=trace_label, trace_choosers=choosers) chunk.log_df(trace_label, 'probs', probs) - if want_logsums: - logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) - chunk.log_df(trace_label, 'logsums', logsums) - del utilities_df chunk.log_df(trace_label, 'utilities_df', None) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 1f884fce8f..37c70da6a3 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -265,7 +265,6 @@ def make_choices_ru_frozen_mnl(utilities): def make_choices_ru_frozen(utilities, alt_order_array, nest_spec=None, trace_label=None): trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen_mnl') if nest_spec is None: - assert (alt_order_array == utilities.columns.values).all() choices = make_choices_ru_frozen_mnl(utilities) else: choices = make_choices_ru_frozen_nl(utilities, alt_order_array, nest_spec) From f6d81d60d1d3035a9f993c76bd8e5af9dd81e2b5 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 17:57:00 +1000 Subject: [PATCH 075/135] bug fix --- activitysim/core/interaction_sample_simulate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 0ef9506eaf..bd63a20411 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -244,14 +244,14 @@ def _interaction_sample_simulate( # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample positions, rands = \ - logit.make_choices(probs, utilities_df, trace_label=trace_label, trace_choosers=choosers) + logit.make_choices(probs, trace_label=trace_label, trace_choosers=choosers) + + del probs + chunk.log_df(trace_label, 'probs', None) chunk.log_df(trace_label, 'positions', positions) chunk.log_df(trace_label, 'rands', rands) - del probs - chunk.log_df(trace_label, 'probs', None) - # shouldn't have chosen any of the dummy pad utilities assert positions.max() < max_sample_count From b4495932ff7122faab31b7f1c11458a2ade7c053 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 18:02:29 +1000 Subject: [PATCH 076/135] bug fix --- activitysim/core/interaction_sample_simulate.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index bd63a20411..720b2b3113 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -267,9 +267,11 @@ def _interaction_sample_simulate( chunk.log_df(trace_label, 'choices', choices) - if allow_zero_probs and zero_probs.any(): - # FIXME this is kind of gnarly, patch choice for zero_probs - choices.loc[zero_probs] = zero_prob_choice_val + # TODO [janzill Jun2022]: Also for utility based choices? + if not config.setting("freeze_unobserved_utilities", False): + if allow_zero_probs and zero_probs.any(): + # FIXME this is kind of gnarly, patch choice for zero_probs + choices.loc[zero_probs] = zero_prob_choice_val if have_trace_targets: tracing.trace_df(choices, tracing.extend_trace_label(trace_label, 'choices'), From dbc809a4ffda797f10bf8b1b377f70ff2a27a4a7 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 18:05:10 +1000 Subject: [PATCH 077/135] add utility based choice option for transit virtual pathbuilder --- activitysim/core/pathbuilder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/activitysim/core/pathbuilder.py b/activitysim/core/pathbuilder.py index d4cee77504..902ba24164 100644 --- a/activitysim/core/pathbuilder.py +++ b/activitysim/core/pathbuilder.py @@ -744,7 +744,12 @@ def build_virtual_path(self, recipe, path_type, orig, dest, tod, demographic_seg self.trace_df(probs, trace_label, 'probs') else: - choices, rands = logit.make_choices(probs, allow_bad_probs=True, trace_label=trace_label) + if config.setting("freeze_unobserved_utilities", False): + choices, rands = logit.make_choices_utility_based( + utilities_df, allow_bad_probs=True, trace_label=trace_label + ) + else: + choices, rands = logit.make_choices(probs, allow_bad_probs=True, trace_label=trace_label) chunk.log_df(trace_label, "rands", rands) del rands From 4c6a524e1eacccbc6327c1cc10f3566d6a31ac0d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 19:47:07 +1000 Subject: [PATCH 078/135] custom chooser --- .../abm/models/joint_tour_participation.py | 112 +++++++++++++++++- activitysim/core/simulate.py | 34 +++--- 2 files changed, 129 insertions(+), 17 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index c69b5093c5..1da9500ecf 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -212,6 +212,109 @@ def participants_chooser(probs, choosers, spec, trace_label): return choices, rands +def participants_chooser_utility_based(utilities, choosers, spec, trace_label): + """ + custom alternative to logit.make_choices for simulate.simple_simulate + + Choosing participants for mixed tours is trickier than adult or child tours becuase we + need at least one adult and one child participant in a mixed tour. We call logit.make_choices + and then check to see if the tour statisfies this requirement, and rechoose for any that + fail until all are satisfied. + + In principal, this shold always occur eventually, but we fail after MAX_ITERATIONS, + just in case there is some failure in program logic (haven't seen this occur.) + + Parameters + ---------- + utilities : pandas.DataFrame + Rows for choosers and columns for the alternatives from which they + are choosing. + choosers : pandas.dataframe + simple_simulate choosers df + spec : pandas.DataFrame + simple_simulate spec df + We only need spec so we can know the column index of the 'participate' alternative + indicating that the participant has been chosen to participate in the tour + trace_label : str + + Returns - same as logit.make_choices + ------- + choices, rands + choices, rands as returned by logit.make_choices (in same order as probs) + + """ + + assert utilities.index.equals(choosers.index) + + # choice is boolean (participate or not) + model_settings = config.read_model_settings('joint_tour_participation.yaml') + + choice_col = model_settings.get('participation_choice', 'participate') + assert choice_col in spec.columns, \ + "couldn't find participation choice column '%s' in spec" + PARTICIPATE_CHOICE = spec.columns.get_loc(choice_col) + MAX_ITERATIONS = model_settings.get('max_participation_choice_iterations', 5000) + + trace_label = tracing.extend_trace_label(trace_label, 'participants_chooser') + + candidates = choosers.copy() + choices_list = [] + rands_list = [] + + num_tours_remaining = len(candidates.tour_id.unique()) + logger.info('%s %s joint tours to satisfy.', trace_label, num_tours_remaining,) + + iter = 0 + while candidates.shape[0] > 0: + + iter += 1 + + if iter > MAX_ITERATIONS: + logger.warning('%s max iterations exceeded (%s).', trace_label, MAX_ITERATIONS) + diagnostic_cols = ['tour_id', 'household_id', 'composition', 'adult'] + unsatisfied_candidates = candidates[diagnostic_cols].join(utilities) + tracing.write_csv(unsatisfied_candidates, + file_name='%s.UNSATISFIED' % trace_label, transpose=False) + print(unsatisfied_candidates.head(20)) + assert False + + choices, rands = logit.make_choices_utility_based(utilities, trace_label=trace_label, trace_choosers=choosers) + participate = (choices == PARTICIPATE_CHOICE) + + # satisfaction indexed by tour_id + tour_satisfaction = get_tour_satisfaction(candidates, participate) + num_tours_satisfied_this_iter = tour_satisfaction.sum() + + if num_tours_satisfied_this_iter > 0: + + num_tours_remaining -= num_tours_satisfied_this_iter + + satisfied = reindex(tour_satisfaction, candidates.tour_id) + + choices_list.append(choices[satisfied]) + rands_list.append(rands[satisfied]) + + # remove candidates of satisfied tours + utilities = utilities[~satisfied] + candidates = candidates[~satisfied] + + logger.debug(f"{trace_label} iteration {iter} : " + f"{num_tours_satisfied_this_iter} joint tours satisfied {num_tours_remaining} remaining") + + choices = pd.concat(choices_list) + rands = pd.concat(rands_list).reindex(choosers.index) + + # reindex choices and rands to match probs and v index + choices = choices.reindex(choosers.index) + rands = rands.reindex(choosers.index) + assert choices.index.equals(choosers.index) + assert rands.index.equals(choosers.index) + + logger.info('%s %s iterations to satisfy all joint tours.', trace_label, iter,) + + return choices, rands + + def annotate_jtp(model_settings, trace_label): # - annotate persons @@ -305,6 +408,13 @@ def joint_tour_participation( household_chunk_ids = pd.Series(range(len(unique_household_ids)), index=unique_household_ids) candidates['chunk_id'] = reindex(household_chunk_ids, candidates.household_id) + + # TODO: loads of code duplication, could make this implicit and get rid of it + if config.setting("freeze_unobserved_utilities", False): + custom_chooser = participants_chooser_utility_based + else: + custom_chooser = participants_chooser + choices = simulate.simple_simulate_by_chunk_id( choosers=candidates, spec=model_spec, @@ -313,7 +423,7 @@ def joint_tour_participation( chunk_size=chunk_size, trace_label=trace_label, trace_choice_name='participation', - custom_chooser=participants_chooser, + custom_chooser=custom_chooser, estimator=estimator) # choice is boolean (participate or not) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 10f27dcf7d..8c1c3dca4f 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -951,11 +951,9 @@ def eval_mnl(choosers, spec, locals_d, custom_chooser, estimator, if config.setting("freeze_unobserved_utilities", False): if custom_chooser: - # TODO JAN: ADD HERE - need to figure out what to do with custom chooser though. leave out for now, - # we only need to fix up joint_tour_participation for now - raise "Not implemented yet" - - choices, rands = logit.make_choices_utility_based(utilities, trace_label=trace_label) + choices, rands = custom_chooser(utilities=utilities, choosers=choosers, spec=spec, trace_label=trace_label) + else: + choices, rands = logit.make_choices_utility_based(utilities, trace_label=trace_label) del utilities chunk.log_df(trace_label, 'utilities', None) @@ -1050,11 +1048,6 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, column_labels=['alternative', 'utility']) if config.setting("freeze_unobserved_utilities", False): - if custom_chooser: - # TODO JAN: ADD HERE - need to figure out what to do with custom chooser though. leave out for now, - # we only need to fix up joit_tour_participation for now - raise "Not implemented yet" - # TODO [janzill Jun2022]: combine with nested_exp_utilities? # utilities of leaves and nests nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) @@ -1076,12 +1069,21 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, del raw_utilities chunk.log_df(trace_label, 'raw_utilities', None) - choices, rands = logit.make_choices_utility_based( - nested_utilities, - name_mapping=name_mapping, - nest_spec=nest_spec, - trace_label=trace_label - ) + if custom_chooser: + choices, rands = custom_chooser( + utilities=nested_utilities, + name_mapping=name_mapping, + choosers=choosers, + spec=spec, + trace_label=trace_label + ) + else: + choices, rands = logit.make_choices_utility_based( + nested_utilities, + name_mapping=name_mapping, + nest_spec=nest_spec, + trace_label=trace_label + ) del(nested_utilities) chunk.log_df(trace_label, "nested_utilities", None) From 2ec41b18f8e8f9d7de378ed1dc5a59f948cd046c Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 20:00:53 +1000 Subject: [PATCH 079/135] mem tracing in interaction_sample --- activitysim/core/interaction_sample.py | 41 ++++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index a85d8113ca..7610d677ab 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -40,18 +40,27 @@ def make_sample_choices_utility_based( rands = inverse_ev1_cdf(rands) chunk.log_df(trace_label, 'rands', rands) - utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented - utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) - utilities += rands + # # use rands + #utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented + #utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) + #utilities += rands + rands += np.repeat(utilities.to_numpy()[:, :, None], sample_size, axis=2) + + # this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative + choices_array = np.argmax(rands, axis=1) + chunk.log_df(trace_label, 'choices_array', choices_array) del rands chunk.log_df(trace_label, 'rands', None) - # this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative - choices_array = np.argmax(utilities, axis=1) - choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) + chunk.log_df(trace_label, 'choosers_index_rep', choosers_index_rep) + choices_flattened = choices_array.flatten(order='F') + chunk.log_df(trace_label, 'choices_flattened', choices_flattened) + + probs_selection = probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F') + chunk.log_df(trace_label, 'probs_selection', probs_selection) # choices_flattened are 0-based index into alternatives, need to map to alternative values given by # alternatives.index.values (they are in this order by construction) @@ -59,10 +68,24 @@ def make_sample_choices_utility_based( choices_df = pd.DataFrame({ alt_col_name: alternatives.index.values[choices_flattened], 'rand': np.zeros_like(choosers_index_rep), # TODO [janzill June2022]: zero out for now - 'prob': probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F'), + 'prob': probs_selection, # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction choosers.index.name: np.tile(choosers.index.values, sample_size) }) + chunk.log_df(trace_label, 'choices_df', choices_df) + + del probs_selection + chunk.log_df(trace_label, 'probs_selection', None) + del choices_array + chunk.log_df(trace_label, 'choices_array', None) + del choosers_index_rep + chunk.log_df(trace_label, 'choosers_index_rep', None) + del choices_flattened + chunk.log_df(trace_label, 'choices_flattened', None) + + # handing this off to caller + chunk.log_df(trace_label, 'choices_df', None) + return choices_df @@ -370,11 +393,11 @@ def _interaction_sample( chunk.log_df(trace_label, 'choices_df', choices_df) - # Now we need to attach probs - memory preserving to do this after making choices - del utilities chunk.log_df(trace_label, 'utilities', None) + del probs + chunk.log_df(trace_label, 'probs', None) else: # convert to probabilities (utilities exponentiated and normalized to probs) # probs is same shape as utilities, one row per chooser and one column for alternative From 47601e3bc97e043deb8a172be3c5fd6d520e855d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 20:18:40 +1000 Subject: [PATCH 080/135] bug fix --- activitysim/core/simulate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 8c1c3dca4f..5929f59e36 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1075,6 +1075,7 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, name_mapping=name_mapping, choosers=choosers, spec=spec, + nest_spec=nest_spec, trace_label=trace_label ) else: @@ -1145,7 +1146,7 @@ def eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec, trace_label=trace_label) else: - choices, rands = logit.make_choices(base_probabilities,nest_spec=nest_spec, trace_label=trace_label) + choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label) del base_probabilities chunk.log_df(trace_label, 'base_probabilities', None) From e0555d439fe2c75265d16f3d78db440130162ec9 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 22:43:13 +1000 Subject: [PATCH 081/135] interaction_sample memory saving implementation --- activitysim/core/interaction_sample.py | 185 ++++++------- .../frozen_impl_interaction_sample_dev.ipynb | 242 ++++++++++++++---- 2 files changed, 283 insertions(+), 144 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 7610d677ab..79eb15dee6 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -25,68 +25,57 @@ def make_sample_choices_utility_based( - choosers, utilities, probs, - alternatives, - sample_size, alternative_count, alt_col_name, - allow_zero_probs, - trace_label): + choosers, + utilities, + sample_size, + alternative_count, + allow_zero_probs +): assert isinstance(utilities, pd.DataFrame) assert utilities.shape == (len(choosers), alternative_count) - choice_dimension = (len(choosers), alternative_count, sample_size) - - rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count*sample_size) - rands = rands.reshape(choice_dimension) - rands = inverse_ev1_cdf(rands) - chunk.log_df(trace_label, 'rands', rands) - - # # use rands - #utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented - #utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) - #utilities += rands - rands += np.repeat(utilities.to_numpy()[:, :, None], sample_size, axis=2) - - # this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative - choices_array = np.argmax(rands, axis=1) - chunk.log_df(trace_label, 'choices_array', choices_array) - - del rands - chunk.log_df(trace_label, 'rands', None) - - choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) - chunk.log_df(trace_label, 'choosers_index_rep', choosers_index_rep) - - choices_flattened = choices_array.flatten(order='F') - chunk.log_df(trace_label, 'choices_flattened', choices_flattened) + #choice_dimension = (len(choosers), alternative_count, sample_size) - probs_selection = probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F') - chunk.log_df(trace_label, 'probs_selection', probs_selection) - - # choices_flattened are 0-based index into alternatives, need to map to alternative values given by - # alternatives.index.values (they are in this order by construction) - # explode to one row per chooser.index, alt_zone_id - choices_df = pd.DataFrame({ - alt_col_name: alternatives.index.values[choices_flattened], - 'rand': np.zeros_like(choosers_index_rep), # TODO [janzill June2022]: zero out for now - 'prob': probs_selection, - # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction - choosers.index.name: np.tile(choosers.index.values, sample_size) - }) - chunk.log_df(trace_label, 'choices_df', choices_df) - - del probs_selection - chunk.log_df(trace_label, 'probs_selection', None) - del choices_array - chunk.log_df(trace_label, 'choices_array', None) - del choosers_index_rep - chunk.log_df(trace_label, 'choosers_index_rep', None) - del choices_flattened - chunk.log_df(trace_label, 'choices_flattened', None) - - # handing this off to caller - chunk.log_df(trace_label, 'choices_df', None) + # TODO [janzill Jun2022]: THIS NEEDS for loop for memory like previous method, an array of dimension + # (len(choosers), alternative_count, sample_size) can get very large + choices = np.zeros_like(utilities, dtype=np.uint32) + zero_dim_index = np.arange(utilities.shape[0]) - return choices_df + for i in range(sample_size): + rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) # * sample_size + #rands = rands.reshape(choice_dimension) + rands = inverse_ev1_cdf(rands) + #chunk.log_df(trace_label, 'rands', rands) + + ## # use rands + ##utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented + ##utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) + ##utilities += rands + #rands += np.repeat(utilities.to_numpy()[:, :, None], sample_size, axis=2) + rands += utilities + + # NOT ANYMORE: this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative + # NOW: len(choosers) dimensional array + #choices_array = np.argmax(rands, axis=1) + #chunk.log_df(trace_label, 'choices_array', choices_array) + + choices[zero_dim_index, np.argmax(rands, axis=1)] += 1 + + #choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) + #choices_flattened = choices_array.flatten(order='F') + #probs_selection = probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F') + + # # choices_flattened are 0-based index into alternatives, need to map to alternative values given by + # # alternatives.index.values (they are in this order by construction) + # # explode to one row per chooser.index, alt_zone_id + # choices_df = pd.DataFrame({ + # alt_col_name: alternatives.index.values[choices_flattened], + # 'rand': np.zeros_like(choosers_index_rep), # zero out for now + # 'prob': probs_selection, + # # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction + # choosers.index.name: np.tile(choosers.index.values, sample_size) + # }) + return choices def make_sample_choices( @@ -374,28 +363,41 @@ def _interaction_sample( tracing.dump_df(DUMP, utilities, trace_label, 'utilities') - # sample size 0 is for estimation mode - see below if config.setting("freeze_unobserved_utilities", False) and (sample_size != 0): - # TODO: calc probs afterwards and merge to keep memory usage down? + choices_array = make_sample_choices_utility_based( + choosers, utilities, sample_size, alternative_count, allow_zero_probs + ) + chunk.log_df(trace_label, 'choices_array', choices_array) + + del utilities + chunk.log_df(trace_label, 'utilities', None) + + # choices array has same dim as utilities, with values indicating number of counts per chooser and alternative + # let's turn the nonzero values into a dataframe + i, j = np.non_zero(choices_array) + chunk.log_df(trace_label, 'i', i) + chunk.log_df(trace_label, 'j', j) + probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs, trace_label=trace_label, trace_choosers=choosers) - probs = probs.astype(np.float32) chunk.log_df(trace_label, 'probs', probs) - choices_df = make_sample_choices_utility_based( - choosers, utilities, probs, - alternatives, - sample_size, alternative_count, alt_col_name, - allow_zero_probs=allow_zero_probs, - trace_label=trace_label) - + choices_df = pd.DataFrame({ + alt_col_name: alternatives.index.values[j], + "pick_count": choices_array[i, j], + "prob": probs[i, j], + choosers.index.name: choosers.index.values[i] + }) chunk.log_df(trace_label, 'choices_df', choices_df) - del utilities - chunk.log_df(trace_label, 'utilities', None) - + del choices_array + chunk.log_df(trace_label, 'choices_array', None) + del i + chunk.log_df(trace_label, 'i', None) + del j + chunk.log_df(trace_label, 'j', None) del probs chunk.log_df(trace_label, 'probs', None) else: @@ -440,25 +442,27 @@ def _interaction_sample( del probs chunk.log_df(trace_label, 'probs', None) - # pick_count and pick_dup - # pick_count is number of duplicate picks - # pick_dup flag is True for all but first of duplicates - pick_group = choices_df.groupby([choosers.index.name, alt_col_name]) - - # number each item in each group from 0 to the length of that group - 1. - choices_df['pick_count'] = pick_group.cumcount(ascending=True) - # flag duplicate rows after first - choices_df['pick_dup'] = choices_df['pick_count'] > 0 - # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge) - choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1 - - # drop the duplicates - choices_df = choices_df[~choices_df['pick_dup']] - del choices_df['pick_dup'] - chunk.log_df(trace_label, 'choices_df', choices_df) + # pick_count and pick_dup + # pick_count is number of duplicate picks + # pick_dup flag is True for all but first of duplicates + pick_group = choices_df.groupby([choosers.index.name, alt_col_name]) + + # number each item in each group from 0 to the length of that group - 1. + choices_df['pick_count'] = pick_group.cumcount(ascending=True) + # flag duplicate rows after first + choices_df['pick_dup'] = choices_df['pick_count'] > 0 + # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge) + choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1 + + # drop the duplicates + choices_df = choices_df[~choices_df['pick_dup']] + del choices_df['pick_dup'] - # set index after groupby so we can trace on it - choices_df.set_index(choosers.index.name, inplace=True) + # set index after groupby so we can trace on it + choices_df.set_index(choosers.index.name, inplace=True) + + + chunk.log_df(trace_label, 'choices_df', choices_df) tracing.dump_df(DUMP, choices_df, trace_label, 'choices_df') @@ -469,8 +473,9 @@ def _interaction_sample( column_labels=['sample_alt', 'alternative']) # don't need this after tracing - del choices_df['rand'] - chunk.log_df(trace_label, 'choices_df', choices_df) + if not config.setting("freeze_unobserved_utilities", False): + del choices_df['rand'] + chunk.log_df(trace_label, 'choices_df', choices_df) # - NARROW choices_df['prob'] = choices_df['prob'].astype(np.float32) diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index 17c559e364..56274754bd 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -1,17 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", - "metadata": {}, - "source": [ - "# validate results\n", - "\n", - "## TODO\n", - "what happened to tracing when I fixed probability calcs - it seems like all trip ids are attached when I add one by\n", - "hand below - why?" - ] - }, { "cell_type": "code", "execution_count": 1, @@ -98,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 5, "outputs": [ { "name": "stdout", @@ -114,7 +102,7 @@ "data": { "text/plain": "array([2, 2, 1, 2, 1, 0])" }, - "execution_count": 112, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -271,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "execution": { @@ -286,12 +274,12 @@ "outputs": [], "source": [ "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_example_mtc\")" + "example_dir = os.path.join(root_dir, \"test_example_mtc_frozen_rand\")" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "execution": { @@ -310,7 +298,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "execution": { @@ -326,7 +314,7 @@ "source": [ "parser = argparse.ArgumentParser()\n", "run.add_run_args(parser)\n", - "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", + "args = parser.parse_args(['-c', 'configs', '-o', 'output_without_before', '-d', 'data'])\n", "#run.run(args) # 2mins full example run\n", "if not inject.is_injectable('preload_injectables'):\n", " from activitysim import abm # register abm steps and other abm-specific injectables\n", @@ -335,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 9, "outputs": [], "source": [ "from activitysim.core import inject\n", @@ -358,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": 234, + "execution_count": 10, "outputs": [], "source": [ "from activitysim.core.logit import inverse_ev1_cdf\n", @@ -503,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": 237, + "execution_count": 11, "outputs": [ { "name": "stderr", @@ -541,8 +529,8 @@ "running gradeschool, 1\n", "dropping 0 of 25 rows where size_term is zero\n", "Running school_location with 17 persons\n", - "(17, 10)\n", - "(170,)\n", + "(17, 25)\n", + "(425,)\n", "(17, 25)\n", " 0 1 2 3 4 5 \\\n", "person_id \n", @@ -573,8 +561,8 @@ "386008 0.025936 \n", "418442 0.028217 \n", "595686 0.040062 \n", - "CPU times: user 281 ms, sys: 31.2 ms, total: 312 ms\n", - "Wall time: 334 ms\n" + "CPU times: user 1.39 s, sys: 328 ms, total: 1.72 s\n", + "Wall time: 2.15 s\n" ] } ], @@ -799,20 +787,50 @@ }, { "cell_type": "code", - "execution_count": 238, + "execution_count": 12, "outputs": [ { "data": { - "text/plain": " alt_dest rand prob person_id\n0 9 0 0.336708 386008\n51 7 0 0.056174 386008\n85 9 0 0.336708 386008\n102 9 0 0.336708 386008\n34 8 0 0.157982 386008\n.. ... ... ... ...\n50 16 0 0.182639 4171620\n33 6 0 0.032124 4171620\n16 9 0 0.111409 4171620\n152 10 0 0.097048 4171620\n169 7 0 0.062739 4171620\n\n[170 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
0900.336708386008
51700.056174386008
85900.336708386008
102900.336708386008
34800.157982386008
...............
501600.1826394171620
33600.0321244171620
16900.1114094171620
1521000.0970484171620
169700.0627394171620
\n

170 rows × 4 columns

\n
" + "text/plain": " alt_dest rand prob person_id\n0 10 0 0.205884 386008\n1 10 0 0.288773 418442\n2 21 0 0.147571 595686\n3 3 0 0.005248 644292\n4 8 0 0.117232 644478\n.. ... ... ... ...\n420 25 0 0.265744 2566702\n421 10 0 0.288773 2877287\n422 9 0 0.336708 3596365\n423 6 0 0.032124 3891104\n424 10 0 0.097048 4171620\n\n[425 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n

425 rows × 4 columns

\n
" }, - "execution_count": 238, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "temp_choices.sort_values(by=[\"person_id\"])" + "temp_choices #.sort_values(by=[\"person_id\"])" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 28, + "outputs": [], + "source": [ + "test_df = temp_choices.copy()\n", + "# pick_count and pick_dup\n", + "# pick_count is number of duplicate picks\n", + "# pick_dup flag is True for all but first of duplicates\n", + "pick_group = test_df.groupby([choosers.index.name, alt_col_name])\n", + "# number each item in each group from 0 to the length of that group - 1.\n", + "test_df['pick_count'] = pick_group.cumcount(ascending=True)\n", + "# flag duplicate rows after first\n", + "test_df['pick_dup'] = test_df['pick_count'] > 0\n", + "# add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", + "test_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", + "# drop the duplicates\n", + "test_df = test_df[~test_df['pick_dup']]\n", + "del test_df['pick_dup']\n", + "# set index after groupby so we can trace on it\n", + "test_df.set_index(choosers.index.name, inplace=True)\n", + "del test_df['rand']" ], "metadata": { "collapsed": false, @@ -823,20 +841,20 @@ }, { "cell_type": "code", - "execution_count": 216, + "execution_count": 29, "outputs": [ { "data": { - "text/plain": " alt_dest rand prob person_id\n0 8 0.222922 0.157982 386008\n1 9 0.404176 0.336708 386008\n2 8 0.223154 0.157982 386008\n3 9 0.457402 0.336708 386008\n4 11 0.812506 0.069858 386008\n.. ... ... ... ...\n165 7 0.107363 0.062739 4171620\n166 11 0.517573 0.070848 4171620\n167 10 0.375828 0.097048 4171620\n168 9 0.279810 0.111409 4171620\n169 20 0.848968 0.026642 4171620\n\n[170 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
080.2229220.157982386008
190.4041760.336708386008
280.2231540.157982386008
390.4574020.336708386008
4110.8125060.069858386008
...............
16570.1073630.0627394171620
166110.5175730.0708484171620
167100.3758280.0970484171620
16890.2798100.1114094171620
169200.8489680.0266424171620
\n

170 rows × 4 columns

\n
" + "text/plain": " alt_dest prob pick_count\nperson_id \n386008 10 0.205884 4\n418442 10 0.288773 9\n595686 21 0.147571 4\n644292 3 0.005248 1\n644478 8 0.117232 4\n... ... ... ...\n644478 6 0.032124 1\n2458502 25 0.032621 1\n2458503 11 0.078812 1\n2566700 16 0.036706 1\n2566701 18 0.014321 1\n\n[165 rows x 3 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
386008100.2058844
418442100.2887739
595686210.1475714
64429230.0052481
64447880.1172324
............
64447860.0321241
2458502250.0326211
2458503110.0788121
2566700160.0367061
2566701180.0143211
\n

165 rows × 3 columns

\n
" }, - "execution_count": 216, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "#temp_choices" + "test_df" ], "metadata": { "collapsed": false, @@ -847,10 +865,22 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], + "execution_count": 39, + "outputs": [ + { + "data": { + "text/plain": " alt_dest prob pick_count\nperson_id \n3596365 9 0.336708 13\n386008 9 0.336708 12\n2877287 10 0.288773 10\n1958678 9 0.336708 10\n2458502 8 0.301414 9\n... ... ... ...\n2458503 11 0.078812 1\n644292 3 0.005248 1\n2458502 10 0.103895 1\n3891104 10 0.097048 1\n2566702 11 0.061290 1\n\n[165 rows x 3 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
359636590.33670813
38600890.33670812
2877287100.28877310
195867890.33670810
245850280.3014149
............
2458503110.0788121
64429230.0052481
2458502100.1038951
3891104100.0970481
2566702110.0612901
\n

165 rows × 3 columns

\n
" + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "choices_df" + "test_df = temp_choices.copy()\n", + "del test_df['rand']\n", + "test_df.value_counts().to_frame(\"pick_count\").reset_index([\"prob\", \"alt_dest\"])" ], "metadata": { "collapsed": false, @@ -861,20 +891,20 @@ }, { "cell_type": "code", - "execution_count": 217, + "execution_count": 31, "outputs": [ { "data": { - "text/plain": " alt_dest prob pick_count\nperson_id \n386008 8 0.157982 2\n386008 9 0.336708 5\n386008 10 0.205884 1\n386008 11 0.069858 1\n386008 17 0.009346 1\n... ... ... ...\n4171620 9 0.111409 1\n4171620 10 0.097048 1\n4171620 11 0.070848 1\n4171620 20 0.026642 1\n4171620 25 0.062053 1\n\n[105 rows x 3 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
38600880.1579822
38600890.3367085
386008100.2058841
386008110.0698581
386008170.0093461
............
417162090.1114091
4171620100.0970481
4171620110.0708481
4171620200.0266421
4171620250.0620531
\n

105 rows × 3 columns

\n
" + "text/plain": " alt_dest rand prob person_id\n0 10 0 0.205884 386008\n1 10 0 0.288773 418442\n2 21 0 0.147571 595686\n3 3 0 0.005248 644292\n4 8 0 0.117232 644478\n.. ... ... ... ...\n420 25 0 0.265744 2566702\n421 10 0 0.288773 2877287\n422 9 0 0.336708 3596365\n423 6 0 0.032124 3891104\n424 10 0 0.097048 4171620\n\n[425 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n

425 rows × 4 columns

\n
" }, - "execution_count": 217, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "#choices_df" + "test_df" ], "metadata": { "collapsed": false, @@ -885,19 +915,20 @@ }, { "cell_type": "code", - "execution_count": 231, + "execution_count": 40, "outputs": [ { "data": { - "text/plain": "array([1, 2, 3, 4, 5])" + "text/plain": " 0 1 2 3 4 5 \\\nperson_id \n386008 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n418442 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n595686 0.363442 1.313932 2.021662 1.599651 3.201645 3.726050 \n644292 0.426142 1.419508 2.267947 1.796678 3.710634 4.948799 \n644478 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n1958678 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n2159059 0.044242 0.994732 1.702462 1.151822 2.664408 3.617750 \n2219998 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n2458502 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n2458503 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n2566700 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n2566701 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n2566702 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n2877287 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n3596365 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n3891104 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n4171620 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n\n 6 7 8 9 10 11 \\\nperson_id \n386008 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n418442 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n595686 4.560310 5.610758 5.254657 5.202408 5.385828 1.836706 \n644292 5.808154 5.939138 5.530123 5.169570 5.057449 1.508326 \n644478 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n1958678 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n2159059 4.338439 5.249540 5.661475 5.941263 5.073867 1.056589 \n2219998 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n2458502 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n2458503 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n2566700 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n2566701 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n2566702 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n2877287 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n3596365 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n3891104 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n4171620 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n\n 12 13 14 15 16 17 \\\nperson_id \n386008 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n418442 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n595686 -0.149232 0.604582 0.207110 4.469041 3.740891 3.991371 \n644292 -0.379098 0.498325 0.133010 3.810490 3.312458 3.137154 \n644478 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n1958678 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n2159059 -0.719988 0.258925 -0.106390 3.844690 3.346658 3.301773 \n2219998 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n2458502 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n2458503 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n2566700 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n2566701 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n2566702 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n2877287 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n3596365 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n3891104 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n4171620 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n\n 18 19 20 21 22 23 \\\nperson_id \n386008 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n418442 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n595686 3.145862 4.005500 5.488831 1.780972 0.846532 2.046901 \n644292 2.751629 3.584558 4.421596 1.843672 0.909232 2.115301 \n644478 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n1958678 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n2159059 3.572756 4.974221 4.914166 1.461772 0.527332 1.733401 \n2219998 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n2458502 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n2458503 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n2566700 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n2566701 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n2566702 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n2877287 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n3596365 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n3891104 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n4171620 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n\n 24 \nperson_id \n386008 4.099442 \n418442 4.042442 \n595686 4.184942 \n644292 4.430042 \n644478 4.276142 \n1958678 4.099442 \n2159059 3.962642 \n2219998 4.276142 \n2458502 4.355942 \n2458503 4.355942 \n2566700 5.864195 \n2566701 5.864195 \n2566702 5.864195 \n2877287 4.042442 \n3596365 4.099442 \n3891104 4.276142 \n4171620 4.276142 ", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
0123456789101112131415161718192021222324
person_id
3860080.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
4184420.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
5956860.3634421.3139322.0216621.5996513.2016453.7260504.5603105.6107585.2546575.2024085.3858281.836706-0.1492320.6045820.2071104.4690413.7408913.9913713.1458624.0055005.4888311.7809720.8465322.0469014.184942
6442920.4261421.4195082.2679471.7966783.7106344.9487995.8081545.9391385.5301235.1695705.0574491.508326-0.3790980.4983250.1330103.8104903.3124583.1371542.7516293.5845584.4215961.8436720.9092322.1153014.430042
6444781.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
19586780.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
21590590.0442420.9947321.7024621.1518222.6644083.6177504.3384395.2495405.6614755.9412635.0738671.056589-0.7199880.258925-0.1063903.8446903.3466583.3017733.5727564.9742214.9141661.4617720.5273321.7334013.962642
22199981.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
24585020.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
24585030.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
25667000.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667010.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667020.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
28772870.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
35963650.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
38911041.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
41716201.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
\n
" }, - "execution_count": 231, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dest_size_terms.index.values[[0,1,2,3,4]]" + "utilities" ], "metadata": { "collapsed": false, @@ -910,7 +941,10 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [], + "source": [ + "display(sample_size)\n", + "sample_list[2].groupby('person_id').pick_count.sum()" + ], "metadata": { "collapsed": false, "pycharm": { @@ -946,9 +980,34 @@ "cell_type": "code", "execution_count": null, "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 46, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2, 6)\n", + "[[1 2 3 4 5 6]\n", + " [4 6 5 9 9 6]]\n", + "[5 3]\n" + ] + } + ], "source": [ - "display(sample_size)\n", - "sample_list[2].groupby('person_id').pick_count.sum()" + "utils = np.array([[1,2,3,4,5,6],[4,6,5,9,9,6]])\n", + "print(utils.shape)\n", + "print(utils)\n", + "print(np.argmax(utils, axis=1))" ], "metadata": { "collapsed": false, @@ -959,9 +1018,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, + "outputs": [ + { + "ename": "TypeError", + "evalue": "_argmax_dispatcher() got an unexpected keyword argument 'keepdim'", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/tmp/ipykernel_10839/398049723.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mutils\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mnp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0margmax\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mutils\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0maxis\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkeepdim\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", + "\u001B[0;32m<__array_function__ internals>\u001B[0m in \u001B[0;36margmax\u001B[0;34m(*args, **kwargs)\u001B[0m\n", + "\u001B[0;31mTypeError\u001B[0m: _argmax_dispatcher() got an unexpected keyword argument 'keepdim'" + ] + } + ], + "source": [ + "utils[np.argmax(utils, axis=1, keepdim=True)]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 71, "outputs": [], - "source": [], + "source": [ + "utils[np.arange(2), np.argmax(utils, axis=1)] += 1" + ], "metadata": { "collapsed": false, "pycharm": { @@ -971,7 +1059,53 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 72, + "outputs": [ + { + "data": { + "text/plain": "array([[ 1, 2, 3, 4, 5, 7],\n [ 4, 6, 5, 10, 9, 6]])" + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "utils" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 74, + "outputs": [ + { + "data": { + "text/plain": "array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,\n 18, 19, 20, 21, 22, 23, 24, 25])" + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dest_size_terms.index.values" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, "outputs": [], "source": [], "metadata": { From 2568b72148f9f2c4be4d5bfe61ea45ce4ccdcff8 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 23:00:59 +1000 Subject: [PATCH 082/135] memory work --- activitysim/core/interaction_sample.py | 8 ++++++-- notebooks/frozen_impl_interaction_sample_dev.ipynb | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 79eb15dee6..eeb89b7bca 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -40,19 +40,23 @@ def make_sample_choices_utility_based( # (len(choosers), alternative_count, sample_size) can get very large choices = np.zeros_like(utilities, dtype=np.uint32) zero_dim_index = np.arange(utilities.shape[0]) + #print(f"shape of choices = {choices.shape}, index shape = {zero_dim_index.shape}", flush=True) for i in range(sample_size): rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) # * sample_size #rands = rands.reshape(choice_dimension) rands = inverse_ev1_cdf(rands) - #chunk.log_df(trace_label, 'rands', rands) + #print(f"shape of rands = {rands.shape}", flush=True) ## # use rands ##utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented ##utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) ##utilities += rands #rands += np.repeat(utilities.to_numpy()[:, :, None], sample_size, axis=2) - rands += utilities + rands += utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? + + #print(f"shape of rands which are now utils = {rands.shape}; values are {rands}", flush=True) + #print(f"argmaxs are {np.argmax(rands, axis=1)}", flush=True) # NOT ANYMORE: this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative # NOW: len(choosers) dimensional array diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index 56274754bd..732ffcfe34 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -1059,19 +1059,19 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 77, "outputs": [ { "data": { - "text/plain": "array([[ 1, 2, 3, 4, 5, 7],\n [ 4, 6, 5, 10, 9, 6]])" + "text/plain": "True" }, - "execution_count": 72, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "utils" + "isinstance(utils, np.ndarray)" ], "metadata": { "collapsed": false, From 78630be674d9b382646f89df2d9045204585e0a3 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 23:01:39 +1000 Subject: [PATCH 083/135] spelling --- activitysim/core/interaction_sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index eeb89b7bca..8d689b0a38 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -380,7 +380,7 @@ def _interaction_sample( # choices array has same dim as utilities, with values indicating number of counts per chooser and alternative # let's turn the nonzero values into a dataframe - i, j = np.non_zero(choices_array) + i, j = np.nonzero(choices_array) chunk.log_df(trace_label, 'i', i) chunk.log_df(trace_label, 'j', j) From c6d75d126735ab4e7fb0291def7c3172c100ff2b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 23:03:21 +1000 Subject: [PATCH 084/135] delete after last ref, not before --- activitysim/core/interaction_sample.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 8d689b0a38..893180284a 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -375,9 +375,6 @@ def _interaction_sample( ) chunk.log_df(trace_label, 'choices_array', choices_array) - del utilities - chunk.log_df(trace_label, 'utilities', None) - # choices array has same dim as utilities, with values indicating number of counts per chooser and alternative # let's turn the nonzero values into a dataframe i, j = np.nonzero(choices_array) @@ -388,6 +385,9 @@ def _interaction_sample( trace_label=trace_label, trace_choosers=choosers) chunk.log_df(trace_label, 'probs', probs) + del utilities + chunk.log_df(trace_label, 'utilities', None) + choices_df = pd.DataFrame({ alt_col_name: alternatives.index.values[j], "pick_count": choices_array[i, j], From 7a66a67ded39e284de7fcecc7d6b4f436fff2bb0 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 23:10:00 +1000 Subject: [PATCH 085/135] set index --- activitysim/core/interaction_sample.py | 7 +- .../frozen_impl_interaction_sample_dev.ipynb | 73 +++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 893180284a..629abce35c 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -391,7 +391,7 @@ def _interaction_sample( choices_df = pd.DataFrame({ alt_col_name: alternatives.index.values[j], "pick_count": choices_array[i, j], - "prob": probs[i, j], + "prob": probs.to_numpy()[i, j], choosers.index.name: choosers.index.values[i] }) chunk.log_df(trace_label, 'choices_df', choices_df) @@ -462,9 +462,8 @@ def _interaction_sample( choices_df = choices_df[~choices_df['pick_dup']] del choices_df['pick_dup'] - # set index after groupby so we can trace on it - choices_df.set_index(choosers.index.name, inplace=True) - + # set index after groupby so we can trace on it + choices_df.set_index(choosers.index.name, inplace=True) chunk.log_df(trace_label, 'choices_df', choices_df) diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index 732ffcfe34..155de2df9a 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -1103,6 +1103,79 @@ } } }, + { + "cell_type": "code", + "execution_count": 81, + "outputs": [ + { + "data": { + "text/plain": " 0 1 2 3 4 5\n0 1 2 3 4 5 6\n1 4 6 5 9 9 6\n2 1 1 1 1 1 1", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
012345
0123456
1465996
2111111
\n
" + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.DataFrame(np.array([[1,2,3,4,5,6],[4,6,5,9,9,6], [1,1,1,1,1,1]]))\n", + "df" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 82, + "outputs": [ + { + "data": { + "text/plain": " 0 0 1 2\n0 1 1 2 3\n0 1 1 2 3\n1 4 4 6 5\n2 1 1 1 1", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
0012
01123
01123
14465
21111
\n
" + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.iloc[np.array([0,0,1,2]), np.array([0,0,1,2])]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 84, + "outputs": [ + { + "data": { + "text/plain": "array([1, 1, 6, 1])" + }, + "execution_count": 84, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x_ = np.array([[1,2,3,4,5,6],[4,6,5,9,9,6], [1,1,1,1,1,1]])\n", + "x_[np.array([0,0,1,2]), np.array([0,0,1,2])]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "code", "execution_count": null, From 7df4c8ddb4bee084c32a1f0dfc17ad9b949cc1ba Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 20 Jun 2022 23:36:17 +1000 Subject: [PATCH 086/135] to numpy once --- activitysim/core/interaction_sample.py | 37 +++----------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 629abce35c..f690d4add3 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -36,49 +36,18 @@ def make_sample_choices_utility_based( assert utilities.shape == (len(choosers), alternative_count) #choice_dimension = (len(choosers), alternative_count, sample_size) - # TODO [janzill Jun2022]: THIS NEEDS for loop for memory like previous method, an array of dimension + # Note [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension # (len(choosers), alternative_count, sample_size) can get very large choices = np.zeros_like(utilities, dtype=np.uint32) zero_dim_index = np.arange(utilities.shape[0]) - #print(f"shape of choices = {choices.shape}, index shape = {zero_dim_index.shape}", flush=True) + utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? for i in range(sample_size): rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) # * sample_size - #rands = rands.reshape(choice_dimension) rands = inverse_ev1_cdf(rands) - #print(f"shape of rands = {rands.shape}", flush=True) - - ## # use rands - ##utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented - ##utilities = np.repeat(utilities[:, :, None], sample_size, axis=2) - ##utilities += rands - #rands += np.repeat(utilities.to_numpy()[:, :, None], sample_size, axis=2) - rands += utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? - - #print(f"shape of rands which are now utils = {rands.shape}; values are {rands}", flush=True) - #print(f"argmaxs are {np.argmax(rands, axis=1)}", flush=True) - - # NOT ANYMORE: this gives us (len(choosers), sample_size) dimensional array, with values the chosen alternative - # NOW: len(choosers) dimensional array - #choices_array = np.argmax(rands, axis=1) - #chunk.log_df(trace_label, 'choices_array', choices_array) - + rands += utils_array choices[zero_dim_index, np.argmax(rands, axis=1)] += 1 - #choosers_index_rep = np.tile(np.arange(0, choices_array.shape[0]), sample_size) - #choices_flattened = choices_array.flatten(order='F') - #probs_selection = probs.to_numpy()[choosers_index_rep, choices_flattened].flatten(order='F') - - # # choices_flattened are 0-based index into alternatives, need to map to alternative values given by - # # alternatives.index.values (they are in this order by construction) - # # explode to one row per chooser.index, alt_zone_id - # choices_df = pd.DataFrame({ - # alt_col_name: alternatives.index.values[choices_flattened], - # 'rand': np.zeros_like(choosers_index_rep), # zero out for now - # 'prob': probs_selection, - # # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction - # choosers.index.name: np.tile(choosers.index.values, sample_size) - # }) return choices From d6f4f97531475c472642fb627b80345efec271a4 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Tue, 21 Jun 2022 07:29:51 +1000 Subject: [PATCH 087/135] remove comment --- activitysim/core/interaction_sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index f690d4add3..268b965c25 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -43,7 +43,7 @@ def make_sample_choices_utility_based( utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? for i in range(sample_size): - rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) # * sample_size + rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) rands = inverse_ev1_cdf(rands) rands += utils_array choices[zero_dim_index, np.argmax(rands, axis=1)] += 1 From 26a7090218577a95c3b8afd6d220517069fb6372 Mon Sep 17 00:00:00 2001 From: janzill Date: Tue, 21 Jun 2022 13:59:47 +1000 Subject: [PATCH 088/135] interaction sample dev --- .../frozen_impl_interaction_sample_dev.ipynb | 1966 +++++++++++++---- 1 file changed, 1513 insertions(+), 453 deletions(-) diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index 155de2df9a..e05e520bee 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -2,35 +2,30 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.385153Z", - "iopub.status.busy": "2022-05-01T10:17:57.384881Z", - "iopub.status.idle": "2022-05-01T10:17:57.534433Z", - "shell.execute_reply": "2022-05-01T10:17:57.533096Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" - }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 8, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.536623Z", - "iopub.status.busy": "2022-05-01T10:17:57.536012Z", - "iopub.status.idle": "2022-05-01T10:17:57.542755Z", - "shell.execute_reply": "2022-05-01T10:17:57.541685Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" - }, "tags": [] }, "outputs": [], @@ -40,16 +35,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 9, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:00.350944Z", - "iopub.status.busy": "2022-05-01T10:18:00.350730Z", - "iopub.status.idle": "2022-05-01T10:18:12.760977Z", - "shell.execute_reply": "2022-05-01T10:18:12.760013Z", - "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" - }, "tags": [] }, "outputs": [], @@ -67,228 +55,34 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:16.944537Z", - "iopub.status.busy": "2022-05-01T10:18:16.944291Z", - "iopub.status.idle": "2022-05-01T10:18:17.124764Z", - "shell.execute_reply": "2022-05-01T10:18:17.123725Z", - "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" - }, "tags": [] }, "outputs": [], "source": [ - "pd.set_option(\"max_columns\", 500)" + "pd.set_option(\"display.max_columns\", 500)" ] }, { "cell_type": "code", - "execution_count": 5, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3, 3, 2)\n", - "[[2 2]\n", - " [2 1]\n", - " [1 0]]\n" - ] - }, - { - "data": { - "text/plain": "array([2, 2, 1, 2, 1, 0])" - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "utils = np.array([[1,2,3,4,5,6],[4,6,5,9,9,6],[7,8,9,1,2,3]]).reshape((3,3,2))\n", - "print(utils.shape)\n", - "print(np.argmax(utils, axis=1))\n", - "np.argmax(utils, axis=1).flatten(order=\"F\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 124, - "outputs": [ - { - "data": { - "text/plain": "array([[2, 2],\n [2, 1],\n [1, 0]])" - }, - "execution_count": 124, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ch_array = np.argmax(utils, axis=1)\n", - "ch_array" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 191, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3, 4)\n" - ] - }, - { - "data": { - "text/plain": "array([[0.1 , 0.7 , 0.1 , 0.1 ],\n [0. , 0.5 , 0.25, 0.25],\n [0.3 , 0.3 , 0.2 , 0.2 ]])" - }, - "execution_count": 191, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "probs = np.array([[0.1,0.7,0.1,0.1], [0.0,0.5,0.25,0.25], [0.3,0.3,0.2,0.2]])\n", - "print(probs.shape)\n", - "probs" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 174, - "outputs": [ - { - "data": { - "text/plain": "(3, 4, 3)" - }, - "execution_count": 174, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.repeat(probs[:,:,None], 3, axis=2).shape" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 198, - "outputs": [ - { - "data": { - "text/plain": "array([0, 0, 1, 1, 2, 2])" - }, - "execution_count": 198, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.repeat(np.arange(0,probs.shape[0]), 2)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 202, - "outputs": [ - { - "data": { - "text/plain": "array([0, 1, 2, 0, 1, 2, 0, 1, 2])" - }, - "execution_count": 202, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.tile(np.arange(0,probs.shape[0]), 3)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 6, + "execution_count": 15, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:17.839947Z", - "iopub.status.busy": "2022-05-01T10:18:17.839070Z", - "iopub.status.idle": "2022-05-01T10:18:18.019676Z", - "shell.execute_reply": "2022-05-01T10:18:18.018689Z", - "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" - }, "tags": [] }, "outputs": [], "source": [ - "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_example_mtc_frozen_rand\")" + "root_dir = \"/home/jan/code/activitysim\"\n", + "example_dir = os.path.join(root_dir, \"test_runs\", \"seq_asim\")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 16, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:18.894533Z", - "iopub.status.busy": "2022-05-01T10:18:18.894303Z", - "iopub.status.idle": "2022-05-01T10:18:19.078807Z", - "shell.execute_reply": "2022-05-01T10:18:19.077951Z", - "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" - }, "tags": [] }, "outputs": [], @@ -298,23 +92,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 18, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:22.111723Z", - "iopub.status.busy": "2022-05-01T10:18:22.111490Z", - "iopub.status.idle": "2022-05-01T10:18:22.297437Z", - "shell.execute_reply": "2022-05-01T10:18:22.296501Z", - "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" - }, "tags": [] }, "outputs": [], "source": [ "parser = argparse.ArgumentParser()\n", "run.add_run_args(parser)\n", - "args = parser.parse_args(['-c', 'configs', '-o', 'output_without_before', '-d', 'data'])\n", + "args = parser.parse_args(['-c', 'configs_lessmodes', '-o', 'output_fru', '-d', 'data'])\n", "#run.run(args) # 2mins full example run\n", "if not inject.is_injectable('preload_injectables'):\n", " from activitysim import abm # register abm steps and other abm-specific injectables\n", @@ -323,7 +110,17 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 19, + "id": "9955cc9c", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "from activitysim.core import inject\n", @@ -336,17 +133,21 @@ "from activitysim.core import logit\n", "from activitysim.core.simulate import set_skim_wrapper_targets\n", "from activitysim.core import chunk" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "3a923505", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 10, + }, "outputs": [], "source": [ "from activitysim.core.logit import inverse_ev1_cdf\n", @@ -481,17 +282,175 @@ " })\n", "\n", " return choices_df" - ], + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7599d0d1-9063-48aa-95de-cef45e926a42", + "metadata": {}, + "outputs": [], + "source": [ + "from activitysim.core.logit import inverse_ev1_cdf\n", + "\n", + "def make_sample_choices_dev(\n", + " choosers, probs,\n", + " alternatives,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs,\n", + " trace_label,\n", + " utilities=None,\n", + " choose_individual_max_utility=False\n", + "):\n", + " assert isinstance(probs, pd.DataFrame)\n", + " assert probs.shape == (len(choosers), alternative_count)\n", + " assert isinstance(alternatives, pd.DataFrame)\n", + " assert len(alternatives) == alternative_count\n", + "\n", + " if allow_zero_probs:\n", + " zero_probs = (probs.sum(axis=1) == 0)\n", + " if zero_probs.all():\n", + " return pd.DataFrame(columns=[alt_col_name, 'rand', 'prob', choosers.index.name])\n", + " if zero_probs.any():\n", + " # remove from sample\n", + " probs = probs[~zero_probs]\n", + " choosers = choosers[~zero_probs]\n", + " # TODO [janzill Jun2022]: do we want this for consistency?\n", + " # might need this in other places too?\n", + " if utilities is not None:\n", + " utilities = utilities[~zero_probs]\n", + "\n", + "\n", + " if choose_individual_max_utility:\n", + " # Note [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension\n", + " # (len(choosers), alternative_count, sample_size) can get very large\n", + " choices_array = np.zeros_like(utilities, dtype=np.uint32)\n", + " zero_dim_index = np.arange(utilities.shape[0])\n", + "\n", + " utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each?\n", + " for i in range(sample_size):\n", + " rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) # TODO: once?\n", + " rands = inverse_ev1_cdf(rands)\n", + " rands += utils_array\n", + " choices_array[zero_dim_index, np.argmax(rands, axis=1)] += 1\n", + "\n", + " i, j = np.nonzero(choices_array)\n", + " chunk.log_df(trace_label, 'i', i)\n", + " chunk.log_df(trace_label, 'j', j)\n", + "\n", + " probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", + " trace_label=trace_label, trace_choosers=choosers)\n", + " del utilities\n", + " choices_df = pd.DataFrame({\n", + " alt_col_name: alternatives.index.values[j],\n", + " \"pick_count\": choices_array[i, j],\n", + " \"prob\": probs.to_numpy()[i, j],\n", + " choosers.index.name: choosers.index.values[i]\n", + " })\n", + " del choices_array\n", + " del i\n", + " del j\n", + " del probs\n", + " \n", + " else:\n", + " cum_probs_array = probs.values.cumsum(axis=1)\n", + " # alt probs in convenient layout to return prob of chose alternative\n", + " # (same layout as cum_probs_arr)\n", + " alt_probs_array = probs.values.flatten()\n", + " # get sample_size rands for each chooser\n", + " rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size)\n", + " # transform as we iterate over alternatives\n", + " # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr\n", + " # i.e rands[i] is a 2-D array of one alt choice rand for each chooser\n", + " rands = rands.T.reshape(sample_size, -1, 1)\n", + " # the alternative value chosen\n", + " choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype)\n", + " # chunk log these later after we populate them...\n", + " # the probability of the chosen alternative\n", + " choice_probs_array = np.empty([sample_size, len(choosers)])\n", + " # chunk log these later after we populate them...\n", + " alts = np.tile(alternatives.index.values, len(choosers))\n", + " # FIXME - do this all at once rather than iterate?\n", + " for i in range(sample_size):\n", + " # FIXME - do this in numpy, not pandas?\n", + " # rands for this alt in broadcastable shape\n", + " r = rands[i]\n", + "\n", + " # position of first occurrence of positive value\n", + " positions = np.argmax(cum_probs_array > r, axis=1)\n", + "\n", + " # FIXME - leave positions as numpy array, not pandas series?\n", + " # positions is series with the chosen alternative represented as a column index in probs\n", + " # which is an integer between zero and num alternatives in the alternative sample\n", + " positions = pd.Series(positions, index=probs.index)\n", + "\n", + " # need to get from an integer offset into the alternative sample to the alternative index\n", + " # that is, we want the index value of the row that is offset by rows into the\n", + " # tranche of this choosers alternatives created by cross join of alternatives and choosers\n", + "\n", + " # offsets is the offset into model_design df of first row of chooser alternatives\n", + " offsets = np.arange(len(positions)) * alternative_count\n", + "\n", + " # choices and choice_probs have one element per chooser and is in same order as choosers\n", + " choices_array[i] = np.take(alts, positions + offsets)\n", + " choice_probs_array[i] = np.take(alt_probs_array, positions + offsets)\n", + " del positions\n", + " del offsets\n", + "\n", + " del alts\n", + " del cum_probs_array\n", + " del alt_probs_array\n", + "\n", + " # explode to one row per chooser.index, alt_zone_id\n", + " choices_df = pd.DataFrame(\n", + " {alt_col_name: choices_array.flatten(order='F'),\n", + " 'rand': rands.flatten(order='F'),\n", + " 'prob': choice_probs_array.flatten(order='F'),\n", + " choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", + " })\n", + "\n", + " # pick_count and pick_dup\n", + " # pick_count is number of duplicate picks\n", + " # pick_dup flag is True for all but first of duplicates\n", + " pick_group = choices_df.groupby([choosers.index.name, alt_col_name])\n", + " # number each item in each group from 0 to the length of that group - 1.\n", + " choices_df['pick_count'] = pick_group.cumcount(ascending=True)\n", + " # flag duplicate rows after first\n", + " choices_df['pick_dup'] = choices_df['pick_count'] > 0\n", + " # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", + " choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", + " # drop the duplicates\n", + " choices_df = choices_df[~choices_df['pick_dup']]\n", + " del choices_df['pick_dup']\n", + " # set index after groupby so we can trace on it\n", + " choices_df.set_index(choosers.index.name, inplace=True)\n", + " # don't need this after tracing\n", + " del choices_df['rand']\n", + " \n", + " return choices_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dec58ae1-e15d-498c-82e4-ba7929aea7c7", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1611fe26", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 11, + }, "outputs": [ { "name": "stderr", @@ -571,9 +530,9 @@ "\n", "choose_individual_max_utility = True\n", "\n", - "resume_after = \"compute_accessibility\"\n", + "resume_after = \"work_from_home\"\n", "model_name = \"school_location\"\n", - "chunk_size = 0 # test_mtc means no chunking\n", + "chunk_size = 0 # no chunking\n", "\n", "pipeline.open_pipeline(resume_after)\n", "# preload any bulky injectables (e.g. skims) not in pipeline\n", @@ -721,7 +680,7 @@ " trace_label=chunk_trace_label, trace_choosers=chooser_chunk)\n", " #del utilities\n", "\n", - " temp_choices = hack_make_sample_choices(\n", + " choices_df = make_sample_choices_dev(\n", " chooser_chunk, probs, dest_size_terms,\n", " sample_size, alternative_count, alt_col_name,\n", " allow_zero_probs=allow_zero_probs,\n", @@ -729,32 +688,11 @@ " utilities=utilities,\n", " choose_individual_max_utility=choose_individual_max_utility\n", " )\n", - "\n", - " choices_df = temp_choices.copy()\n", - "\n", - " # pick_count and pick_dup\n", - " # pick_count is number of duplicate picks\n", - " # pick_dup flag is True for all but first of duplicates\n", - " pick_group = choices_df.groupby([choosers.index.name, alt_col_name])\n", - " # number each item in each group from 0 to the length of that group - 1.\n", - " choices_df['pick_count'] = pick_group.cumcount(ascending=True)\n", - " # flag duplicate rows after first\n", - " choices_df['pick_dup'] = choices_df['pick_count'] > 0\n", - " # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", - " choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", - " # drop the duplicates\n", - " choices_df = choices_df[~choices_df['pick_dup']]\n", - " del choices_df['pick_dup']\n", - " # set index after groupby so we can trace on it\n", - " choices_df.set_index(choosers.index.name, inplace=True)\n", - " # don't need this after tracing\n", - " del choices_df['rand']\n", " # - NARROW\n", " choices_df['prob'] = choices_df['prob'].astype(np.float32)\n", " assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty)\n", " choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32)\n", "\n", - "\n", " if choices_df.shape[0] > 0:\n", " result_list.append(choices_df)\n", "\n", @@ -777,22 +715,148 @@ " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", "\n", " pipeline.close_pipeline()" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "783732f3", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 12, + }, "outputs": [ { "data": { - "text/plain": " alt_dest rand prob person_id\n0 10 0 0.205884 386008\n1 10 0 0.288773 418442\n2 21 0 0.147571 595686\n3 3 0 0.005248 644292\n4 8 0 0.117232 644478\n.. ... ... ... ...\n420 25 0 0.265744 2566702\n421 10 0 0.288773 2877287\n422 9 0 0.336708 3596365\n423 6 0 0.032124 3891104\n424 10 0 0.097048 4171620\n\n[425 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n

425 rows × 4 columns

\n
" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n", + "

425 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " alt_dest rand prob person_id\n", + "0 10 0 0.205884 386008\n", + "1 10 0 0.288773 418442\n", + "2 21 0 0.147571 595686\n", + "3 3 0 0.005248 644292\n", + "4 8 0 0.117232 644478\n", + ".. ... ... ... ...\n", + "420 25 0 0.265744 2566702\n", + "421 10 0 0.288773 2877287\n", + "422 9 0 0.336708 3596365\n", + "423 6 0 0.032124 3891104\n", + "424 10 0 0.097048 4171620\n", + "\n", + "[425 rows x 4 columns]" + ] }, "execution_count": 12, "metadata": {}, @@ -801,17 +865,21 @@ ], "source": [ "temp_choices #.sort_values(by=[\"person_id\"])" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "e7be6961", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 28, + }, "outputs": [], "source": [ "test_df = temp_choices.copy()\n", @@ -831,22 +899,143 @@ "# set index after groupby so we can trace on it\n", "test_df.set_index(choosers.index.name, inplace=True)\n", "del test_df['rand']" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "64053012", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 29, + }, "outputs": [ { "data": { - "text/plain": " alt_dest prob pick_count\nperson_id \n386008 10 0.205884 4\n418442 10 0.288773 9\n595686 21 0.147571 4\n644292 3 0.005248 1\n644478 8 0.117232 4\n... ... ... ...\n644478 6 0.032124 1\n2458502 25 0.032621 1\n2458503 11 0.078812 1\n2566700 16 0.036706 1\n2566701 18 0.014321 1\n\n[165 rows x 3 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
386008100.2058844
418442100.2887739
595686210.1475714
64429230.0052481
64447880.1172324
............
64447860.0321241
2458502250.0326211
2458503110.0788121
2566700160.0367061
2566701180.0143211
\n

165 rows × 3 columns

\n
" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
alt_destprobpick_count
person_id
386008100.2058844
418442100.2887739
595686210.1475714
64429230.0052481
64447880.1172324
............
64447860.0321241
2458502250.0326211
2458503110.0788121
2566700160.0367061
2566701180.0143211
\n", + "

165 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " alt_dest prob pick_count\n", + "person_id \n", + "386008 10 0.205884 4\n", + "418442 10 0.288773 9\n", + "595686 21 0.147571 4\n", + "644292 3 0.005248 1\n", + "644478 8 0.117232 4\n", + "... ... ... ...\n", + "644478 6 0.032124 1\n", + "2458502 25 0.032621 1\n", + "2458503 11 0.078812 1\n", + "2566700 16 0.036706 1\n", + "2566701 18 0.014321 1\n", + "\n", + "[165 rows x 3 columns]" + ] }, "execution_count": 29, "metadata": {}, @@ -855,22 +1044,143 @@ ], "source": [ "test_df" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "2b75344c", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 39, + }, "outputs": [ { "data": { - "text/plain": " alt_dest prob pick_count\nperson_id \n3596365 9 0.336708 13\n386008 9 0.336708 12\n2877287 10 0.288773 10\n1958678 9 0.336708 10\n2458502 8 0.301414 9\n... ... ... ...\n2458503 11 0.078812 1\n644292 3 0.005248 1\n2458502 10 0.103895 1\n3891104 10 0.097048 1\n2566702 11 0.061290 1\n\n[165 rows x 3 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destprobpick_count
person_id
359636590.33670813
38600890.33670812
2877287100.28877310
195867890.33670810
245850280.3014149
............
2458503110.0788121
64429230.0052481
2458502100.1038951
3891104100.0970481
2566702110.0612901
\n

165 rows × 3 columns

\n
" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
alt_destprobpick_count
person_id
359636590.33670813
38600890.33670812
2877287100.28877310
195867890.33670810
245850280.3014149
............
2458503110.0788121
64429230.0052481
2458502100.1038951
3891104100.0970481
2566702110.0612901
\n", + "

165 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " alt_dest prob pick_count\n", + "person_id \n", + "3596365 9 0.336708 13\n", + "386008 9 0.336708 12\n", + "2877287 10 0.288773 10\n", + "1958678 9 0.336708 10\n", + "2458502 8 0.301414 9\n", + "... ... ... ...\n", + "2458503 11 0.078812 1\n", + "644292 3 0.005248 1\n", + "2458502 10 0.103895 1\n", + "3891104 10 0.097048 1\n", + "2566702 11 0.061290 1\n", + "\n", + "[165 rows x 3 columns]" + ] }, "execution_count": 39, "metadata": {}, @@ -881,22 +1191,148 @@ "test_df = temp_choices.copy()\n", "del test_df['rand']\n", "test_df.value_counts().to_frame(\"pick_count\").reset_index([\"prob\", \"alt_dest\"])" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "504b03dc", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 31, + }, "outputs": [ { "data": { - "text/plain": " alt_dest rand prob person_id\n0 10 0 0.205884 386008\n1 10 0 0.288773 418442\n2 21 0 0.147571 595686\n3 3 0 0.005248 644292\n4 8 0 0.117232 644478\n.. ... ... ... ...\n420 25 0 0.265744 2566702\n421 10 0 0.288773 2877287\n422 9 0 0.336708 3596365\n423 6 0 0.032124 3891104\n424 10 0 0.097048 4171620\n\n[425 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n

425 rows × 4 columns

\n
" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n", + "

425 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " alt_dest rand prob person_id\n", + "0 10 0 0.205884 386008\n", + "1 10 0 0.288773 418442\n", + "2 21 0 0.147571 595686\n", + "3 3 0 0.005248 644292\n", + "4 8 0 0.117232 644478\n", + ".. ... ... ... ...\n", + "420 25 0 0.265744 2566702\n", + "421 10 0 0.288773 2877287\n", + "422 9 0 0.336708 3596365\n", + "423 6 0 0.032124 3891104\n", + "424 10 0 0.097048 4171620\n", + "\n", + "[425 rows x 4 columns]" + ] }, "execution_count": 31, "metadata": {}, @@ -905,22 +1341,680 @@ ], "source": [ "test_df" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "f332d85e", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 40, + }, "outputs": [ { "data": { - "text/plain": " 0 1 2 3 4 5 \\\nperson_id \n386008 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n418442 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n595686 0.363442 1.313932 2.021662 1.599651 3.201645 3.726050 \n644292 0.426142 1.419508 2.267947 1.796678 3.710634 4.948799 \n644478 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n1958678 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n2159059 0.044242 0.994732 1.702462 1.151822 2.664408 3.617750 \n2219998 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n2458502 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n2458503 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n2566700 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n2566701 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n2566702 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n2877287 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n3596365 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n3891104 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n4171620 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n\n 6 7 8 9 10 11 \\\nperson_id \n386008 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n418442 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n595686 4.560310 5.610758 5.254657 5.202408 5.385828 1.836706 \n644292 5.808154 5.939138 5.530123 5.169570 5.057449 1.508326 \n644478 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n1958678 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n2159059 4.338439 5.249540 5.661475 5.941263 5.073867 1.056589 \n2219998 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n2458502 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n2458503 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n2566700 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n2566701 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n2566702 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n2877287 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n3596365 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n3891104 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n4171620 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n\n 12 13 14 15 16 17 \\\nperson_id \n386008 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n418442 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n595686 -0.149232 0.604582 0.207110 4.469041 3.740891 3.991371 \n644292 -0.379098 0.498325 0.133010 3.810490 3.312458 3.137154 \n644478 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n1958678 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n2159059 -0.719988 0.258925 -0.106390 3.844690 3.346658 3.301773 \n2219998 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n2458502 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n2458503 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n2566700 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n2566701 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n2566702 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n2877287 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n3596365 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n3891104 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n4171620 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n\n 18 19 20 21 22 23 \\\nperson_id \n386008 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n418442 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n595686 3.145862 4.005500 5.488831 1.780972 0.846532 2.046901 \n644292 2.751629 3.584558 4.421596 1.843672 0.909232 2.115301 \n644478 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n1958678 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n2159059 3.572756 4.974221 4.914166 1.461772 0.527332 1.733401 \n2219998 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n2458502 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n2458503 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n2566700 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n2566701 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n2566702 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n2877287 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n3596365 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n3891104 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n4171620 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n\n 24 \nperson_id \n386008 4.099442 \n418442 4.042442 \n595686 4.184942 \n644292 4.430042 \n644478 4.276142 \n1958678 4.099442 \n2159059 3.962642 \n2219998 4.276142 \n2458502 4.355942 \n2458503 4.355942 \n2566700 5.864195 \n2566701 5.864195 \n2566702 5.864195 \n2877287 4.042442 \n3596365 4.099442 \n3891104 4.276142 \n4171620 4.276142 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
0123456789101112131415161718192021222324
person_id
3860080.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
4184420.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
5956860.3634421.3139322.0216621.5996513.2016453.7260504.5603105.6107585.2546575.2024085.3858281.836706-0.1492320.6045820.2071104.4690413.7408913.9913713.1458624.0055005.4888311.7809720.8465322.0469014.184942
6442920.4261421.4195082.2679471.7966783.7106344.9487995.8081545.9391385.5301235.1695705.0574491.508326-0.3790980.4983250.1330103.8104903.3124583.1371542.7516293.5845584.4215961.8436720.9092322.1153014.430042
6444781.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
19586780.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
21590590.0442420.9947321.7024621.1518222.6644083.6177504.3384395.2495405.6614755.9412635.0738671.056589-0.7199880.258925-0.1063903.8446903.3466583.3017733.5727564.9742214.9141661.4617720.5273321.7334013.962642
22199981.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
24585020.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
24585030.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
25667000.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667010.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667020.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
28772870.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
35963650.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
38911041.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
41716201.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
\n
" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0123456789101112131415161718192021222324
person_id
3860080.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
4184420.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
5956860.3634421.3139322.0216621.5996513.2016453.7260504.5603105.6107585.2546575.2024085.3858281.836706-0.1492320.6045820.2071104.4690413.7408913.9913713.1458624.0055005.4888311.7809720.8465322.0469014.184942
6442920.4261421.4195082.2679471.7966783.7106344.9487995.8081545.9391385.5301235.1695705.0574491.508326-0.3790980.4983250.1330103.8104903.3124583.1371542.7516293.5845584.4215961.8436720.9092322.1153014.430042
6444781.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
19586780.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
21590590.0442420.9947321.7024621.1518222.6644083.6177504.3384395.2495405.6614755.9412635.0738671.056589-0.7199880.258925-0.1063903.8446903.3466583.3017733.5727564.9742214.9141661.4617720.5273321.7334013.962642
22199981.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
24585020.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
24585030.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
25667000.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667010.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667020.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
28772870.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
35963650.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
38911041.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
41716201.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4 5 \\\n", + "person_id \n", + "386008 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n", + "418442 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n", + "595686 0.363442 1.313932 2.021662 1.599651 3.201645 3.726050 \n", + "644292 0.426142 1.419508 2.267947 1.796678 3.710634 4.948799 \n", + "644478 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", + "1958678 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n", + "2159059 0.044242 0.994732 1.702462 1.151822 2.664408 3.617750 \n", + "2219998 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", + "2458502 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n", + "2458503 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n", + "2566700 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n", + "2566701 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n", + "2566702 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n", + "2877287 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n", + "3596365 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n", + "3891104 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", + "4171620 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", + "\n", + " 6 7 8 9 10 11 \\\n", + "person_id \n", + "386008 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n", + "418442 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n", + "595686 4.560310 5.610758 5.254657 5.202408 5.385828 1.836706 \n", + "644292 5.808154 5.939138 5.530123 5.169570 5.057449 1.508326 \n", + "644478 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", + "1958678 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n", + "2159059 4.338439 5.249540 5.661475 5.941263 5.073867 1.056589 \n", + "2219998 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", + "2458502 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n", + "2458503 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n", + "2566700 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n", + "2566701 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n", + "2566702 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n", + "2877287 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n", + "3596365 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n", + "3891104 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", + "4171620 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", + "\n", + " 12 13 14 15 16 17 \\\n", + "person_id \n", + "386008 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n", + "418442 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n", + "595686 -0.149232 0.604582 0.207110 4.469041 3.740891 3.991371 \n", + "644292 -0.379098 0.498325 0.133010 3.810490 3.312458 3.137154 \n", + "644478 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", + "1958678 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n", + "2159059 -0.719988 0.258925 -0.106390 3.844690 3.346658 3.301773 \n", + "2219998 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", + "2458502 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n", + "2458503 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n", + "2566700 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n", + "2566701 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n", + "2566702 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n", + "2877287 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n", + "3596365 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n", + "3891104 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", + "4171620 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", + "\n", + " 18 19 20 21 22 23 \\\n", + "person_id \n", + "386008 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n", + "418442 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n", + "595686 3.145862 4.005500 5.488831 1.780972 0.846532 2.046901 \n", + "644292 2.751629 3.584558 4.421596 1.843672 0.909232 2.115301 \n", + "644478 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", + "1958678 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n", + "2159059 3.572756 4.974221 4.914166 1.461772 0.527332 1.733401 \n", + "2219998 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", + "2458502 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n", + "2458503 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n", + "2566700 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n", + "2566701 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n", + "2566702 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n", + "2877287 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n", + "3596365 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n", + "3891104 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", + "4171620 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", + "\n", + " 24 \n", + "person_id \n", + "386008 4.099442 \n", + "418442 4.042442 \n", + "595686 4.184942 \n", + "644292 4.430042 \n", + "644478 4.276142 \n", + "1958678 4.099442 \n", + "2159059 3.962642 \n", + "2219998 4.276142 \n", + "2458502 4.355942 \n", + "2458503 4.355942 \n", + "2566700 5.864195 \n", + "2566701 5.864195 \n", + "2566702 5.864195 \n", + "2877287 4.042442 \n", + "3596365 4.099442 \n", + "3891104 4.276142 \n", + "4171620 4.276142 " + ] }, "execution_count": 40, "metadata": {}, @@ -929,68 +2023,88 @@ ], "source": [ "utilities" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "display(sample_size)\n", - "sample_list[2].groupby('person_id').pick_count.sum()" - ], + "id": "c582bf26", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "display(sample_size)\n", + "sample_list[2].groupby('person_id').pick_count.sum()" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [], + "id": "de698ddc", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [], + "id": "0c8cb9b5", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [], + "id": "0301d604", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [] }, { "cell_type": "code", "execution_count": 46, + "id": "60fb8dac", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -1008,190 +2122,136 @@ "print(utils.shape)\n", "print(utils)\n", "print(np.argmax(utils, axis=1))" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "d43aeee6", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 49, + }, "outputs": [ { "ename": "TypeError", "evalue": "_argmax_dispatcher() got an unexpected keyword argument 'keepdim'", "output_type": "error", "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/tmp/ipykernel_10839/398049723.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mutils\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mnp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0margmax\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mutils\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0maxis\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkeepdim\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", - "\u001B[0;32m<__array_function__ internals>\u001B[0m in \u001B[0;36margmax\u001B[0;34m(*args, **kwargs)\u001B[0m\n", - "\u001B[0;31mTypeError\u001B[0m: _argmax_dispatcher() got an unexpected keyword argument 'keepdim'" + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_10839/398049723.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mutils\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36margmax\u001b[0;34m(*args, **kwargs)\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: _argmax_dispatcher() got an unexpected keyword argument 'keepdim'" ] } ], "source": [ - "utils[np.argmax(utils, axis=1, keepdim=True)]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + "utils[np.argmax(utils, axis=1)]" + ] }, { "cell_type": "code", "execution_count": 71, - "outputs": [], - "source": [ - "utils[np.arange(2), np.argmax(utils, axis=1)] += 1" - ], + "id": "85892d45", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 77, - "outputs": [ - { - "data": { - "text/plain": "True" - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - } - ], + }, + "outputs": [], "source": [ - "isinstance(utils, np.ndarray)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + "utils[np.arange(2), np.argmax(utils, axis=1)] += 1" + ] }, { "cell_type": "code", - "execution_count": 74, - "outputs": [ - { - "data": { - "text/plain": "array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,\n 18, 19, 20, 21, 22, 23, 24, 25])" - }, - "execution_count": 74, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dest_size_terms.index.values" - ], + "execution_count": 77, + "id": "477c7f3d", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 81, + }, "outputs": [ { "data": { - "text/plain": " 0 1 2 3 4 5\n0 1 2 3 4 5 6\n1 4 6 5 9 9 6\n2 1 1 1 1 1 1", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
012345
0123456
1465996
2111111
\n
" + "text/plain": [ + "True" + ] }, - "execution_count": 81, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = pd.DataFrame(np.array([[1,2,3,4,5,6],[4,6,5,9,9,6], [1,1,1,1,1,1]]))\n", - "df" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + "isinstance(utils, np.ndarray)" + ] }, { "cell_type": "code", - "execution_count": 82, - "outputs": [ - { - "data": { - "text/plain": " 0 0 1 2\n0 1 1 2 3\n0 1 1 2 3\n1 4 4 6 5\n2 1 1 1 1", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
0012
01123
01123
14465
21111
\n
" - }, - "execution_count": 82, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.iloc[np.array([0,0,1,2]), np.array([0,0,1,2])]" - ], + "execution_count": 74, + "id": "a576a8a0", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 84, + }, "outputs": [ { "data": { - "text/plain": "array([1, 1, 6, 1])" + "text/plain": [ + "array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,\n", + " 18, 19, 20, 21, 22, 23, 24, 25])" + ] }, - "execution_count": 84, + "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "x_ = np.array([[1,2,3,4,5,6],[4,6,5,9,9,6], [1,1,1,1,1,1]])\n", - "x_[np.array([0,0,1,2]), np.array([0,0,1,2])]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + "dest_size_terms.index.values" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [], + "id": "72c709ab", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1205,9 +2265,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.2" + "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} From 313bcd0926bd955a48aa59d046561762812fb60b Mon Sep 17 00:00:00 2001 From: janzill Date: Tue, 21 Jun 2022 15:16:51 +1000 Subject: [PATCH 089/135] remove scale and location params from ev1 quartile fct --- activitysim/core/interaction_sample.py | 4 +- activitysim/core/logit.py | 7 +- .../frozen_impl_interaction_sample_dev.ipynb | 2486 +++++++---------- 3 files changed, 942 insertions(+), 1555 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 268b965c25..d3838dde31 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -44,9 +44,7 @@ def make_sample_choices_utility_based( utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? for i in range(sample_size): rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) - rands = inverse_ev1_cdf(rands) - rands += utils_array - choices[zero_dim_index, np.argmax(rands, axis=1)] += 1 + choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 return choices diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 37c70da6a3..a79be72041 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -183,13 +183,14 @@ def utils_to_probs(utils, trace_label=None, exponentiated=False, allow_zero_prob return probs - -def inverse_ev1_cdf(x, location=0.0, scale=1.0): +# OPTIMISE for runtime in interaction_sample - do not need scale nor location +def inverse_ev1_cdf(x): #, location=0.0, scale=1.0): # quantile function of EV1 # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution (and biogeme and larch convention) where the scale # is proportional to variance (not variance^{-1}). this means nested scales are between 0 and 1. # x can be number or np array or pd df for vecops - return location - scale * np.log(-np.log(x)) + #return location - scale * np.log(-np.log(x)) + return -np.log(-np.log(x)) # def add_ev1_random(df, nest_spec): diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index e05e520bee..cf210fcccd 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -2,28 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext autoreload" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "id": "c3dba451-1e10-403e-8614-35d57e6577f4", "metadata": { "tags": [] @@ -35,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", "metadata": { "tags": [] @@ -55,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", "metadata": { "tags": [] @@ -67,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 5, "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", "metadata": { "tags": [] @@ -80,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 6, "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", "metadata": { "tags": [] @@ -92,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 7, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "tags": [] @@ -110,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 8, "id": "9955cc9c", "metadata": { "collapsed": false, @@ -137,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 19, "id": "3a923505", "metadata": { "collapsed": false, @@ -286,13 +277,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "7599d0d1-9063-48aa-95de-cef45e926a42", "metadata": {}, "outputs": [], "source": [ - "from activitysim.core.logit import inverse_ev1_cdf\n", - "\n", "def make_sample_choices_dev(\n", " choosers, probs,\n", " alternatives,\n", @@ -329,17 +318,15 @@ "\n", " utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each?\n", " for i in range(sample_size):\n", - " rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) # TODO: once?\n", - " rands = inverse_ev1_cdf(rands)\n", - " rands += utils_array\n", - " choices_array[zero_dim_index, np.argmax(rands, axis=1)] += 1\n", + " rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count)\n", + " choices_array[zero_dim_index, np.argmax(utils_array - np.log(-np.log(rands)), axis=1)] += 1\n", "\n", " i, j = np.nonzero(choices_array)\n", " chunk.log_df(trace_label, 'i', i)\n", " chunk.log_df(trace_label, 'j', j)\n", "\n", - " probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", - " trace_label=trace_label, trace_choosers=choosers)\n", + " #probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", + " # trace_label=trace_label, trace_choosers=choosers)\n", " del utilities\n", " choices_df = pd.DataFrame({\n", " alt_col_name: alternatives.index.values[j],\n", @@ -350,7 +337,7 @@ " del choices_array\n", " del i\n", " del j\n", - " del probs\n", + " # del probs\n", " \n", " else:\n", " cum_probs_array = probs.values.cumsum(axis=1)\n", @@ -430,14 +417,6 @@ " return choices_df" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "dec58ae1-e15d-498c-82e4-ba7929aea7c7", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 11, @@ -464,64 +443,16 @@ "output_type": "stream", "text": [ "running university, 3\n", - "dropping 19 of 25 rows where size_term is zero\n", - "Running school_location with 17 persons\n", - "(17, 6)\n", - "(102,)\n", - "(17, 6)\n", - " 0 1 2 3 4 5\n", - "person_id \n", - "325623 0.001874 0.005184 0.002695 0.299427 0.508256 0.182563\n", - "386007 0.001584 0.724879 0.145533 0.080880 0.041571 0.005553\n", - "1774265 0.019349 0.146800 0.038209 0.486232 0.273819 0.035591\n", + "dropping 4288 of 4380 rows where size_term is zero\n", + "Running school_location with 5377 persons\n", "running highschool, 2\n", - "dropping 23 of 25 rows where size_term is zero\n", - "Running school_location with 5 persons\n", - "(5, 2)\n", - "(10,)\n", - "(5, 2)\n", - " 0 1\n", - "person_id \n", - "386062 0.176063 0.823937\n", - "595685 0.057237 0.942763\n", - "2877285 0.126995 0.873005\n", + "dropping 4000 of 4380 rows where size_term is zero\n", + "Running school_location with 2888 persons\n", "running gradeschool, 1\n", - "dropping 0 of 25 rows where size_term is zero\n", - "Running school_location with 17 persons\n", - "(17, 25)\n", - "(425,)\n", - "(17, 25)\n", - " 0 1 2 3 4 5 \\\n", - "person_id \n", - "386008 0.000504 0.001303 0.002645 0.001534 0.007081 0.023018 \n", - "418442 0.000554 0.001434 0.002911 0.001678 0.007704 0.020100 \n", - "595686 0.000877 0.002269 0.004605 0.003020 0.014986 0.025318 \n", - "\n", - " 6 7 8 9 10 11 \\\n", - "person_id \n", - "386008 0.056174 0.157982 0.336708 0.205884 0.069858 0.001223 \n", - "418442 0.042219 0.120700 0.229318 0.288773 0.101255 0.001526 \n", - "595686 0.058311 0.166706 0.116761 0.110817 0.133127 0.003827 \n", - "\n", - " 12 13 14 15 16 17 \\\n", - "person_id \n", - "386008 0.000206 0.000548 0.000380 0.015291 0.009346 0.008790 \n", - "418442 0.000257 0.000683 0.000474 0.019075 0.011592 0.010965 \n", - "595686 0.000525 0.001116 0.000750 0.053224 0.025697 0.033011 \n", - "\n", - " 18 19 20 21 22 23 \\\n", - "person_id \n", - "386008 0.008610 0.032788 0.028567 0.002079 0.000817 0.002728 \n", - "418442 0.012703 0.052444 0.039245 0.002288 0.000899 0.002985 \n", - "595686 0.014173 0.033481 0.147571 0.003620 0.001422 0.004723 \n", - "\n", - " 24 \n", - "person_id \n", - "386008 0.025936 \n", - "418442 0.028217 \n", - "595686 0.040062 \n", - "CPU times: user 1.39 s, sys: 328 ms, total: 1.72 s\n", - "Wall time: 2.15 s\n" + "dropping 200 of 4380 rows where size_term is zero\n", + "Running school_location with 13471 persons\n", + "CPU times: user 3min 15s, sys: 1min 17s, total: 4min 32s\n", + "Wall time: 4min 19s\n" ] } ], @@ -698,8 +629,11 @@ "\n", " if len(result_list) > 1:\n", " choices_df = pd.concat(result_list)\n", - " assert allow_zero_probs or (len(choosers_location_sample.index) == len(np.unique(choices_df.index.values))), \\\n", - " \"what is this\"\n", + " \n", + " # TODO: why does this fail\n", + " #assert allow_zero_probs or (len(choosers_location_sample.index) == len(np.unique(choices_df.index.values))), \\\n", + " # \"what is this\"\n", + " \n", " # keep alts in canonical order so choices based on their probs are stable across runs\n", " choices_df = choices_df.sort_values(by=alt_col_name).sort_index(kind='mergesort')\n", "\n", @@ -719,1532 +653,986 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "783732f3", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n", - "

425 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " alt_dest rand prob person_id\n", - "0 10 0 0.205884 386008\n", - "1 10 0 0.288773 418442\n", - "2 21 0 0.147571 595686\n", - "3 3 0 0.005248 644292\n", - "4 8 0 0.117232 644478\n", - ".. ... ... ... ...\n", - "420 25 0 0.265744 2566702\n", - "421 10 0 0.288773 2877287\n", - "422 9 0 0.336708 3596365\n", - "423 6 0 0.032124 3891104\n", - "424 10 0 0.097048 4171620\n", - "\n", - "[425 rows x 4 columns]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "temp_choices #.sort_values(by=[\"person_id\"])" - ] + "execution_count": null, + "id": "0db07776-70ab-41d1-8dff-84cd7c30065a", + "metadata": {}, + "outputs": [], + "source": [] }, { "cell_type": "code", - "execution_count": 28, - "id": "e7be6961", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, + "execution_count": 37, + "id": "87f5534d-ae21-4f7b-9c85-cd910cc375ae", + "metadata": {}, "outputs": [], "source": [ - "test_df = temp_choices.copy()\n", - "# pick_count and pick_dup\n", - "# pick_count is number of duplicate picks\n", - "# pick_dup flag is True for all but first of duplicates\n", - "pick_group = test_df.groupby([choosers.index.name, alt_col_name])\n", - "# number each item in each group from 0 to the length of that group - 1.\n", - "test_df['pick_count'] = pick_group.cumcount(ascending=True)\n", - "# flag duplicate rows after first\n", - "test_df['pick_dup'] = test_df['pick_count'] > 0\n", - "# add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", - "test_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", - "# drop the duplicates\n", - "test_df = test_df[~test_df['pick_dup']]\n", - "del test_df['pick_dup']\n", - "# set index after groupby so we can trace on it\n", - "test_df.set_index(choosers.index.name, inplace=True)\n", - "del test_df['rand']" + "def run_fru():\n", + " test_df = make_sample_choices_dev(\n", + " chooser_chunk, probs, dest_size_terms,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs=allow_zero_probs,\n", + " trace_label=chunk_trace_label,\n", + " utilities=utilities,\n", + " choose_individual_max_utility=True\n", + " )\n", + " \n", + "\n", + "def run_previous():\n", + " test_df = make_sample_choices_dev(\n", + " chooser_chunk, probs, dest_size_terms,\n", + " sample_size, alternative_count, alt_col_name,\n", + " allow_zero_probs=allow_zero_probs,\n", + " trace_label=chunk_trace_label,\n", + " utilities=None,\n", + " choose_individual_max_utility=False\n", + " )" ] }, { "cell_type": "code", - "execution_count": 29, - "id": "64053012", + "execution_count": 17, + "id": "22315306-6ec6-4daf-9176-ba397493c945", "metadata": { - "collapsed": false, + "collapsed": true, "jupyter": { - "outputs_hidden": false + "outputs_hidden": true }, - "pycharm": { - "name": "#%%\n" - } + "tags": [] }, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
alt_destprobpick_count
person_id
386008100.2058844
418442100.2887739
595686210.1475714
64429230.0052481
64447880.1172324
............
64447860.0321241
2458502250.0326211
2458503110.0788121
2566700160.0367061
2566701180.0143211
\n", - "

165 rows × 3 columns

\n", - "
" - ], - "text/plain": [ - " alt_dest prob pick_count\n", - "person_id \n", - "386008 10 0.205884 4\n", - "418442 10 0.288773 9\n", - "595686 21 0.147571 4\n", - "644292 3 0.005248 1\n", - "644478 8 0.117232 4\n", - "... ... ... ...\n", - "644478 6 0.032124 1\n", - "2458502 25 0.032621 1\n", - "2458503 11 0.078812 1\n", - "2566700 16 0.036706 1\n", - "2566701 18 0.014321 1\n", - "\n", - "[165 rows x 3 columns]" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "test_df" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "2b75344c", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false + "name": "stdout", + "output_type": "stream", + "text": [ + " " + ] }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
alt_destprobpick_count
person_id
359636590.33670813
38600890.33670812
2877287100.28877310
195867890.33670810
245850280.3014149
............
2458503110.0788121
64429230.0052481
2458502100.1038951
3891104100.0970481
2566702110.0612901
\n", - "

165 rows × 3 columns

\n", - "
" - ], "text/plain": [ - " alt_dest prob pick_count\n", - "person_id \n", - "3596365 9 0.336708 13\n", - "386008 9 0.336708 12\n", - "2877287 10 0.288773 10\n", - "1958678 9 0.336708 10\n", - "2458502 8 0.301414 9\n", - "... ... ... ...\n", - "2458503 11 0.078812 1\n", - "644292 3 0.005248 1\n", - "2458502 10 0.103895 1\n", - "3891104 10 0.097048 1\n", - "2566702 11 0.061290 1\n", + " 37642 function calls (37233 primitive calls) in 3.187 seconds\n", + "\n", + " Ordered by: cumulative time\n", "\n", - "[165 rows x 3 columns]" + " ncalls tottime percall cumtime percall filename:lineno(function)\n", + " 1 0.000 0.000 3.187 3.187 {built-in method builtins.exec}\n", + " 1 0.003 0.003 3.187 3.187 :1()\n", + " 1 0.000 0.000 3.185 3.185 2292568830.py:12(run_previous)\n", + " 1 2.165 2.165 3.185 3.185 3493973527.py:3(make_sample_choices_dev)\n", + " 170/148 0.006 0.000 0.344 0.002 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", + " 3 0.332 0.111 0.332 0.111 {method 'cumsum' of 'numpy.ndarray' objects}\n", + " 4 0.241 0.060 0.241 0.060 {method 'repeat' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.236 0.236 <__array_function__ internals>:177(tile)\n", + " 1 0.000 0.000 0.236 0.236 shape_base.py:1171(tile)\n", + " 4 0.221 0.055 0.221 0.055 {method 'flatten' of 'numpy.ndarray' objects}\n", + " 98 0.000 0.000 0.101 0.001 fromnumeric.py:51(_wrapfunc)\n", + " 30 0.000 0.000 0.077 0.003 <__array_function__ internals>:177(argmax)\n", + " 30 0.000 0.000 0.077 0.003 fromnumeric.py:1127(argmax)\n", + " 30 0.076 0.003 0.076 0.003 {method 'argmax' of 'numpy.ndarray' objects}\n", + " 2 0.000 0.000 0.065 0.033 groupby.py:3040(cumcount)\n", + " 2 0.004 0.002 0.058 0.029 groupby.py:1699(_cumcount_array)\n", + " 1 0.000 0.000 0.042 0.042 ops.py:832(group_info)\n", + " 1 0.000 0.000 0.042 0.042 ops.py:854(_get_compressed_codes)\n", + " 1 0.000 0.000 0.030 0.030 sorting.py:661(compress_group_index)\n", + " 73 0.023 0.000 0.023 0.000 {method 'take' of 'numpy.ndarray' objects}\n", + " 1 0.001 0.001 0.022 0.022 random.py:562(random_for_df)\n", + " 60 0.000 0.000 0.019 0.000 <__array_function__ internals>:177(take)\n", + " 60 0.000 0.000 0.019 0.000 fromnumeric.py:93(take)\n", + " 1 0.000 0.000 0.018 0.018 sorting.py:683(_reorder_by_uniques)\n", + " 1 0.005 0.005 0.018 0.018 random.py:598()\n", + " 63 0.000 0.000 0.016 0.000 common.py:55(new_method)\n", + " 4 0.000 0.000 0.015 0.004 frame.py:587(__init__)\n", + " 1 0.000 0.000 0.015 0.015 construction.py:425(dict_to_mgr)\n", + " 1 0.000 0.000 0.015 0.015 construction.py:102(arrays_to_mgr)\n", + " 62 0.000 0.000 0.015 0.000 arraylike.py:98(__add__)\n", + " 1 0.000 0.000 0.015 0.015 managers.py:1937(create_block_manager_from_column_arrays)\n", + " 62 0.000 0.000 0.015 0.000 series.py:5637(_arith_method)\n", + " 62 0.001 0.000 0.014 0.000 base.py:1286(_arith_method)\n", + " 9 0.000 0.000 0.013 0.001 frame.py:3463(__getitem__)\n", + " 13471 0.013 0.000 0.013 0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n", + " 101 0.001 0.000 0.012 0.000 series.py:323(__init__)\n", + " 1 0.011 0.011 0.011 0.011 {method 'get_labels_groupby' of 'pandas._libs.hashtable.Int64HashTable' objects}\n", + " 3 0.000 0.000 0.011 0.004 generic.py:3708(_take_with_is_copy)\n", + " 3 0.000 0.000 0.011 0.004 generic.py:3609(take)\n", + " 1 0.000 0.000 0.011 0.011 ops.py:790(codes)\n", + " 1 0.000 0.000 0.011 0.011 ops.py:793()\n", + " 2 0.000 0.000 0.011 0.005 grouper.py:616(codes)\n", + " 2 0.000 0.000 0.011 0.005 grouper.py:659(_codes_and_uniques)\n", + " 2 0.000 0.000 0.011 0.005 algorithms.py:568(factorize)\n", + " 3 0.000 0.000 0.011 0.004 managers.py:1683(_consolidate_inplace)\n", + " 3 0.000 0.000 0.010 0.003 managers.py:2074(_consolidate)\n", + " 8 0.005 0.001 0.010 0.001 managers.py:2091(_merge_blocks)\n", + " 2 0.000 0.000 0.009 0.005 algorithms.py:524(factorize_array)\n", + " 2 0.009 0.004 0.009 0.004 {method 'factorize' of 'pandas._libs.hashtable.Int64HashTable' objects}\n", + " 63 0.000 0.000 0.009 0.000 series.py:2988(_construct_result)\n", + " 8 0.009 0.001 0.009 0.001 {method 'argsort' of 'numpy.ndarray' objects}\n", + " 3 0.000 0.000 0.008 0.003 managers.py:875(take)\n", + " 1 0.000 0.000 0.007 0.007 managers.py:2008(_form_blocks)\n", + " 3 0.007 0.002 0.007 0.002 managers.py:2060(_stack_arrays)\n", + " 1 0.000 0.000 0.007 0.007 frame.py:3530(_getitem_bool_array)\n", + " 19 0.000 0.000 0.006 0.000 take.py:57(take_nd)\n", + " 19 0.000 0.000 0.006 0.000 take.py:120(_take_nd_ndarray)\n", + " 59/44 0.000 0.000 0.006 0.000 groupby.py:908(__getattribute__)\n", + " 3 0.000 0.000 0.006 0.002 managers.py:634(reindex_indexer)\n", + " 2 0.000 0.000 0.006 0.003 groupby.py:675(_selected_obj)\n", + " 7 0.000 0.000 0.006 0.001 blocks.py:1114(take_nd)\n", + " 17 0.000 0.000 0.005 0.000 <__array_function__ internals>:177(concatenate)\n", + " 3 0.000 0.000 0.005 0.002 <__array_function__ internals>:177(vstack)\n", + " 3 0.000 0.000 0.005 0.002 shape_base.py:222(vstack)\n", + " 3 0.000 0.000 0.005 0.002 <__array_function__ internals>:177(repeat)\n", + " 3 0.000 0.000 0.005 0.002 fromnumeric.py:436(repeat)\n", + " 1 0.004 0.004 0.004 0.004 {method 'put' of 'numpy.ndarray' objects}\n", + " 38 0.004 0.000 0.004 0.000 {built-in method numpy.arange}\n", + " 4861 0.003 0.000 0.004 0.000 {built-in method builtins.isinstance}\n", + " 1 0.000 0.000 0.003 0.003 managers.py:692()\n", + " 5 0.000 0.000 0.003 0.001 generic.py:5646(_consolidate_inplace)\n", + " 5 0.000 0.000 0.003 0.001 generic.py:5632(_protect_consolidate)\n", + " 5 0.000 0.000 0.003 0.001 generic.py:5650(f)\n", + " 5 0.000 0.000 0.003 0.001 managers.py:618(consolidate)\n", + " 102 0.001 0.000 0.003 0.000 construction.py:470(sanitize_array)\n", + " 5 0.000 0.000 0.003 0.001 managers.py:713(_slice_take_blocks_ax0)\n", + " 21 0.003 0.000 0.003 0.000 {built-in method numpy.asanyarray}\n", + " 62 0.000 0.000 0.002 0.000 array_ops.py:183(arithmetic_op)\n", + " 95 0.000 0.000 0.002 0.000 managers.py:1731(from_array)\n", + " 2 0.000 0.000 0.002 0.001 sorting.py:618(get_group_index_sorter)\n", + " 2 0.002 0.001 0.002 0.001 {pandas._libs.algos.groupsort_indexer}\n", + " 63 0.000 0.000 0.002 0.000 array_ops.py:134(_na_arithmetic_op)\n", + " 181 0.000 0.000 0.002 0.000 generic.py:5577(__setattr__)\n", + " 3 0.000 0.000 0.002 0.001 frame.py:3630(__setitem__)\n", + " 3 0.000 0.000 0.002 0.001 frame.py:3822(_set_item)\n", + " 96 0.000 0.000 0.002 0.000 blocks.py:2041(new_block)\n", + " 63 0.000 0.000 0.002 0.000 expressions.py:223(evaluate)\n", + " 63 0.000 0.000 0.002 0.000 expressions.py:92(_evaluate_numexpr)\n", + " 96 0.000 0.000 0.002 0.000 config.py:255(__call__)\n", + " 2 0.000 0.000 0.002 0.001 take.py:350(wrapper)\n", + " 1354 0.001 0.000 0.002 0.000 generic.py:43(_check)\n", + " 223 0.000 0.000 0.001 0.000 series.py:640(name)\n", + " 15/12 0.000 0.000 0.001 0.000 base.py:397(__new__)\n", + " 96 0.000 0.000 0.001 0.000 config.py:127(_get_option)\n", + " 63 0.000 0.000 0.001 0.000 expressions.py:63(_evaluate_standard)\n", + " 3 0.000 0.000 0.001 0.000 frame.py:3797(_set_item_mgr)\n", + " 2 0.000 0.000 0.001 0.001 base.py:5768(_get_indexer_strict)\n", + " 72 0.001 0.000 0.001 0.000 generic.py:5517(__finalize__)\n", + "1525/1157 0.001 0.000 0.001 0.000 {built-in method builtins.len}\n", + " 2 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis0_float64_float64}\n", + " 10 0.001 0.000 0.001 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", + " 4 0.000 0.000 0.001 0.000 {built-in method builtins.next}\n", + " 4 0.000 0.000 0.001 0.000 groupby.py:998(_group_selection_context)\n", + " 171 0.000 0.000 0.001 0.000 construction.py:379(extract_array)\n", + " 62 0.001 0.000 0.001 0.000 {built-in method _operator.add}\n", + " 2 0.000 0.000 0.001 0.001 algorithms.py:1663(safe_sort)\n", + " 2 0.000 0.000 0.001 0.001 contextlib.py:108(__enter__)\n", + " 2 0.000 0.000 0.001 0.001 groupby.py:958(_set_group_selection)\n", + " 6 0.000 0.000 0.001 0.000 base.py:3706(get_indexer)\n", + " 5 0.000 0.000 0.001 0.000 base.py:1098(take)\n", + " 223 0.000 0.000 0.001 0.000 common.py:1721(validate_all_hashable)\n", + " 8 0.000 0.000 0.001 0.000 base.py:672(_with_infer)\n", + " 1 0.001 0.001 0.001 0.001 sorting.py:100(get_group_index)\n", + " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis1_int64_int64}\n", + " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis1_float64_float64}\n", + " 128 0.000 0.000 0.001 0.000 _ufunc_config.py:32(seterr)\n", + " 96 0.000 0.000 0.001 0.000 config.py:109(_get_single_key)\n", + " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis0_bool_bool}\n", + " 4 0.000 0.000 0.001 0.000 base.py:5744(get_indexer_for)\n", + " 228 0.000 0.000 0.001 0.000 {built-in method builtins.all}\n", + " 5 0.000 0.000 0.001 0.000 algorithms.py:1352(take)\n", + " 2 0.000 0.000 0.001 0.000 <__array_function__ internals>:177(nonzero)\n", + " 2 0.000 0.000 0.001 0.000 fromnumeric.py:1866(nonzero)\n", + " 1 0.000 0.000 0.001 0.001 generic.py:11363(__iadd__)\n", + " 1 0.000 0.000 0.001 0.001 generic.py:11337(_inplace_method)\n", + " 106 0.000 0.000 0.001 0.000 base.py:6987(ensure_index)\n", + " 1709 0.001 0.000 0.001 0.000 {built-in method builtins.getattr}\n", + " 2 0.000 0.000 0.001 0.000 managers.py:1219(insert)\n", + " 64 0.000 0.000 0.001 0.000 _ufunc_config.py:429(__enter__)\n", + " 2 0.000 0.000 0.001 0.000 base.py:3400(difference)\n", + " 110 0.000 0.000 0.001 0.000 base.py:7082(maybe_extract_name)\n", + " 3 0.000 0.000 0.001 0.000 generic.py:4020(__delitem__)\n", + " 2 0.000 0.000 0.001 0.000 base.py:4109(reindex)\n", + " 1 0.000 0.000 0.001 0.001 frame.py:3790(_iset_item_mgr)\n", + " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis1_bool_bool}\n", + " 1 0.000 0.000 0.001 0.001 managers.py:1061(iset)\n", + " 12 0.001 0.000 0.001 0.000 {pandas._libs.algos.take_1d_int64_int64}\n", + " 102 0.000 0.000 0.001 0.000 construction.py:695(_try_cast)\n", + " 6 0.000 0.000 0.001 0.000 utils.py:249(maybe_convert_indices)\n", + " 5 0.000 0.000 0.001 0.000 index_tricks.py:322(__getitem__)\n", + " 3 0.000 0.000 0.001 0.000 managers.py:1299(idelete)\n", + " 446 0.000 0.000 0.001 0.000 common.py:1740()\n", + " 102 0.000 0.000 0.001 0.000 blocks.py:1960(maybe_coerce_values)\n", + " 105 0.000 0.000 0.001 0.000 generic.py:239(__init__)\n", + " 134 0.000 0.000 0.001 0.000 series.py:687(_values)\n", + " 1 0.000 0.000 0.001 0.001 blocks.py:378(delete)\n", + " 2 0.000 0.000 0.000 0.000 base.py:6561(insert)\n", + " 1 0.000 0.000 0.000 0.000 _decorators.py:302(wrapper)\n", + " 1 0.000 0.000 0.000 0.000 frame.py:5365(set_index)\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(delete)\n", + " 3 0.000 0.000 0.000 0.000 frame.py:4515(_sanitize_column)\n", + " 2 0.000 0.000 0.000 0.000 function_base.py:4958(delete)\n", + " 64 0.000 0.000 0.000 0.000 _ufunc_config.py:434(__exit__)\n", + " 99 0.000 0.000 0.000 0.000 common.py:552(require_length_match)\n", + " 3 0.000 0.000 0.000 0.000 frame.py:10986(_reindex_for_setitem)\n", + " 96 0.000 0.000 0.000 0.000 blocks.py:2055(check_ndim)\n", + " 2 0.000 0.000 0.000 0.000 base.py:3451(_difference)\n", + " 344 0.000 0.000 0.000 0.000 inference.py:321(is_hashable)\n", + " 6 0.000 0.000 0.000 0.000 {method 'copy' of 'numpy.ndarray' objects}\n", + " 105 0.000 0.000 0.000 0.000 blocks.py:1989(get_block_type)\n", + " 293 0.000 0.000 0.000 0.000 series.py:590(name)\n", + " 1 0.000 0.000 0.000 0.000 arraylike.py:54(__gt__)\n", + " 1 0.000 0.000 0.000 0.000 series.py:5613(_cmp_method)\n", + " 1 0.000 0.000 0.000 0.000 frame.py:7595(groupby)\n", + " 67 0.000 0.000 0.000 0.000 common.py:75(get_op_result_name)\n", + " 6 0.000 0.000 0.000 0.000 frame.py:3923(_get_item_cache)\n", + " 1 0.000 0.000 0.000 0.000 groupby.py:839(__init__)\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(diff)\n", + " 2 0.000 0.000 0.000 0.000 function_base.py:1295(diff)\n", + " 1 0.000 0.000 0.000 0.000 grouper.py:700(get_grouper)\n", + " 15 0.000 0.000 0.000 0.000 {pandas._libs.lib.maybe_convert_objects}\n", + " 61 0.000 0.000 0.000 0.000 series.py:825(__array__)\n", + " 13 0.000 0.000 0.000 0.000 base.py:5178(equals)\n", + " 134 0.000 0.000 0.000 0.000 managers.py:1848(internal_values)\n", + " 16 0.000 0.000 0.000 0.000 {method 'astype' of 'numpy.ndarray' objects}\n", + " 261 0.000 0.000 0.000 0.000 base.py:884(__len__)\n", + " 19 0.000 0.000 0.000 0.000 take.py:554(_take_preprocess_indexer_and_fill_value)\n", + " 63 0.000 0.000 0.000 0.000 dispatch.py:11(should_extension_dispatch)\n", + " 102 0.000 0.000 0.000 0.000 construction.py:627(_sanitize_ndim)\n", + " 9 0.000 0.000 0.000 0.000 base.py:7106(_maybe_cast_data_without_dtype)\n", + " 6 0.000 0.000 0.000 0.000 base.py:6293(_maybe_cast_listlike_indexer)\n", + " 1 0.000 0.000 0.000 0.000 {pandas._libs.algos.take_2d_axis0_int64_int64}\n", + " 128 0.000 0.000 0.000 0.000 _ufunc_config.py:131(geterr)\n", + " 95 0.000 0.000 0.000 0.000 construction.py:802(is_empty_data)\n", + " 96 0.000 0.000 0.000 0.000 config.py:589(_get_root)\n", + " 90 0.000 0.000 0.000 0.000 common.py:1587(_is_dtype_type)\n", + " 96 0.000 0.000 0.000 0.000 config.py:642(_warn_if_deprecated)\n", + " 178 0.000 0.000 0.000 0.000 construction.py:438(ensure_wrapped_if_datetimelike)\n", + " 5 0.000 0.000 0.000 0.000 frame.py:3411(_ixs)\n", + " 38 0.000 0.000 0.000 0.000 base.py:286(is_dtype)\n", + " 218 0.000 0.000 0.000 0.000 generic.py:5561(__getattr__)\n", + " 108 0.000 0.000 0.000 0.000 base.py:4820(_values)\n", + " 1 0.000 0.000 0.000 0.000 array_ops.py:227(comparison_op)\n", + " 28 0.000 0.000 0.000 0.000 common.py:497(is_categorical_dtype)\n", + " 12 0.000 0.000 0.000 0.000 base.py:554(_dtype_to_subclass)\n", + " 51 0.000 0.000 0.000 0.000 {built-in method numpy.empty}\n", + " 192 0.000 0.000 0.000 0.000 config.py:603(_get_deprecated_option)\n", + " 1 0.000 0.000 0.000 0.000 ops.py:684(shape)\n", + " 31 0.000 0.000 0.000 0.000 series.py:743(__len__)\n", + " 3 0.000 0.000 0.000 0.000 ops.py:686()\n", + " 8 0.000 0.000 0.000 0.000 base.py:6004(_should_compare)\n", + " 23 0.000 0.000 0.000 0.000 _dtype.py:328(_name_get)\n", + " 11 0.000 0.000 0.000 0.000 base.py:3577(get_loc)\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:603(ngroups)\n", + " 62 0.000 0.000 0.000 0.000 array_ops.py:441(maybe_prepare_scalar_for_op)\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:650(group_index)\n", + " 40 0.000 0.000 0.000 0.000 common.py:161(is_object_dtype)\n", + " 13 0.000 0.000 0.000 0.000 {pandas._libs.lib.infer_dtype}\n", + " 1 0.000 0.000 0.000 0.000 range.py:189(_data)\n", + " 4 0.000 0.000 0.000 0.000 algorithms.py:115(_ensure_data)\n", + " 62 0.000 0.000 0.000 0.000 __init__.py:152(align_method_SERIES)\n", + " 11 0.000 0.000 0.000 0.000 base.py:2596(inferred_type)\n", + " 1 0.000 0.000 0.000 0.000 {built-in method _operator.gt}\n", + " 322 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_list_like}\n", + " 30 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects}\n", + " 95 0.000 0.000 0.000 0.000 series.py:542(_set_axis)\n", + " 396 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass}\n", + " 17 0.000 0.000 0.000 0.000 base.py:53(shape)\n", + " 102 0.000 0.000 0.000 0.000 construction.py:664(_sanitize_str_dtypes)\n", + " 5 0.000 0.000 0.000 0.000 {built-in method builtins.sorted}\n", + " 128 0.000 0.000 0.000 0.000 {built-in method numpy.seterrobj}\n", + " 359 0.000 0.000 0.000 0.000 {built-in method builtins.hash}\n", + " 8 0.000 0.000 0.000 0.000 warnings.py:130(filterwarnings)\n", + " 2 0.000 0.000 0.000 0.000 algorithms.py:299(_get_data_algo)\n", + " 22 0.000 0.000 0.000 0.000 managers.py:2079()\n", + " 11 0.000 0.000 0.000 0.000 base.py:6284(_maybe_cast_indexer)\n", + " 14 0.000 0.000 0.000 0.000 common.py:229(asarray_tuplesafe)\n", + " 14 0.000 0.000 0.000 0.000 {method 'any' of 'numpy.ndarray' objects}\n", + " 104 0.000 0.000 0.000 0.000 common.py:1416(is_1d_only_ea_dtype)\n", + " 31 0.000 0.000 0.000 0.000 base.py:45(__len__)\n", + " 105 0.000 0.000 0.000 0.000 flags.py:47(__init__)\n", + " 256 0.000 0.000 0.000 0.000 {built-in method numpy.geterrobj}\n", + " 6 0.000 0.000 0.000 0.000 base.py:3823(_check_indexing_method)\n", + " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(putmask)\n", + " 15 0.000 0.000 0.000 0.000 numeric.py:289(full)\n", + " 1 0.000 0.000 0.000 0.000 generic.py:1516(__invert__)\n", + " 102 0.000 0.000 0.000 0.000 construction.py:684(_maybe_repeat)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:596(_homogenize)\n", + " 8 0.000 0.000 0.000 0.000 blocks.py:166(_consolidate_key)\n", + " 96 0.000 0.000 0.000 0.000 config.py:630(_translate_key)\n", + " 23 0.000 0.000 0.000 0.000 common.py:581(is_dtype_equal)\n", + " 101 0.000 0.000 0.000 0.000 managers.py:1700(__init__)\n", + " 5 0.000 0.000 0.000 0.000 frame.py:3906(_box_col_values)\n", + " 13 0.000 0.000 0.000 0.000 base.py:5023(__getitem__)\n", + " 14 0.000 0.000 0.000 0.000 _methods.py:54(_any)\n", + " 167/166 0.000 0.000 0.000 0.000 {built-in method numpy.asarray}\n", + " 172 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", + " 24 0.000 0.000 0.000 0.000 base.py:654(_simple_new)\n", + " 101 0.000 0.000 0.000 0.000 cast.py:1711(sanitize_to_nanoseconds)\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(insert)\n", + " 51 0.000 0.000 0.000 0.000 base.py:55()\n", + " 19 0.000 0.000 0.000 0.000 common.py:1240(is_float_dtype)\n", + " 16 0.000 0.000 0.000 0.000 base.py:2272(is_boolean)\n", + " 2 0.000 0.000 0.000 0.000 algorithms.py:289(_get_values_for_rank)\n", + " 2 0.000 0.000 0.000 0.000 function_base.py:5143(insert)\n", + " 11 0.000 0.000 0.000 0.000 base.py:2344(is_floating)\n", + " 72 0.000 0.000 0.000 0.000 flags.py:83(allows_duplicate_labels)\n", + " 96 0.000 0.000 0.000 0.000 config.py:571(_select_options)\n", + " 5 0.000 0.000 0.000 0.000 {method '_rebuild_blknos_and_blklocs' of 'pandas._libs.internals.BlockManager' objects}\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(setdiff1d)\n", + " 23 0.000 0.000 0.000 0.000 _dtype.py:314(_name_includes_bit_suffix)\n", + " 1 0.000 0.000 0.000 0.000 contextlib.py:72(inner)\n", + " 12 0.000 0.000 0.000 0.000 common.py:1274(is_bool_dtype)\n", + " 6 0.000 0.000 0.000 0.000 base.py:3786(_get_indexer)\n", + " 63 0.000 0.000 0.000 0.000 missing.py:138(dispatch_fill_zeros)\n", + " 6 0.000 0.000 0.000 0.000 common.py:105(is_bool_indexer)\n", + " 2 0.000 0.000 0.000 0.000 arraysetops.py:784(setdiff1d)\n", + " 5 0.000 0.000 0.000 0.000 managers.py:2137(_preprocess_slice_or_indexer)\n", + " 2 0.000 0.000 0.000 0.000 managers.py:1277(_insert_update_blklocs_and_blknos)\n", + " 4 0.000 0.000 0.000 0.000 blocks.py:238(fill_value)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:253(apply)\n", + " 2 0.000 0.000 0.000 0.000 frame.py:10808(values)\n", + " 30 0.000 0.000 0.000 0.000 common.py:1483(is_ea_or_datetimelike_dtype)\n", + " 19 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(copyto)\n", + " 15 0.000 0.000 0.000 0.000 base.py:4973(__contains__)\n", + " 209 0.000 0.000 0.000 0.000 typing.py:1149(cast)\n", + " 8 0.000 0.000 0.000 0.000 base.py:7168(unpack_nested_dtype)\n", + " 6 0.000 0.000 0.000 0.000 blocks.py:2030(new_block_2d)\n", + " 1 0.000 0.000 0.000 0.000 base.py:6947(ensure_index_from_sequences)\n", + " 96 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}\n", + " 4 0.000 0.000 0.000 0.000 missing.py:571(na_value_for_dtype)\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(in1d)\n", + " 4 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(append)\n", + " 1 0.000 0.000 0.000 0.000 indexing.py:2352(check_bool_indexer)\n", + " 6 0.000 0.000 0.000 0.000 common.py:459(is_interval_dtype)\n", + " 58 0.000 0.000 0.000 0.000 common.py:1552(get_dtype)\n", + " 13 0.000 0.000 0.000 0.000 {method 'max' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:396(apply)\n", + " 11 0.000 0.000 0.000 0.000 numerictypes.py:356(issubdtype)\n", + " 5 0.000 0.000 0.000 0.000 managers.py:1016(iget)\n", + " 59 0.000 0.000 0.000 0.000 common.py:147()\n", + " 2 0.000 0.000 0.000 0.000 arraysetops.py:519(in1d)\n", + " 31 0.000 0.000 0.000 0.000 common.py:155()\n", + " 7 0.000 0.000 0.000 0.000 common.py:680(is_integer_dtype)\n", + " 101 0.000 0.000 0.000 0.000 managers.py:1792(_block)\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(moveaxis)\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:468(__init__)\n", + " 4 0.000 0.000 0.000 0.000 function_base.py:5337(append)\n", + " 8 0.000 0.000 0.000 0.000 managers.py:1665(is_consolidated)\n", + " 5 0.000 0.000 0.000 0.000 numerictypes.py:597(find_common_type)\n", + " 6 0.000 0.000 0.000 0.000 dtypes.py:1206(is_dtype)\n", + " 1 0.000 0.000 0.000 0.000 {function SeedSequence.generate_state at 0x7fb5e410ac10}\n", + " 12 0.000 0.000 0.000 0.000 common.py:786(is_unsigned_integer_dtype)\n", + " 6 0.000 0.000 0.000 0.000 base.py:845(_engine)\n", + " 63 0.000 0.000 0.000 0.000 expressions.py:72(_can_use_numexpr)\n", + " 145 0.000 0.000 0.000 0.000 generic.py:349(flags)\n", + " 12 0.000 0.000 0.000 0.000 common.py:732(is_signed_integer_dtype)\n", + " 6 0.000 0.000 0.000 0.000 base.py:5917(_maybe_promote)\n", + " 13 0.000 0.000 0.000 0.000 _methods.py:38(_amax)\n", + " 2 0.000 0.000 0.000 0.000 numeric.py:1404(moveaxis)\n", + " 2 0.000 0.000 0.000 0.000 contextlib.py:117(__exit__)\n", + " 5 0.000 0.000 0.000 0.000 managers.py:1673(_consolidate_check)\n", + " 63 0.000 0.000 0.000 0.000 _ufunc_config.py:425(__init__)\n", + " 17 0.000 0.000 0.000 0.000 common.py:1429(is_extension_array_dtype)\n", + " 8 0.000 0.000 0.000 0.000 warnings.py:181(_add_filter)\n", + " 8 0.000 0.000 0.000 0.000 re.py:250(compile)\n", + " 34 0.000 0.000 0.000 0.000 range.py:909(__len__)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:635(_extract_index)\n", + " 2 0.000 0.000 0.000 0.000 base.py:3465(_wrap_difference_result)\n", + " 2 0.000 0.000 0.000 0.000 base.py:5799(_raise_if_missing)\n", + " 1 0.000 0.000 0.000 0.000 utils.py:457(check_array_indexer)\n", + " 6 0.000 0.000 0.000 0.000 {method 'get_indexer' of 'pandas._libs.index.IndexEngine' objects}\n", + " 1 0.000 0.000 0.000 0.000 {built-in method _operator.invert}\n", + " 2 0.000 0.000 0.000 0.000 base.py:3258(_wrap_setop_result)\n", + " 10 0.000 0.000 0.000 0.000 frame.py:1413(__len__)\n", + " 95 0.000 0.000 0.000 0.000 numeric.py:331(_is_all_dates)\n", + " 72 0.000 0.000 0.000 0.000 generic.py:328(attrs)\n", + " 62 0.000 0.000 0.000 0.000 array_ops.py:510(_bool_arith_check)\n", + " 3 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(argsort)\n", + " 75 0.000 0.000 0.000 0.000 {pandas._libs.lib.item_from_zerodim}\n", + " 2 0.000 0.000 0.000 0.000 algorithms.py:308(_check_object_for_strings)\n", + " 2 0.000 0.000 0.000 0.000 algorithms.py:193(_reconstruct_data)\n", + " 8 0.000 0.000 0.000 0.000 re.py:289(_compile)\n", + " 22 0.000 0.000 0.000 0.000 numerictypes.py:282(issubclass_)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:487()\n", + " 15 0.000 0.000 0.000 0.000 :1017(_handle_fromlist)\n", + " 3 0.000 0.000 0.000 0.000 numeric.py:139(_ensure_array)\n", + " 4 0.000 0.000 0.000 0.000 numeric.py:149(ones)\n", + " 15 0.000 0.000 0.000 0.000 managers.py:156(blknos)\n", + " 5 0.000 0.000 0.000 0.000 common.py:97(_maybe_match_name)\n", + " 3 0.000 0.000 0.000 0.000 _asarray.py:22(require)\n", + " 4 0.000 0.000 0.000 0.000 base.py:118(_reset_cache)\n", + " 6 0.000 0.000 0.000 0.000 indexing.py:2318(convert_to_index_sliceable)\n", + " 2 0.000 0.000 0.000 0.000 generic.py:1991(__contains__)\n", + " 2 0.000 0.000 0.000 0.000 managers.py:1541(as_array)\n", + " 4 0.000 0.000 0.000 0.000 common.py:348(is_datetime64tz_dtype)\n", + " 12 0.000 0.000 0.000 0.000 base.py:4846(_get_engine_target)\n", + " 2 0.000 0.000 0.000 0.000 managers.py:1267(_insert_update_mgr_locs)\n", + " 13 0.000 0.000 0.000 0.000 missing.py:390(array_equivalent)\n", + " 2 0.000 0.000 0.000 0.000 groupby.py:985(_reset_group_selection)\n", + " 73 0.000 0.000 0.000 0.000 flags.py:51(allows_duplicate_labels)\n", + " 11 0.000 0.000 0.000 0.000 inference.py:184(is_array_like)\n", + " 18 0.000 0.000 0.000 0.000 {built-in method numpy.array}\n", + " 1 0.000 0.000 0.000 0.000 construction.py:483()\n", + " 4 0.000 0.000 0.000 0.000 base.py:2232(is_unique)\n", + " 21 0.000 0.000 0.000 0.000 common.py:160(cast_scalar_indexer)\n", + " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(prod)\n", + " 9 0.000 0.000 0.000 0.000 indexing.py:2481(check_deprecated_indexers)\n", + " 9 0.000 0.000 0.000 0.000 blocks.py:2119(extend_blocks)\n", + " 64 0.000 0.000 0.000 0.000 series.py:523(_constructor)\n", + " 60 0.000 0.000 0.000 0.000 fromnumeric.py:89(_take_dispatcher)\n", + " 11 0.000 0.000 0.000 0.000 {method 'get_loc' of 'pandas._libs.index.IndexEngine' objects}\n", + " 4 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(ravel)\n", + " 2 0.000 0.000 0.000 0.000 base.py:3560(_convert_can_do_setop)\n", + " 3 0.000 0.000 0.000 0.000 fromnumeric.py:1012(argsort)\n", + " 4 0.000 0.000 0.000 0.000 common.py:287(maybe_iterable_to_list)\n", + " 59 0.000 0.000 0.000 0.000 common.py:145(classes)\n", + " 4 0.000 0.000 0.000 0.000 numeric.py:1341(normalize_axis_tuple)\n", + " 9 0.000 0.000 0.000 0.000 base.py:2624(_is_multi)\n", + " 30 0.000 0.000 0.000 0.000 fromnumeric.py:1123(_argmax_dispatcher)\n", + " 3 0.000 0.000 0.000 0.000 cast.py:1960(construct_1d_object_array_from_listlike)\n", + " 74 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}\n", + " 1 0.000 0.000 0.000 0.000 {method 'delete' of 'pandas._libs.internals.BlockPlacement' objects}\n", + " 3 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(atleast_2d)\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:2970(prod)\n", + " 8 0.000 0.000 0.000 0.000 warnings.py:458(__enter__)\n", + " 5 0.000 0.000 0.000 0.000 managers.py:1679()\n", + " 2 0.000 0.000 0.000 0.000 {method 'sum' of 'numpy.ndarray' objects}\n", + " 5 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(ndim)\n", + " 9 0.000 0.000 0.000 0.000 series.py:575(dtype)\n", + " 5 0.000 0.000 0.000 0.000 series.py:1238(_set_as_cached)\n", + " 11 0.000 0.000 0.000 0.000 generic.py:560(_get_axis)\n", + " 37 0.000 0.000 0.000 0.000 managers.py:222(items)\n", + " 11 0.000 0.000 0.000 0.000 abc.py:96(__instancecheck__)\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:69(_wrapreduction)\n", + " 9 0.000 0.000 0.000 0.000 blocks.py:332(getitem_block_columns)\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:427(_split_op_result)\n", + " 2 0.000 0.000 0.000 0.000 generic.py:1752(_check_label_or_level_ambiguity)\n", + " 2 0.000 0.000 0.000 0.000 _methods.py:46(_sum)\n", + " 13 0.000 0.000 0.000 0.000 base.py:803(is_)\n", + " 46 0.000 0.000 0.000 0.000 {pandas._libs.algos.ensure_platform_int}\n", + " 19 0.000 0.000 0.000 0.000 take.py:326(_get_take_nd_function)\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:925(_convert_grouper)\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:252(make_block)\n", + " 1 0.000 0.000 0.000 0.000 generic.py:767(_set_axis)\n", + " 3 0.000 0.000 0.000 0.000 blocks.py:2135(ensure_block_shape)\n", + " 3 0.000 0.000 0.000 0.000 {built-in method builtins.any}\n", + " 25 0.000 0.000 0.000 0.000 base.py:834(_reset_identity)\n", + " 2 0.000 0.000 0.000 0.000 managers.py:2128(_fast_count_smallints)\n", + " 3 0.000 0.000 0.000 0.000 shape_base.py:81(atleast_2d)\n", + " 26 0.000 0.000 0.000 0.000 {built-in method __new__ of type object at 0x908780}\n", + " 7 0.000 0.000 0.000 0.000 blocks.py:267(make_block_same_class)\n", + " 12 0.000 0.000 0.000 0.000 common.py:346(apply_if_callable)\n", + " 4 0.000 0.000 0.000 0.000 fromnumeric.py:1755(ravel)\n", + " 2 0.000 0.000 0.000 0.000 missing.py:625(is_valid_na_for_dtype)\n", + " 8 0.000 0.000 0.000 0.000 warnings.py:477(__exit__)\n", + " 36 0.000 0.000 0.000 0.000 base.py:1650(name)\n", + " 9 0.000 0.000 0.000 0.000 cast.py:468(maybe_promote)\n", + " 2 0.000 0.000 0.000 0.000 base.py:2844(unique)\n", + " 3 0.000 0.000 0.000 0.000 missing.py:66(isna)\n", + " 9 0.000 0.000 0.000 0.000 managers.py:1837(dtype)\n", + " 1 0.000 0.000 0.000 0.000 frame.py:821(shape)\n", + " 1 0.000 0.000 0.000 0.000 random.py:721(getrandbits)\n", + " 23 0.000 0.000 0.000 0.000 _dtype.py:24(_kind_name)\n", + " 8 0.000 0.000 0.000 0.000 {method 'remove' of 'list' objects}\n", + " 8 0.000 0.000 0.000 0.000 types.py:171(__get__)\n", + " 9 0.000 0.000 0.000 0.000 generic.py:636(_info_axis)\n", + " 2 0.000 0.000 0.000 0.000 groupby.py:1734(_obj_1d_constructor)\n", + " 9 0.000 0.000 0.000 0.000 range.py:522(equals)\n", + " 6 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.000 0.000 generic.py:1399(_indexed_same)\n", + " 10 0.000 0.000 0.000 0.000 numerictypes.py:573(_can_coerce_all)\n", + " 11 0.000 0.000 0.000 0.000 {method 'format' of 'str' objects}\n", + " 4 0.000 0.000 0.000 0.000 managers.py:1992(_grouping_func)\n", + " 3 0.000 0.000 0.000 0.000 managers.py:2102()\n", + " 31 0.000 0.000 0.000 0.000 common.py:150(classes_and_not_datetimelike)\n", + " 2 0.000 0.000 0.000 0.000 base.py:4931(_validate_fill_value)\n", + " 18 0.000 0.000 0.000 0.000 generic.py:546(_get_axis_number)\n", + " 2 0.000 0.000 0.000 0.000 base.py:785(_view)\n", + " 3 0.000 0.000 0.000 0.000 missing.py:149(_isna)\n", + " 13 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", + " 1 0.000 0.000 0.000 0.000 base.py:841(_cleanup)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:212(set_axis)\n", + " 2 0.000 0.000 0.000 0.000 base.py:4226(_wrap_reindex_result)\n", + " 2 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", + " 2 0.000 0.000 0.000 0.000 contextlib.py:238(helper)\n", + " 3 0.000 0.000 0.000 0.000 base.py:358(size)\n", + " 6 0.000 0.000 0.000 0.000 missing.py:911(clean_reindex_fill_method)\n", + " 8 0.000 0.000 0.000 0.000 warnings.py:437(__init__)\n", + " 9 0.000 0.000 0.000 0.000 {pandas._libs.lib.maybe_indices_to_slice}\n", + " 24 0.000 0.000 0.000 0.000 {built-in method _warnings._filters_mutated}\n", + " 7 0.000 0.000 0.000 0.000 managers.py:217(is_single_block)\n", + " 3 0.000 0.000 0.000 0.000 managers.py:973(from_blocks)\n", + " 3 0.000 0.000 0.000 0.000 frame.py:3883(_ensure_valid_index)\n", + " 2 0.000 0.000 0.000 0.000 base.py:768(_shallow_copy)\n", + " 2 0.000 0.000 0.000 0.000 groupby.py:977()\n", + " 4 0.000 0.000 0.000 0.000 common.py:1747(pandas_dtype)\n", + " 11 0.000 0.000 0.000 0.000 {built-in method _abc._abc_instancecheck}\n", + " 24 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_float}\n", + " 2 0.000 0.000 0.000 0.000 generic.py:1401()\n", + " 1 0.000 0.000 0.000 0.000 api.py:322(default_index)\n", + " 4 0.000 0.000 0.000 0.000 frame.py:3920(_clear_item_cache)\n", + " 3 0.000 0.000 0.000 0.000 generic.py:3925(_set_is_copy)\n", + " 4 0.000 0.000 0.000 0.000 common.py:1148(needs_i8_conversion)\n", + " 3 0.000 0.000 0.000 0.000 _asarray.py:111()\n", + " 3 0.000 0.000 0.000 0.000 shape_base.py:218(_vhstack_dispatcher)\n", + " 19 0.000 0.000 0.000 0.000 multiarray.py:1071(copyto)\n", + " 1 0.000 0.000 0.000 0.000 base.py:57(_validate_set_axis)\n", + " 3 0.000 0.000 0.000 0.000 base.py:2642(_na_value)\n", + " 16 0.000 0.000 0.000 0.000 base.py:937(dtype)\n", + " 2 0.000 0.000 0.000 0.000 base.py:4230(_maybe_preserve_names)\n", + " 13 0.000 0.000 0.000 0.000 blocks.py:354(dtype)\n", + " 2 0.000 0.000 0.000 0.000 algorithms.py:237(_ensure_arraylike)\n", + " 2 0.000 0.000 0.000 0.000 cast.py:2147(can_hold_element)\n", + " 6 0.000 0.000 0.000 0.000 {pandas._libs.internals.get_blkno_placements}\n", + " 1 0.000 0.000 0.000 0.000 {built-in method posix.urandom}\n", + " 15 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_scalar}\n", + " 17 0.000 0.000 0.000 0.000 multiarray.py:148(concatenate)\n", + " 9 0.000 0.000 0.000 0.000 blocks.py:310(_slice)\n", + " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(bincount)\n", + " 2 0.000 0.000 0.000 0.000 base.py:3554(_assert_can_do_setop)\n", + " 19 0.000 0.000 0.000 0.000 base.py:326(ndim)\n", + " 3 0.000 0.000 0.000 0.000 generic.py:566(_get_block_manager_axis)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:494()\n", + " 14 0.000 0.000 0.000 0.000 base.py:5905(_index_as_unique)\n", + " 3 0.000 0.000 0.000 0.000 {built-in method numpy.zeros}\n", + " 2 0.000 0.000 0.000 0.000 contextlib.py:82(__init__)\n", + " 14 0.000 0.000 0.000 0.000 managers.py:172(blklocs)\n", + " 5 0.000 0.000 0.000 0.000 numerictypes.py:649()\n", + " 10 0.000 0.000 0.000 0.000 {method 'insert' of 'list' objects}\n", + " 2 0.000 0.000 0.000 0.000 base.py:723(tolist)\n", + " 2 0.000 0.000 0.000 0.000 blocks.py:222(get_values)\n", + " 4 0.000 0.000 0.000 0.000 {method 'transpose' of 'numpy.ndarray' objects}\n", + " 15 0.000 0.000 0.000 0.000 {built-in method builtins.callable}\n", + " 1 0.000 0.000 0.000 0.000 range.py:167(_simple_new)\n", + " 5 0.000 0.000 0.000 0.000 blocks.py:358(iget)\n", + " 4 0.000 0.000 0.000 0.000 numeric.py:1391()\n", + " 6 0.000 0.000 0.000 0.000 base.py:518()\n", + " 14 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_integer}\n", + " 3 0.000 0.000 0.000 0.000 grouper.py:802()\n", + " 15 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_iterator}\n", + " 4 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:570(name)\n", + " 10 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", + " 9 0.000 0.000 0.000 0.000 base.py:540(_ensure_array)\n", + " 2 0.000 0.000 0.000 0.000 groupby.py:898(__getattr__)\n", + " 10 0.000 0.000 0.000 0.000 {method 'index' of 'list' objects}\n", + " 12 0.000 0.000 0.000 0.000 base.py:229(disallow_kwargs)\n", + " 9 0.000 0.000 0.000 0.000 blocks.py:244(mgr_locs)\n", + " 1 0.000 0.000 0.000 0.000 typing.py:868(__new__)\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:838(is_in_axis)\n", + " 3 0.000 0.000 0.000 0.000 shape_base.py:207(_arrays_for_stack_dispatcher)\n", + " 8 0.000 0.000 0.000 0.000 {built-in method numpy.core._multiarray_umath.normalize_axis_index}\n", + " 1 0.000 0.000 0.000 0.000 managers.py:1722(from_blocks)\n", + " 2 0.000 0.000 0.000 0.000 base.py:1701(_get_names)\n", + " 9 0.000 0.000 0.000 0.000 managers.py:919(__init__)\n", + " 9 0.000 0.000 0.000 0.000 managers.py:1257()\n", + " 8 0.000 0.000 0.000 0.000 enum.py:753(value)\n", + " 8 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", + " 1 0.000 0.000 0.000 0.000 pipeline.py:140(get_rn_generator)\n", + " 2 0.000 0.000 0.000 0.000 range.py:347(dtype)\n", + " 5 0.000 0.000 0.000 0.000 frame.py:804(axes)\n", + " 3 0.000 0.000 0.000 0.000 {built-in method pandas._libs.missing.checknull}\n", + " 1 0.000 0.000 0.000 0.000 common.py:1042(is_numeric_v_string_like)\n", + " 5 0.000 0.000 0.000 0.000 {method 'clear' of 'dict' objects}\n", + " 1 0.000 0.000 0.000 0.000 sorting.py:148()\n", + " 1 0.000 0.000 0.000 0.000 construction.py:486()\n", + " 3 0.000 0.000 0.000 0.000 grouper.py:804()\n", + " 5 0.000 0.000 0.000 0.000 generic.py:660(ndim)\n", + " 7 0.000 0.000 0.000 0.000 ops.py:680(groupings)\n", + " 5 0.000 0.000 0.000 0.000 fromnumeric.py:3164(ndim)\n", + " 3 0.000 0.000 0.000 0.000 grouper.py:803()\n", + " 1 0.000 0.000 0.000 0.000 ops.py:660(__init__)\n", + " 6 0.000 0.000 0.000 0.000 missing.py:107(clean_fill_method)\n", + " 1 0.000 0.000 0.000 0.000 sorting.py:135(_int64_cut_off)\n", + " 5 0.000 0.000 0.000 0.000 {pandas._libs.algos.ensure_int64}\n", + " 1 0.000 0.000 0.000 0.000 generic.py:4060(_check_inplace_and_allows_duplicate_labels)\n", + " 8 0.000 0.000 0.000 0.000 base.py:6022(_is_comparable_dtype)\n", + " 2 0.000 0.000 0.000 0.000 base.py:7046(ensure_has_len)\n", + " 5 0.000 0.000 0.000 0.000 base.py:692(_constructor)\n", + " 2 0.000 0.000 0.000 0.000 common.py:315(is_datetime64_dtype)\n", + " 5 0.000 0.000 0.000 0.000 numerictypes.py:650()\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:921(_is_label_like)\n", + " 5 0.000 0.000 0.000 0.000 base.py:1124(_maybe_disallow_fill)\n", + " 1 0.000 0.000 0.000 0.000 _decorators.py:214(_format_argument_list)\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:851(is_in_obj)\n", + " 2 0.000 0.000 0.000 0.000 common.py:389(is_timedelta64_dtype)\n", + " 1 0.000 0.000 0.000 0.000 base.py:890(__array__)\n", + " 5 0.000 0.000 0.000 0.000 fromnumeric.py:3160(_ndim_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 _validators.py:218(validate_bool_kwarg)\n", + " 3 0.000 0.000 0.000 0.000 generic.py:3948(_check_setitem_copy)\n", + " 3 0.000 0.000 0.000 0.000 fromnumeric.py:432(_repeat_dispatcher)\n", + " 3 0.000 0.000 0.000 0.000 {method 'upper' of 'str' objects}\n", + " 1 0.000 0.000 0.000 0.000 construction.py:233(mgr_to_mgr)\n", + " 3 0.000 0.000 0.000 0.000 managers.py:2111()\n", + " 4 0.000 0.000 0.000 0.000 fromnumeric.py:1751(_ravel_dispatcher)\n", + " 4 0.000 0.000 0.000 0.000 {built-in method _operator.index}\n", + " 3 0.000 0.000 0.000 0.000 function.py:49(__call__)\n", + " 3 0.000 0.000 0.000 0.000 frame.py:578(_constructor)\n", + " 2 0.000 0.000 0.000 0.000 cast.py:2185(np_can_hold_element)\n", + " 4 0.000 0.000 0.000 0.000 function_base.py:5333(_append_dispatcher)\n", + " 3 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", + " 3 0.000 0.000 0.000 0.000 numeric.py:199(_ensure_dtype)\n", + " 1 0.000 0.000 0.000 0.000 pipeline.py:72(rng)\n", + " 3 0.000 0.000 0.000 0.000 numeric.py:188(_validate_dtype)\n", + " 1 0.000 0.000 0.000 0.000 {built-in method pandas._libs.missing.is_matching_na}\n", + " 3 0.000 0.000 0.000 0.000 fromnumeric.py:1008(_argsort_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 base.py:3050(_validate_sort_keyword)\n", + " 3 0.000 0.000 0.000 0.000 shape_base.py:77(_atleast_2d_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 {method 'tolist' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:306(__len__)\n", + " 2 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", + " 2 0.000 0.000 0.000 0.000 shape_base.py:1243()\n", + " 2 0.000 0.000 0.000 0.000 shape_base.py:1253()\n", + " 2 0.000 0.000 0.000 0.000 grouper.py:588(_ilevel)\n", + " 2 0.000 0.000 0.000 0.000 function_base.py:1291(_diff_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 numeric.py:1466()\n", + " 1 0.000 0.000 0.000 0.000 managers.py:1118(value_getitem)\n", + " 1 0.000 0.000 0.000 0.000 range.py:241(start)\n", + " 2 0.000 0.000 0.000 0.000 generic.py:1772()\n", + " 2 0.000 0.000 0.000 0.000 base.py:7191(_maybe_try_sort)\n", + " 2 0.000 0.000 0.000 0.000 function_base.py:5139(_insert_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 fromnumeric.py:1862(_nonzero_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 multiarray.py:883(bincount)\n", + " 1 0.000 0.000 0.000 0.000 range.py:287(step)\n", + " 2 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_bool_list}\n", + " 2 0.000 0.000 0.000 0.000 arraysetops.py:515(_in1d_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 arraysetops.py:780(_setdiff1d_dispatcher)\n", + " 2 0.000 0.000 0.000 0.000 function_base.py:4954(_delete_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:70()\n", + " 2 0.000 0.000 0.000 0.000 {method 'add' of 'set' objects}\n", + " 1 0.000 0.000 0.000 0.000 {built-in method from_bytes}\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:248(mgr_locs)\n", + " 2 0.000 0.000 0.000 0.000 base.py:1898(nlevels)\n", + " 2 0.000 0.000 0.000 0.000 numeric.py:1400(_moveaxis_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 multiarray.py:1106(putmask)\n", + " 1 0.000 0.000 0.000 0.000 {method 'keys' of 'dict' objects}\n", + " 1 0.000 0.000 0.000 0.000 contextlib.py:59(_recreate_cm)\n", + " 1 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", + " 1 0.000 0.000 0.000 0.000 {method 'append' of 'pandas._libs.internals.BlockPlacement' objects}\n", + " 1 0.000 0.000 0.000 0.000 range.py:183(_constructor)\n", + " 1 0.000 0.000 0.000 0.000 range.py:264(stop)\n", + " 1 0.000 0.000 0.000 0.000 shape_base.py:1167(_tile_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:2965(_prod_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 base.py:4786(values)\n", + " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", + " 1 0.000 0.000 0.000 0.000 managers.py:282()\n", + " 1 0.000 0.000 0.000 0.000 {method 'clear_mapping' of 'pandas._libs.index.IndexEngine' objects}\n", + " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_bool}" ] }, - "execution_count": 39, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "test_df = temp_choices.copy()\n", - "del test_df['rand']\n", - "test_df.value_counts().to_frame(\"pick_count\").reset_index([\"prob\", \"alt_dest\"])" + "%prun -s cumulative run_previous()" ] }, { "cell_type": "code", - "execution_count": 31, - "id": "504b03dc", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
alt_destrandprobperson_id
01000.205884386008
11000.288773418442
22100.147571595686
3300.005248644292
4800.117232644478
...............
4202500.2657442566702
4211000.2887732877287
422900.3367083596365
423600.0321243891104
4241000.0970484171620
\n", - "

425 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " alt_dest rand prob person_id\n", - "0 10 0 0.205884 386008\n", - "1 10 0 0.288773 418442\n", - "2 21 0 0.147571 595686\n", - "3 3 0 0.005248 644292\n", - "4 8 0 0.117232 644478\n", - ".. ... ... ... ...\n", - "420 25 0 0.265744 2566702\n", - "421 10 0 0.288773 2877287\n", - "422 9 0 0.336708 3596365\n", - "423 6 0 0.032124 3891104\n", - "424 10 0 0.097048 4171620\n", - "\n", - "[425 rows x 4 columns]" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "id": "35a9df01-2b32-41c3-b9e2-259b21f214c6", + "metadata": {}, + "outputs": [], "source": [ - "test_df" + "37642 function calls (37233 primitive calls) in 3.187 seconds\n", + "406159 function calls (406079 primitive calls) in 60.697 seconds\n", + "\n", + " Ordered by: cumulative time\n", + "\n", + " ncalls tottime percall cumtime percall filename:lineno(function)\n", + " 1 0.000 0.000 60.697 60.697 {built-in method builtins.exec}\n", + " 1 0.000 0.000 60.697 60.697 :1()\n", + " 1 0.033 0.033 60.697 60.697 2292568830.py:1(run_fru)\n", + " 1 37.147 37.147 60.664 60.664 3636660809.py:1(make_sample_choices_dev)\n", + " 30 0.745 0.025 22.155 0.738 random.py:562(random_for_df)\n", + " 30 0.192 0.006 13.381 0.446 random.py:598()\n", + " 404130 13.189 0.000 13.189 0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n", + " 32 8.025 0.251 8.025 0.251 {built-in method numpy.asanyarray}\n", + " 70/66 0.128 0.002 1.361 0.021 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", + " 32 0.000 0.000 1.232 0.039 fromnumeric.py:51(_wrapfunc)\n", + " 30 0.000 0.000 1.104 0.037 <__array_function__ internals>:177(argmax)\n", + " 30 0.000 0.000 1.103 0.037 fromnumeric.py:1127(argmax)\n", + " 30 1.103 0.037 1.103 0.037 {method 'argmax' of 'numpy.ndarray' objects}\n", + " \n", + "w/o random calls (test where we simply take max(obs_utlils) but everything else is identical)\n", + " 1039 function calls (1019 primitive calls) in 1.403 seconds\n", + " Ordered by: cumulative time\n", + " ncalls tottime percall cumtime percall filename:lineno(function)\n", + " 1 0.000 0.000 1.403 1.403 {built-in method builtins.exec}\n", + " 1 0.000 0.000 1.403 1.403 :1()\n", + " 1 0.000 0.000 1.403 1.403 2292568830.py:1(run_fru)\n", + " 1 0.021 0.021 1.403 1.403 691721383.py:1(make_sample_choices_dev)\n", + " 40/36 0.120 0.003 1.380 0.038 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", + " 32 0.000 0.000 1.260 0.039 fromnumeric.py:51(_wrapfunc)\n", + " 30 0.000 0.000 1.134 0.038 <__array_function__ internals>:177(argmax)\n", + " 30 0.000 0.000 1.133 0.038 fromnumeric.py:1127(argmax)\n", + " 30 1.133 0.038 1.133 0.038 {method 'argmax' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.127 0.127 <__array_function__ internals>:177(nonzero)\n", + " 1 0.000 0.000 0.127 0.127 fromnumeric.py:1866(nonzero)\n", + " 1 0.127 0.127 0.127 0.127 {method 'nonzero' of 'numpy.ndarray' objects}" ] }, { "cell_type": "code", - "execution_count": 40, - "id": "f332d85e", + "execution_count": 38, + "id": "082ecf8f-bf23-4a61-be4f-a0d914469f18", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } + "tags": [] }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " " + ] + }, { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
0123456789101112131415161718192021222324
person_id
3860080.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
4184420.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
5956860.3634421.3139322.0216621.5996513.2016453.7260504.5603105.6107585.2546575.2024085.3858281.836706-0.1492320.6045820.2071104.4690413.7408913.9913713.1458624.0055005.4888311.7809720.8465322.0469014.184942
6442920.4261421.4195082.2679471.7966783.7106344.9487995.8081545.9391385.5301235.1695705.0574491.508326-0.3790980.4983250.1330103.8104903.3124583.1371542.7516293.5845584.4215961.8436720.9092322.1153014.430042
6444781.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
19586780.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
21590590.0442420.9947321.7024621.1518222.6644083.6177504.3384395.2495405.6614755.9412635.0738671.056589-0.7199880.258925-0.1063903.8446903.3466583.3017733.5727564.9742214.9141661.4617720.5273321.7334013.962642
22199981.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
24585020.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
24585030.4090421.3702512.1530141.7310023.4807684.7189335.6439646.5794796.0719505.5143695.2380571.590421-0.3955170.4869250.1159103.7990903.3067583.1656542.8143293.6985584.5693671.8265720.8921322.0925014.355942
25667000.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667010.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
25667020.8808552.2568772.9903832.0101253.4643494.1442684.6424055.0491074.9810574.7404564.3972831.278460-0.3955170.6538390.3204293.8845903.3067582.9433542.3868293.2197584.0991682.5970801.3424323.0830895.864195
28772870.1126421.0631321.7708621.2202222.7442083.7032504.4453775.4958256.1376266.3681575.3201521.124989-0.6572880.321625-0.0436903.6508903.1528583.0972543.2443764.6622604.3723391.5301720.5957321.7961014.042442
35963650.1582421.1087321.8164621.2715222.8012083.9800784.8722715.9063006.6630346.1711295.0902861.045189-0.7370880.241825-0.1234903.5710903.0787583.0174542.9967294.3338804.1960681.5757720.6413321.8474014.099442
38911041.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
41716201.2420731.9120782.2679471.9280312.9389413.6177504.2871394.9123074.8613574.7233564.4086831.6725160.3433381.6553981.1742175.3556674.3648133.2454542.6889293.4306584.2902442.0659721.2341322.3934914.276142
\n", - "
" - ], "text/plain": [ - " 0 1 2 3 4 5 \\\n", - "person_id \n", - "386008 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n", - "418442 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n", - "595686 0.363442 1.313932 2.021662 1.599651 3.201645 3.726050 \n", - "644292 0.426142 1.419508 2.267947 1.796678 3.710634 4.948799 \n", - "644478 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", - "1958678 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n", - "2159059 0.044242 0.994732 1.702462 1.151822 2.664408 3.617750 \n", - "2219998 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", - "2458502 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n", - "2458503 0.409042 1.370251 2.153014 1.731002 3.480768 4.718933 \n", - "2566700 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n", - "2566701 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n", - "2566702 0.880855 2.256877 2.990383 2.010125 3.464349 4.144268 \n", - "2877287 0.112642 1.063132 1.770862 1.220222 2.744208 3.703250 \n", - "3596365 0.158242 1.108732 1.816462 1.271522 2.801208 3.980078 \n", - "3891104 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", - "4171620 1.242073 1.912078 2.267947 1.928031 2.938941 3.617750 \n", - "\n", - " 6 7 8 9 10 11 \\\n", - "person_id \n", - "386008 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n", - "418442 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n", - "595686 4.560310 5.610758 5.254657 5.202408 5.385828 1.836706 \n", - "644292 5.808154 5.939138 5.530123 5.169570 5.057449 1.508326 \n", - "644478 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", - "1958678 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n", - "2159059 4.338439 5.249540 5.661475 5.941263 5.073867 1.056589 \n", - "2219998 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", - "2458502 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n", - "2458503 5.643964 6.579479 6.071950 5.514369 5.238057 1.590421 \n", - "2566700 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n", - "2566701 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n", - "2566702 4.642405 5.049107 4.981057 4.740456 4.397283 1.278460 \n", - "2877287 4.445377 5.495825 6.137626 6.368157 5.320152 1.124989 \n", - "3596365 4.872271 5.906300 6.663034 6.171129 5.090286 1.045189 \n", - "3891104 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", - "4171620 4.287139 4.912307 4.861357 4.723356 4.408683 1.672516 \n", + " 406159 function calls (406079 primitive calls) in 61.192 seconds\n", "\n", - " 12 13 14 15 16 17 \\\n", - "person_id \n", - "386008 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n", - "418442 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n", - "595686 -0.149232 0.604582 0.207110 4.469041 3.740891 3.991371 \n", - "644292 -0.379098 0.498325 0.133010 3.810490 3.312458 3.137154 \n", - "644478 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", - "1958678 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n", - "2159059 -0.719988 0.258925 -0.106390 3.844690 3.346658 3.301773 \n", - "2219998 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", - "2458502 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n", - "2458503 -0.395517 0.486925 0.115910 3.799090 3.306758 3.165654 \n", - "2566700 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n", - "2566701 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n", - "2566702 -0.395517 0.653839 0.320429 3.884590 3.306758 2.943354 \n", - "2877287 -0.657288 0.321625 -0.043690 3.650890 3.152858 3.097254 \n", - "3596365 -0.737088 0.241825 -0.123490 3.571090 3.078758 3.017454 \n", - "3891104 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", - "4171620 0.343338 1.655398 1.174217 5.355667 4.364813 3.245454 \n", + " Ordered by: cumulative time\n", "\n", - " 18 19 20 21 22 23 \\\n", - "person_id \n", - "386008 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n", - "418442 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n", - "595686 3.145862 4.005500 5.488831 1.780972 0.846532 2.046901 \n", - "644292 2.751629 3.584558 4.421596 1.843672 0.909232 2.115301 \n", - "644478 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", - "1958678 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n", - "2159059 3.572756 4.974221 4.914166 1.461772 0.527332 1.733401 \n", - "2219998 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", - "2458502 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n", - "2458503 2.814329 3.698558 4.569367 1.826572 0.892132 2.092501 \n", - "2566700 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n", - "2566701 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n", - "2566702 2.386829 3.219758 4.099168 2.597080 1.342432 3.083089 \n", - "2877287 3.244376 4.662260 4.372339 1.530172 0.595732 1.796101 \n", - "3596365 2.996729 4.333880 4.196068 1.575772 0.641332 1.847401 \n", - "3891104 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", - "4171620 2.688929 3.430658 4.290244 2.065972 1.234132 2.393491 \n", - "\n", - " 24 \n", - "person_id \n", - "386008 4.099442 \n", - "418442 4.042442 \n", - "595686 4.184942 \n", - "644292 4.430042 \n", - "644478 4.276142 \n", - "1958678 4.099442 \n", - "2159059 3.962642 \n", - "2219998 4.276142 \n", - "2458502 4.355942 \n", - "2458503 4.355942 \n", - "2566700 5.864195 \n", - "2566701 5.864195 \n", - "2566702 5.864195 \n", - "2877287 4.042442 \n", - "3596365 4.099442 \n", - "3891104 4.276142 \n", - "4171620 4.276142 " + " ncalls tottime percall cumtime percall filename:lineno(function)\n", + " 1 0.000 0.000 61.192 61.192 {built-in method builtins.exec}\n", + " 1 0.000 0.000 61.192 61.192 :1()\n", + " 1 0.027 0.027 61.192 61.192 2292568830.py:1(run_fru)\n", + " 1 37.470 37.470 61.165 61.165 1111942506.py:1(make_sample_choices_dev)\n", + " 30 0.778 0.026 22.305 0.744 random.py:562(random_for_df)\n", + " 30 0.195 0.006 13.419 0.447 random.py:598()\n", + " 404130 13.225 0.000 13.225 0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n", + " 32 8.104 0.253 8.104 0.253 {built-in method numpy.asanyarray}\n", + " 70/66 0.122 0.002 1.388 0.021 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", + " 32 0.000 0.000 1.265 0.040 fromnumeric.py:51(_wrapfunc)\n", + " 30 0.000 0.000 1.129 0.038 <__array_function__ internals>:177(argmax)\n", + " 30 0.000 0.000 1.128 0.038 fromnumeric.py:1127(argmax)\n", + " 30 1.128 0.038 1.128 0.038 {method 'argmax' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.137 0.137 <__array_function__ internals>:177(nonzero)\n", + " 1 0.000 0.000 0.137 0.137 fromnumeric.py:1866(nonzero)\n", + " 1 0.137 0.137 0.137 0.137 {method 'nonzero' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.122 0.122 <__array_function__ internals>:177(zeros_like)\n", + " 1 0.000 0.000 0.122 0.122 numeric.py:76(zeros_like)\n", + " 2 0.000 0.000 0.122 0.061 <__array_function__ internals>:177(copyto)\n", + " 30 0.000 0.000 0.003 0.000 contextlib.py:72(inner)\n", + " 30 0.002 0.000 0.002 0.000 {function SeedSequence.generate_state at 0x7fb5e410ac10}\n", + " 1 0.000 0.000 0.001 0.001 frame.py:587(__init__)\n", + " 1 0.000 0.000 0.001 0.001 construction.py:425(dict_to_mgr)\n", + " 1 0.000 0.000 0.001 0.001 construction.py:102(arrays_to_mgr)\n", + " 60 0.000 0.000 0.001 0.000 _ufunc_config.py:32(seterr)\n", + " 30 0.000 0.000 0.000 0.000 random.py:721(getrandbits)\n", + " 30 0.000 0.000 0.000 0.000 _ufunc_config.py:429(__enter__)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:1937(create_block_manager_from_column_arrays)\n", + " 158/83 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", + " 32 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(concatenate)\n", + " 30 0.000 0.000 0.000 0.000 {built-in method posix.urandom}\n", + " 2/1 0.000 0.000 0.000 0.000 base.py:397(__new__)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:1683(_consolidate_inplace)\n", + " 32 0.000 0.000 0.000 0.000 frame.py:1413(__len__)\n", + " 30 0.000 0.000 0.000 0.000 _ufunc_config.py:434(__exit__)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:2074(_consolidate)\n", + " 60 0.000 0.000 0.000 0.000 _ufunc_config.py:131(geterr)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:2008(_form_blocks)\n", + " 190 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", + " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(empty_like)\n", + " 5 0.000 0.000 0.000 0.000 construction.py:470(sanitize_array)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:596(_homogenize)\n", + " 38 0.000 0.000 0.000 0.000 abc.py:96(__instancecheck__)\n", + " 30 0.000 0.000 0.000 0.000 pipeline.py:140(get_rn_generator)\n", + " 3 0.000 0.000 0.000 0.000 managers.py:2091(_merge_blocks)\n", + " 60 0.000 0.000 0.000 0.000 {built-in method numpy.seterrobj}\n", + " 2 0.000 0.000 0.000 0.000 frame.py:1693(to_numpy)\n", + " 36 0.000 0.000 0.000 0.000 base.py:884(__len__)\n", + " 1 0.000 0.000 0.000 0.000 {built-in method builtins.sorted}\n", + " 8 0.000 0.000 0.000 0.000 managers.py:2079()\n", + " 38 0.000 0.000 0.000 0.000 {built-in method _abc._abc_instancecheck}\n", + " 120 0.000 0.000 0.000 0.000 {built-in method numpy.geterrobj}\n", + " 1 0.000 0.000 0.000 0.000 base.py:7106(_maybe_cast_data_without_dtype)\n", + " 4 0.000 0.000 0.000 0.000 blocks.py:166(_consolidate_key)\n", + " 88 0.000 0.000 0.000 0.000 {built-in method builtins.getattr}\n", + " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.maybe_convert_objects}\n", + " 4 0.000 0.000 0.000 0.000 managers.py:2060(_stack_arrays)\n", + " 3 0.000 0.000 0.000 0.000 generic.py:5646(_consolidate_inplace)\n", + " 4 0.000 0.000 0.000 0.000 _dtype.py:328(_name_get)\n", + " 3 0.000 0.000 0.000 0.000 generic.py:5632(_protect_consolidate)\n", + " 41 0.000 0.000 0.000 0.000 generic.py:43(_check)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:635(_extract_index)\n", + " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(vstack)\n", + " 2 0.000 0.000 0.000 0.000 chunk.py:599(log_df)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:487()\n", + " 2 0.000 0.000 0.000 0.000 chunk.py:140(chunk_training_mode)\n", + " 1 0.000 0.000 0.000 0.000 generic.py:2063(__array__)\n", + " 4 0.000 0.000 0.000 0.000 common.py:348(is_datetime64tz_dtype)\n", + " 2 0.000 0.000 0.000 0.000 frame.py:821(shape)\n", + " 1 0.000 0.000 0.000 0.000 shape_base.py:222(vstack)\n", + " 5 0.000 0.000 0.000 0.000 construction.py:379(extract_array)\n", + " 2 0.000 0.000 0.000 0.000 managers.py:1541(as_array)\n", + " 4 0.000 0.000 0.000 0.000 _asarray.py:22(require)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:483()\n", + " 2 0.000 0.000 0.000 0.000 common.py:229(asarray_tuplesafe)\n", + " 1 0.000 0.000 0.000 0.000 frame.py:898(_values)\n", + " 5 0.000 0.000 0.000 0.000 construction.py:695(_try_cast)\n", + " 4 0.000 0.000 0.000 0.000 _dtype.py:314(_name_includes_bit_suffix)\n", + " 2 0.000 0.000 0.000 0.000 config.py:108(setting)\n", + " 4 0.000 0.000 0.000 0.000 base.py:286(is_dtype)\n", + " 4 0.000 0.000 0.000 0.000 common.py:287(maybe_iterable_to_list)\n", + " 1 0.000 0.000 0.000 0.000 config.py:255(__call__)\n", + " 3 0.000 0.000 0.000 0.000 generic.py:5650(f)\n", + " 1 0.000 0.000 0.000 0.000 config.py:127(_get_option)\n", + " 30 0.000 0.000 0.000 0.000 {built-in method from_bytes}\n", + " 30 0.000 0.000 0.000 0.000 fromnumeric.py:1123(_argmax_dispatcher)\n", + " 30 0.000 0.000 0.000 0.000 pipeline.py:72(rng)\n", + " 4 0.000 0.000 0.000 0.000 managers.py:1665(is_consolidated)\n", + " 4 0.000 0.000 0.000 0.000 numerictypes.py:356(issubdtype)\n", + " 1 0.000 0.000 0.000 0.000 {built-in method numpy.arange}\n", + " 1 0.000 0.000 0.000 0.000 {method '_rebuild_blknos_and_blklocs' of 'pandas._libs.internals.BlockManager' objects}\n", + " 1 0.000 0.000 0.000 0.000 base.py:554(_dtype_to_subclass)\n", + " 5 0.000 0.000 0.000 0.000 blocks.py:1989(get_block_type)\n", + " 32 0.000 0.000 0.000 0.000 multiarray.py:148(concatenate)\n", + " 1 0.000 0.000 0.000 0.000 numeric.py:289(full)\n", + " 2 0.000 0.000 0.000 0.000 inject.py:121(get_injectable)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:1673(_consolidate_check)\n", + " 2 0.000 0.000 0.000 0.000 base.py:7082(maybe_extract_name)\n", + " 5 0.000 0.000 0.000 0.000 common.py:1587(_is_dtype_type)\n", + " 1 0.000 0.000 0.000 0.000 config.py:109(_get_single_key)\n", + " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(argsort)\n", + " 4 0.000 0.000 0.000 0.000 common.py:552(require_length_match)\n", + " 1 0.000 0.000 0.000 0.000 base.py:654(_simple_new)\n", + " 6 0.000 0.000 0.000 0.000 {built-in method numpy.empty}\n", + " 5 0.000 0.000 0.000 0.000 construction.py:627(_sanitize_ndim)\n", + " 2 0.000 0.000 0.000 0.000 common.py:1240(is_float_dtype)\n", + " 7 0.000 0.000 0.000 0.000 range.py:909(__len__)\n", + " 32 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass}\n", + " 1 0.000 0.000 0.000 0.000 cast.py:1960(construct_1d_object_array_from_listlike)\n", + " 30 0.000 0.000 0.000 0.000 contextlib.py:59(_recreate_cm)\n", + " 4 0.000 0.000 0.000 0.000 managers.py:1992(_grouping_func)\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:2030(new_block_2d)\n", + " 8 0.000 0.000 0.000 0.000 numerictypes.py:282(issubclass_)\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:1012(argsort)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:1679()\n", + " 1 0.000 0.000 0.000 0.000 base.py:53(shape)\n", + " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(atleast_2d)\n", + " 6 0.000 0.000 0.000 0.000 {built-in method numpy.array}\n", + " 2 0.000 0.000 0.000 0.000 orca.py:1610(get_injectable)\n", + " 3 0.000 0.000 0.000 0.000 managers.py:618(consolidate)\n", + " 3 0.000 0.000 0.000 0.000 blocks.py:2119(extend_blocks)\n", + " 1 0.000 0.000 0.000 0.000 api.py:322(default_index)\n", + " 1 0.000 0.000 0.000 0.000 generic.py:239(__init__)\n", + " 4 0.000 0.000 0.000 0.000 common.py:1483(is_ea_or_datetimelike_dtype)\n", + " 4 0.000 0.000 0.000 0.000 _asarray.py:111()\n", + " 3 0.000 0.000 0.000 0.000 generic.py:5577(__setattr__)\n", + " 3 0.000 0.000 0.000 0.000 base.py:55()\n", + " 1 0.000 0.000 0.000 0.000 construction.py:494()\n", + " 2 0.000 0.000 0.000 0.000 :1017(_handle_fromlist)\n", + " 2 0.000 0.000 0.000 0.000 blocks.py:222(get_values)\n", + " 5 0.000 0.000 0.000 0.000 construction.py:664(_sanitize_str_dtypes)\n", + " 5 0.000 0.000 0.000 0.000 {built-in method numpy.asarray}\n", + " 17 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}\n", + " 1 0.000 0.000 0.000 0.000 config.py:589(_get_root)\n", + " 1 0.000 0.000 0.000 0.000 common.py:786(is_unsigned_integer_dtype)\n", + " 3 0.000 0.000 0.000 0.000 construction.py:438(ensure_wrapped_if_datetimelike)\n", + " 1 0.000 0.000 0.000 0.000 blocks.py:1960(maybe_coerce_values)\n", + " 1 0.000 0.000 0.000 0.000 shape_base.py:81(atleast_2d)\n", + " 1 0.000 0.000 0.000 0.000 {method 'argsort' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.000 0.000 range.py:167(_simple_new)\n", + " 5 0.000 0.000 0.000 0.000 construction.py:684(_maybe_repeat)\n", + " 14 0.000 0.000 0.000 0.000 typing.py:1149(cast)\n", + " 1 0.000 0.000 0.000 0.000 common.py:161(is_object_dtype)\n", + " 1 0.000 0.000 0.000 0.000 config.py:642(_warn_if_deprecated)\n", + " 1 0.000 0.000 0.000 0.000 common.py:732(is_signed_integer_dtype)\n", + " 3 0.000 0.000 0.000 0.000 common.py:147()\n", + " 10 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_list_like}\n", + " 4 0.000 0.000 0.000 0.000 common.py:1416(is_1d_only_ea_dtype)\n", + " 2 0.000 0.000 0.000 0.000 inject.py:116(is_injectable)\n", + " 9 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", + " 2 0.000 0.000 0.000 0.000 inference.py:321(is_hashable)\n", + " 1 0.000 0.000 0.000 0.000 {built-in method numpy.zeros}\n", + " 1 0.000 0.000 0.000 0.000 managers.py:2102()\n", + " 1 0.000 0.000 0.000 0.000 construction.py:486()\n", + " 2 0.000 0.000 0.000 0.000 generic.py:5561(__getattr__)\n", + " 2 0.000 0.000 0.000 0.000 config.py:603(_get_deprecated_option)\n", + " 4 0.000 0.000 0.000 0.000 {method 'format' of 'str' objects}\n", + " 2 0.000 0.000 0.000 0.000 orca.py:1568(get_raw_injectable)\n", + " 1 0.000 0.000 0.000 0.000 shape_base.py:218(_vhstack_dispatcher)\n", + " 4 0.000 0.000 0.000 0.000 cast.py:1711(sanitize_to_nanoseconds)\n", + " 2 0.000 0.000 0.000 0.000 {method 'transpose' of 'numpy.ndarray' objects}\n", + " 2 0.000 0.000 0.000 0.000 managers.py:217(is_single_block)\n", + " 1 0.000 0.000 0.000 0.000 common.py:1747(pandas_dtype)\n", + " 2 0.000 0.000 0.000 0.000 base.py:6987(ensure_index)\n", + " 1 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", + " 4 0.000 0.000 0.000 0.000 orca.py:1560(is_injectable)\n", + " 1 0.000 0.000 0.000 0.000 flags.py:47(__init__)\n", + " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_scalar}\n", + " 2 0.000 0.000 0.000 0.000 common.py:155()\n", + " 1 0.000 0.000 0.000 0.000 config.py:571(_select_options)\n", + " 4 0.000 0.000 0.000 0.000 blocks.py:354(dtype)\n", + " 4 0.000 0.000 0.000 0.000 _dtype.py:24(_kind_name)\n", + " 4 0.000 0.000 0.000 0.000 {method 'upper' of 'str' objects}\n", + " 2 0.000 0.000 0.000 0.000 base.py:4786(values)\n", + " 1 0.000 0.000 0.000 0.000 construction.py:233(mgr_to_mgr)\n", + " 2 0.000 0.000 0.000 0.000 base.py:518()\n", + " 2 0.000 0.000 0.000 0.000 base.py:834(_reset_identity)\n", + " 1 0.000 0.000 0.000 0.000 config.py:630(_translate_key)\n", + " 1 0.000 0.000 0.000 0.000 shape_base.py:207(_arrays_for_stack_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 base.py:1650(name)\n", + " 2 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", + " 3 0.000 0.000 0.000 0.000 common.py:145(classes)\n", + " 2 0.000 0.000 0.000 0.000 common.py:150(classes_and_not_datetimelike)\n", + " 2 0.000 0.000 0.000 0.000 blocks.py:244(mgr_locs)\n", + " 2 0.000 0.000 0.000 0.000 {built-in method __new__ of type object at 0x908780}\n", + " 2 0.000 0.000 0.000 0.000 {built-in method builtins.hash}\n", + " 2 0.000 0.000 0.000 0.000 multiarray.py:1071(copyto)\n", + " 1 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}\n", + " 2 0.000 0.000 0.000 0.000 {method 'setdefault' of 'dict' objects}\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:1862(_nonzero_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 {method 'astype' of 'numpy.ndarray' objects}\n", + " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_iterator}\n", + " 1 0.000 0.000 0.000 0.000 base.py:540(_ensure_array)\n", + " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", + " 1 0.000 0.000 0.000 0.000 {method 'keys' of 'dict' objects}\n", + " 1 0.000 0.000 0.000 0.000 managers.py:2111()\n", + " 1 0.000 0.000 0.000 0.000 multiarray.py:80(empty_like)\n", + " 1 0.000 0.000 0.000 0.000 managers.py:919(__init__)\n", + " 1 0.000 0.000 0.000 0.000 base.py:229(disallow_kwargs)\n", + " 1 0.000 0.000 0.000 0.000 numeric.py:72(_zeros_like_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 shape_base.py:77(_atleast_2d_dispatcher)\n", + " 1 0.000 0.000 0.000 0.000 fromnumeric.py:1008(_argsort_dispatcher)" ] }, - "execution_count": 40, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "utilities" + "%prun -s cumulative run_fru()" ] }, { "cell_type": "code", "execution_count": null, - "id": "c582bf26", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "display(sample_size)\n", - "sample_list[2].groupby('person_id').pick_count.sum()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de698ddc", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, + "id": "66c833e5-2ad3-4052-ae11-2aa887ad9a4f", + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "0c8cb9b5", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, + "id": "5e0f3f75-1150-4712-bfb4-435e85af8b52", + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "0301d604", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, + "id": "1fc21256-564e-4842-82fb-4ba5358fb644", + "metadata": {}, "outputs": [], "source": [] }, - { - "cell_type": "code", - "execution_count": 46, - "id": "60fb8dac", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(2, 6)\n", - "[[1 2 3 4 5 6]\n", - " [4 6 5 9 9 6]]\n", - "[5 3]\n" - ] - } - ], - "source": [ - "utils = np.array([[1,2,3,4,5,6],[4,6,5,9,9,6]])\n", - "print(utils.shape)\n", - "print(utils)\n", - "print(np.argmax(utils, axis=1))" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "id": "d43aeee6", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "ename": "TypeError", - "evalue": "_argmax_dispatcher() got an unexpected keyword argument 'keepdim'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_10839/398049723.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mutils\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36margmax\u001b[0;34m(*args, **kwargs)\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: _argmax_dispatcher() got an unexpected keyword argument 'keepdim'" - ] - } - ], - "source": [ - "utils[np.argmax(utils, axis=1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "id": "85892d45", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "utils[np.arange(2), np.argmax(utils, axis=1)] += 1" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "id": "477c7f3d", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "isinstance(utils, np.ndarray)" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "id": "a576a8a0", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,\n", - " 18, 19, 20, 21, 22, 23, 24, 25])" - ] - }, - "execution_count": 74, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dest_size_terms.index.values" - ] - }, { "cell_type": "code", "execution_count": null, - "id": "72c709ab", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, + "id": "ab332607-5f2e-4c7f-bfe4-5407c32644fb", + "metadata": {}, "outputs": [], "source": [] } From 6451abd8ac1dcc10158d0aa045a84359d1a71f6c Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 24 Jun 2022 10:24:33 +1000 Subject: [PATCH 090/135] use numpy's gumbel to draw from ev1 --- activitysim/core/interaction_sample.py | 10 ++- activitysim/core/logit.py | 25 +----- activitysim/core/random.py | 79 +++++++++++++++++++ .../frozen_impl_interaction_sample_dev.ipynb | 2 +- 4 files changed, 89 insertions(+), 27 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index d3838dde31..5e40f39dbb 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -13,7 +13,6 @@ from . import chunk from . import config from .simulate import set_skim_wrapper_targets -from .logit import inverse_ev1_cdf from . import interaction_simulate @@ -43,8 +42,13 @@ def make_sample_choices_utility_based( utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? for i in range(sample_size): - rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) - choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 + #rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) + #choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 + choices[ + zero_dim_index, + np.argmax(pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + utils_array, axis=1) + ] += 1 + return choices diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index a79be72041..fac9007948 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -183,31 +183,10 @@ def utils_to_probs(utils, trace_label=None, exponentiated=False, allow_zero_prob return probs -# OPTIMISE for runtime in interaction_sample - do not need scale nor location -def inverse_ev1_cdf(x): #, location=0.0, scale=1.0): - # quantile function of EV1 - # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution (and biogeme and larch convention) where the scale - # is proportional to variance (not variance^{-1}). this means nested scales are between 0 and 1. - # x can be number or np array or pd df for vecops - #return location - scale * np.log(-np.log(x)) - return -np.log(-np.log(x)) - - -# def add_ev1_random(df, nest_spec): -# # TODO: generalise to logit for nest_spec==None by adding one rand (scale=1) to all columns -# nest_utils_for_choice = df.copy() -# for n in each_nest(nest_spec): -# if n.level == 1: -# continue # skip the root level, not needed -# uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice) -# rands = inverse_ev1_cdf(uniform_rands) -# nest_utils_for_choice.loc[:, n.name] += rands[:, 0] # inverse_ev1_cdf of single-row df adds dimension -# return nest_utils_for_choice + def add_ev1_random(df): nest_utils_for_choice = df.copy() - uniform_rands = pipeline.get_rn_generator().random_for_df(nest_utils_for_choice, n=df.shape[1]) - rands = inverse_ev1_cdf(uniform_rands) - nest_utils_for_choice += rands + nest_utils_for_choice += pipeline.get_rn_generator().gumbel_for_df(nest_utils_for_choice, n=df.shape[1]) return nest_utils_for_choice diff --git a/activitysim/core/random.py b/activitysim/core/random.py index a9c9770340..548b1d7b84 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -196,6 +196,8 @@ def _generators_for_df(self, df): df_row_states = self.row_states.loc[df.index] + # https://numpy.org/doc/stable/reference/random/generator.html + # np.random.default_rng() prng = np.random.RandomState() for row in df_row_states.itertuples(): @@ -248,6 +250,47 @@ def random_for_df(self, df, step_name, n=1): self.row_states.loc[df.index, 'offset'] += n return rands + def gumbel_for_df(self, df, step_name, n=1): + """ + Return n floating point gumbel-distributed numbers for each row in df + using the appropriate random channel for each row. + + Subsequent calls (in the same step) will return the next rand for each df row + + The resulting array will be the same length (and order) as df + This method is designed to support alternative selection from a probability array + + The columns in df are ignored; the index name and values are used to determine + which random number sequence to to use. + + If "true pseudo random" behavior is desired (i.e. NOT repeatable) the set_base_seed + method (q.v.) may be used to globally reseed all random streams. + + Parameters + ---------- + df : pandas.DataFrame + df with index name and values corresponding to a registered channel + + n : int + number of rands desired per df row + + Returns + ------- + rands : 2-D ndarray + array the same length as df, with n floats in range [0, 1) for each df row + """ + + assert self.step_name + assert self.step_name == step_name + + # - reminder: prng must be called when yielded as generated sequence, not serialized + generators = self._generators_for_df(df) + + rands = np.asanyarray([prng.gumbel(size=n) for prng in generators]) + # update offset for rows we handled + self.row_states.loc[df.index, 'offset'] += n + return rands + def normal_for_df(self, df, step_name, mu, sigma, lognormal=False): """ Return a floating point random number in normal (or lognormal) distribution @@ -602,6 +645,42 @@ def random_for_df(self, df, n=1): rands = channel.random_for_df(df, self.step_name, n) return rands + def gumbel_for_df(self, df, n=1): + """ + Return a single floating point gumbel for each row in df + using the appropriate random channel for each row. + + Subsequent calls (in the same step) will return the next rand for each df row + + The resulting array will be the same length (and order) as df + This method is designed to support alternative selection from a probability array + + The columns in df are ignored; the index name and values are used to determine + which random number sequence to to use. + + We assume that we can identify the channel to used based on the name of df.index + This channel should have already been registered by a call to add_channel (q.v.) + + If "true pseudo random" behavior is desired (i.e. NOT repeatable) the set_base_seed + method (q.v.) may be used to globally reseed all random streams. + + Parameters + ---------- + df : pandas.DataFrame + df with index name and values corresponding to a registered channel + + n : int + number of rands desired (default 1) + + Returns + ------- + choices : 1-D ndarray the same length as df + a single float in range [0, 1) for each row in df + """ + channel = self.get_channel_for_df(df) + rands = channel.gumbel_for_df(df, self.step_name, n) + return rands + def normal_for_df(self, df, mu=0, sigma=1, broadcast=False): """ Return a single floating point normal random number in range (-inf, inf) for each row in df diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index cf210fcccd..7980bfb87e 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -1658,4 +1658,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From 3973bcde0f590408ab4fdc3587e1b04ebb758ac9 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 24 Jun 2022 10:34:24 +1000 Subject: [PATCH 091/135] clean up --- activitysim/core/logit.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index fac9007948..b4eaa56e43 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -202,6 +202,8 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter raise ValueError("This should never happen - no alternative found") +# TODO [janzill Jun2022]: make everything in nested and here numpy from beginning to make choices consistent with +# previous impl (want column index and not alternative name) # Note: this is relatively slow due to the apply. # It could *maybe* be sped up by using the fact that the nesting structure is the same for all rows: Add ev1(0,1) to # all entries (as is currently being done). Then, at each level, pick the maximum of the available composite @@ -226,15 +228,10 @@ def make_choices_ru_frozen_nl(nested_utilities, alt_order_array, nest_spec): # In order for choice indexing to be consistent with MNL and cumsum MC choices, we need to index in the order # alternatives were originally created before adding nest nodes that are not elemental alternatives choices = choices.map({v: k for k, v in enumerate(alt_order_array)}) - ## the following is wrong, the order might be changed - #choices = choices.map({v: k for k, v in enumerate(nest_utils_for_choice.columns)}) return choices -# TODO [janzill Jun2022]: integrate with nested impl above -# TODO [janzill Jun2022]: make everything in nested and here numpy from beginning to make choices consistent with -# previous impl (want column index and not alternative name) def make_choices_ru_frozen_mnl(utilities): utilities_incl_unobs = add_ev1_random(utilities) choices = np.argmax(utilities_incl_unobs.to_numpy(), axis=1) @@ -242,8 +239,9 @@ def make_choices_ru_frozen_mnl(utilities): choices = pd.Series(choices, index=utilities_incl_unobs.index) return choices + def make_choices_ru_frozen(utilities, alt_order_array, nest_spec=None, trace_label=None): - trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen_mnl') + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') if nest_spec is None: choices = make_choices_ru_frozen_mnl(utilities) else: From c0d1ca17ab4fa8d5bd643a97687e21a8af8835dd Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 24 Jun 2022 16:04:36 +1000 Subject: [PATCH 092/135] gumbel by hand by inverse cdf --- activitysim/core/random.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 548b1d7b84..54def227bd 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -286,7 +286,10 @@ def gumbel_for_df(self, df, step_name, n=1): # - reminder: prng must be called when yielded as generated sequence, not serialized generators = self._generators_for_df(df) - rands = np.asanyarray([prng.gumbel(size=n) for prng in generators]) + # this is up to 20% faster for large arrays, like for destination choice + #rands = np.asanyarray([prng.gumbel(size=n) for prng in generators]) + rands = np.asanyarray([-np.log(-np.log(prng.rand(n))) for prng in generators]) + # update offset for rows we handled self.row_states.loc[df.index, 'offset'] += n return rands From 6b0b4e93e575bd5d31dd6930b5b526c74fe91be5 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 27 Jun 2022 11:13:44 +1000 Subject: [PATCH 093/135] memory logging --- activitysim/core/interaction_sample.py | 109 ++++++++++-------- activitysim/core/random.py | 2 +- .../frozen_impl_interaction_sample_dev.ipynb | 51 ++++---- 3 files changed, 87 insertions(+), 75 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 5e40f39dbb..441d3236b8 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -28,29 +28,48 @@ def make_sample_choices_utility_based( utilities, sample_size, alternative_count, - allow_zero_probs + allow_zero_probs, + trace_label ): assert isinstance(utilities, pd.DataFrame) assert utilities.shape == (len(choosers), alternative_count) - #choice_dimension = (len(choosers), alternative_count, sample_size) # Note [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension # (len(choosers), alternative_count, sample_size) can get very large - choices = np.zeros_like(utilities, dtype=np.uint32) - zero_dim_index = np.arange(utilities.shape[0]) + #choices = np.zeros_like(utilities, dtype=np.uint32) + #zero_dim_index = np.arange(utilities.shape[0]) + + utils_array = utilities.to_numpy() + chunk.log_df(trace_label, 'utils_array', utils_array) + chosen_destinations = [] + + rands = pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + chunk.log_df(trace_label, 'rands', rands) - utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each? for i in range(sample_size): #rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) #choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 - choices[ - zero_dim_index, - np.argmax(pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + utils_array, axis=1) - ] += 1 + #choices[ + # zero_dim_index, + # np.argmax(pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + utils_array, axis=1) + #] += 1 + # created this once for memory logging + if i > 0: + rands = pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + chosen_destinations.append(np.argmax(utils_array + rands, axis=1)) + chosen_destinations = np.concatenate(chosen_destinations, axis=0) + + chunk.log_df(trace_label, 'chosen_destinations', chosen_destinations) + + del utils_array + chunk.log_df(trace_label, 'utils_array', None) + del rands + chunk.log_df(trace_label, 'rands', None) + chunk.log_df(trace_label, 'chosen_destinations', None) # handing off to caller - return choices + return chosen_destinations def make_sample_choices( @@ -341,16 +360,16 @@ def _interaction_sample( # sample size 0 is for estimation mode - see below if config.setting("freeze_unobserved_utilities", False) and (sample_size != 0): - choices_array = make_sample_choices_utility_based( - choosers, utilities, sample_size, alternative_count, allow_zero_probs + chosen_alts = make_sample_choices_utility_based( + choosers, utilities, sample_size, alternative_count, allow_zero_probs, trace_label ) - chunk.log_df(trace_label, 'choices_array', choices_array) - + chooser_idx = np.tile(np.arange(utilities.shape[0]), sample_size) + #chunk.log_df(trace_label, 'choices_array', choices_array) # choices array has same dim as utilities, with values indicating number of counts per chooser and alternative # let's turn the nonzero values into a dataframe - i, j = np.nonzero(choices_array) - chunk.log_df(trace_label, 'i', i) - chunk.log_df(trace_label, 'j', j) + #i, j = np.nonzero(choices_array) + chunk.log_df(trace_label, 'chooser_idx', chooser_idx) + chunk.log_df(trace_label, 'chosen_alts', chosen_alts) probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs, trace_label=trace_label, trace_choosers=choosers) @@ -360,21 +379,16 @@ def _interaction_sample( chunk.log_df(trace_label, 'utilities', None) choices_df = pd.DataFrame({ - alt_col_name: alternatives.index.values[j], - "pick_count": choices_array[i, j], - "prob": probs.to_numpy()[i, j], - choosers.index.name: choosers.index.values[i] + alt_col_name: alternatives.index.values[chosen_alts], + #"pick_count": choices_array[i, j], + "prob": probs.to_numpy()[chooser_idx, chosen_alts], + choosers.index.name: choosers.index.values[chooser_idx] }) - chunk.log_df(trace_label, 'choices_df', choices_df) - - del choices_array - chunk.log_df(trace_label, 'choices_array', None) - del i - chunk.log_df(trace_label, 'i', None) - del j - chunk.log_df(trace_label, 'j', None) - del probs - chunk.log_df(trace_label, 'probs', None) + del chooser_idx + chunk.log_df(trace_label, 'chooser_idx', None) + del chosen_alts + chunk.log_df(trace_label, 'chosen_alts', None) + else: # convert to probabilities (utilities exponentiated and normalized to probs) # probs is same shape as utilities, one row per chooser and one column for alternative @@ -412,26 +426,26 @@ def _interaction_sample( allow_zero_probs=allow_zero_probs, trace_label=trace_label) - chunk.log_df(trace_label, 'choices_df', choices_df) + chunk.log_df(trace_label, 'choices_df', choices_df) - del probs - chunk.log_df(trace_label, 'probs', None) + del probs + chunk.log_df(trace_label, 'probs', None) - # pick_count and pick_dup - # pick_count is number of duplicate picks - # pick_dup flag is True for all but first of duplicates - pick_group = choices_df.groupby([choosers.index.name, alt_col_name]) + # pick_count and pick_dup + # pick_count is number of duplicate picks + # pick_dup flag is True for all but first of duplicates + pick_group = choices_df.groupby([choosers.index.name, alt_col_name]) - # number each item in each group from 0 to the length of that group - 1. - choices_df['pick_count'] = pick_group.cumcount(ascending=True) - # flag duplicate rows after first - choices_df['pick_dup'] = choices_df['pick_count'] > 0 - # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge) - choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1 + # number each item in each group from 0 to the length of that group - 1. + choices_df['pick_count'] = pick_group.cumcount(ascending=True) + # flag duplicate rows after first + choices_df['pick_dup'] = choices_df['pick_count'] > 0 + # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge) + choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1 - # drop the duplicates - choices_df = choices_df[~choices_df['pick_dup']] - del choices_df['pick_dup'] + # drop the duplicates + choices_df = choices_df[~choices_df['pick_dup']] + del choices_df['pick_dup'] # set index after groupby so we can trace on it choices_df.set_index(choosers.index.name, inplace=True) @@ -449,7 +463,6 @@ def _interaction_sample( # don't need this after tracing if not config.setting("freeze_unobserved_utilities", False): del choices_df['rand'] - chunk.log_df(trace_label, 'choices_df', choices_df) # - NARROW choices_df['prob'] = choices_df['prob'].astype(np.float32) diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 54def227bd..a92dfc44d9 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -286,8 +286,8 @@ def gumbel_for_df(self, df, step_name, n=1): # - reminder: prng must be called when yielded as generated sequence, not serialized generators = self._generators_for_df(df) - # this is up to 20% faster for large arrays, like for destination choice #rands = np.asanyarray([prng.gumbel(size=n) for prng in generators]) + # this is about 20% faster for large arrays, like for destination choice rands = np.asanyarray([-np.log(-np.log(prng.rand(n))) for prng in generators]) # update offset for rows we handled diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb index 7980bfb87e..79412c9265 100644 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ b/notebooks/frozen_impl_interaction_sample_dev.ipynb @@ -76,14 +76,26 @@ "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '/home/jan/code/activitysim/test_runs/seq_asim'", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mFileNotFoundError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/tmp/ipykernel_24910/2284515201.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mos\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchdir\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mexample_dir\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", + "\u001B[0;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: '/home/jan/code/activitysim/test_runs/seq_asim'" + ] + } + ], "source": [ "os.chdir(example_dir)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", "metadata": { "tags": [] @@ -101,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9955cc9c", "metadata": { "collapsed": false, @@ -277,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 7, "id": "7599d0d1-9063-48aa-95de-cef45e926a42", "metadata": {}, "outputs": [], @@ -419,7 +431,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "id": "1611fe26", "metadata": { "collapsed": false, @@ -432,27 +444,14 @@ }, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle school_location not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "running university, 3\n", - "dropping 4288 of 4380 rows where size_term is zero\n", - "Running school_location with 5377 persons\n", - "running highschool, 2\n", - "dropping 4000 of 4380 rows where size_term is zero\n", - "Running school_location with 2888 persons\n", - "running gradeschool, 1\n", - "dropping 200 of 4380 rows where size_term is zero\n", - "Running school_location with 13471 persons\n", - "CPU times: user 3min 15s, sys: 1min 17s, total: 4min 32s\n", - "Wall time: 4min 19s\n" + "ename": "NameError", + "evalue": "name 'pipeline' is not defined", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n", + "\u001B[0;31mNameError\u001B[0m: name 'pipeline' is not defined" ] } ], From 792d183eb76010803c477e5c61fead77c71205f2 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 29 Mar 2025 17:24:25 +1000 Subject: [PATCH 094/135] merge bug --- activitysim/core/interaction_sample_simulate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index d7b9a78875..74f32b570c 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -283,7 +283,7 @@ def _interaction_sample_simulate( utilities_df, trace_label=trace_label, trace_choosers=choosers ) del utilities_df - chunk.log_df(trace_label, "utilities_df", None) + chunk_sizer.log_df(trace_label, "utilities_df", None) else: # convert to probabilities (utilities exponentiated and normalized to probs) # probs is same shape as utilities, one row per chooser and one column for alternative @@ -310,7 +310,7 @@ def _interaction_sample_simulate( chunk_sizer.log_df(trace_label, "probs", probs) del utilities_df - chunk.log_df(trace_label, "utilities_df", None) + chunk_sizer.log_df(trace_label, "utilities_df", None) if have_trace_targets: state.tracing.trace_df( From 08d013ab1fcfbdb944062b657e0b68fc38d711aa Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 29 Mar 2025 17:49:10 +1000 Subject: [PATCH 095/135] more merge bugs --- activitysim/core/interaction_sample.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index e1138abb31..8c3645da9f 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -81,8 +81,9 @@ def make_sample_choices_utility_based( #i, j = np.nonzero(choices_array) chunk_sizer.log_df(trace_label, 'chooser_idx', chooser_idx) - probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs, - trace_label=trace_label, trace_choosers=choosers) + probs = logit.utils_to_probs( + state, utilities, allow_zero_probs=allow_zero_probs, trace_label=trace_label, trace_choosers=choosers + ) chunk_sizer.log_df(trace_label, 'probs', probs) choices_df = pd.DataFrame({ From 75bb84d9423134ee5781bd61433db10793e0d035 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 29 Mar 2025 17:56:50 +1000 Subject: [PATCH 096/135] more merge fixes, example runs through --- activitysim/core/interaction_sample_simulate.py | 2 +- activitysim/core/logit.py | 2 +- activitysim/core/simulate.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 74f32b570c..0b112ce1f2 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -280,7 +280,7 @@ def _interaction_sample_simulate( # positions is series with the chosen alternative represented as a column index in utilities_df # which is an integer between zero and num alternatives in the alternative sample positions, rands = logit.make_choices_utility_based( - utilities_df, trace_label=trace_label, trace_choosers=choosers + state, utilities_df, trace_label=trace_label, trace_choosers=choosers ) del utilities_df chunk_sizer.log_df(trace_label, "utilities_df", None) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 5465f8e0c9..2a716a0acb 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -330,7 +330,7 @@ def make_choices_explicit_error_term_mnl(state, utilities): def make_choices_explicit_error_term(state, utilities, alt_order_array, nest_spec=None, trace_label=None): - trace_label = state.tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') + trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') if nest_spec is None: choices = make_choices_explicit_error_term_mnl(state, utilities) else: diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index e2c70b08c3..a395236624 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1304,7 +1304,7 @@ def eval_mnl( if state.settings.use_explicit_error_terms: if custom_chooser: - choices, rands = custom_chooser(state, utilities=utilities, choosers=choosers, spec=spec, trace_label=trace_label) + choices, rands = custom_chooser(state, utilities, choosers, spec, trace_label) else: choices, rands = logit.make_choices_utility_based(state, utilities, trace_label=trace_label) From c6feaefa22ce60dcf4d9557066aa095e61a1a4a5 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 29 Mar 2025 18:17:34 +1000 Subject: [PATCH 097/135] log eet setting on start up --- activitysim/core/configuration/top.py | 1 + 1 file changed, 1 insertion(+) diff --git a/activitysim/core/configuration/top.py b/activitysim/core/configuration/top.py index 2873cef760..486c5b78f4 100644 --- a/activitysim/core/configuration/top.py +++ b/activitysim/core/configuration/top.py @@ -647,6 +647,7 @@ def _check_store_skims_in_shm(self): "memory_profile", "instrument", "sharrow", + "use_explicit_error_terms", ) """ Setting to log on startup. From 114b747b2e9615a53c7cc453557f7ab07088372d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 2 Apr 2025 14:37:12 +1000 Subject: [PATCH 098/135] some variation checks, dev notebooks, to delete --- notebooks/check_sf_probs.ipynb | 396 -- notebooks/example_dev-Copy1.ipynb | 4295 ----------------- notebooks/frozen_impl_dev.ipynb | 661 --- .../frozen_impl_interaction_sample_dev.ipynb | 1660 ------- notebooks/frozen_rand_mode_choice_dev.ipynb | 3418 ------------- notebooks/fru_utils.py | 582 --- .../larch_probabilities_via_estimation.ipynb | 592 --- ...ested_logit_frozen_rand_individ_util.ipynb | 440 -- notebooks/validate_frozen_impl.ipynb | 1651 ------- 9 files changed, 13695 deletions(-) delete mode 100644 notebooks/check_sf_probs.ipynb delete mode 100644 notebooks/example_dev-Copy1.ipynb delete mode 100644 notebooks/frozen_impl_dev.ipynb delete mode 100644 notebooks/frozen_impl_interaction_sample_dev.ipynb delete mode 100644 notebooks/frozen_rand_mode_choice_dev.ipynb delete mode 100644 notebooks/fru_utils.py delete mode 100644 notebooks/larch_probabilities_via_estimation.ipynb delete mode 100644 notebooks/nested_logit_frozen_rand_individ_util.ipynb delete mode 100644 notebooks/validate_frozen_impl.ipynb diff --git a/notebooks/check_sf_probs.ipynb b/notebooks/check_sf_probs.ipynb deleted file mode 100644 index eb5c355cad..0000000000 --- a/notebooks/check_sf_probs.ipynb +++ /dev/null @@ -1,396 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", - "metadata": {}, - "source": [ - "# Compare results against larch\n", - "\n", - "We use estimation data as prepared by larch and draw the Asim way and our way, then compare to larch probabilities.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.385153Z", - "iopub.status.busy": "2022-05-01T10:17:57.384881Z", - "iopub.status.idle": "2022-05-01T10:17:57.534433Z", - "shell.execute_reply": "2022-05-01T10:17:57.533096Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%load_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "c3dba451-1e10-403e-8614-35d57e6577f4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.536623Z", - "iopub.status.busy": "2022-05-01T10:17:57.536012Z", - "iopub.status.idle": "2022-05-01T10:17:57.542755Z", - "shell.execute_reply": "2022-05-01T10:17:57.541685Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:00.350944Z", - "iopub.status.busy": "2022-05-01T10:18:00.350730Z", - "iopub.status.idle": "2022-05-01T10:18:12.760977Z", - "shell.execute_reply": "2022-05-01T10:18:12.760013Z", - "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import argparse\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from activitysim.cli import run\n", - "from activitysim.core import inject\n", - "\n", - "from fru_utils import mode_choice_for_trip\n", - "\n", - "pd.set_option(\"max_columns\", 500)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:17.839947Z", - "iopub.status.busy": "2022-05-01T10:18:17.839070Z", - "iopub.status.idle": "2022-05-01T10:18:18.019676Z", - "shell.execute_reply": "2022-05-01T10:18:18.018689Z", - "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_example_sf\")\n", - "os.chdir(example_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "outputs": [], - "source": [ - "#!activitysim run -c configs -o output -d data_sf" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:22.111723Z", - "iopub.status.busy": "2022-05-01T10:18:22.111490Z", - "iopub.status.idle": "2022-05-01T10:18:22.297437Z", - "shell.execute_reply": "2022-05-01T10:18:22.296501Z", - "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "parser = argparse.ArgumentParser()\n", - "run.add_run_args(parser)\n", - "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data_sf'])\n", - "if not inject.is_injectable('preload_injectables'):\n", - " from activitysim import abm # register abm steps and other abm-specific injectables\n", - "run.handle_standard_args(args) # possibly update injectables" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "outputs": [], - "source": [ - "larch_chooser_data = pd.read_csv(os.path.join(root_dir, \"test_estimation_sf\", \"choosers_larch.csv\"))\n", - "larch_probs = pd.read_csv(os.path.join(root_dir, \"test_estimation_sf\", \"probabilities_larch.csv\"), index_col=0)\n", - "# __caseids__ are tour ids, with trips in same order\n", - "assert (larch_probs.index.values == larch_chooser_data.tour_id).all()\n", - "larch_probs.index = larch_chooser_data.trip_id" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 50, - "outputs": [], - "source": [ - "larch_tours = pd.read_csv(os.path.join(root_dir, \"test_estimation_sf\", \"output\", \"estimation_data_bundle\",\n", - " \"trip_mode_choice\", \"trip_mode_choice_values_combined.csv\"))\n", - "trips_merged_cols = ['person_id', 'household_id', 'primary_purpose', 'trip_num', 'outbound',\n", - " 'trip_count', 'destination', 'origin', 'tour_id', 'purpose',\n", - " 'destination_logsum', 'depart', 'hhsize', 'age', 'auto_ownership',\n", - " 'number_of_participants', 'tour_category', 'parent_tour_id',\n", - " 'tour_mode', 'duration', 'value_of_time', 'tour_type',\n", - " 'free_parking_at_work', 'trip_period', \"trip_id\"]\n", - "\n", - "larch_trips_merged = larch_tours[trips_merged_cols].set_index(\"trip_id\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 87, - "outputs": [], - "source": [ - "def compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs):\n", - " t_, c_, ns_ = mode_choice_for_trip(choose_individual_max_utility=True, trip_id_to_check=trip_id_to_check,\n", - " num_samples=num_samples, trips_merged=larch_trips_merged)\n", - " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", - "\n", - " t_asim, c_asim, _ = mode_choice_for_trip(choose_individual_max_utility=False, trip_id_to_check=trip_id_to_check,\n", - " num_samples=num_samples, trips_merged=larch_trips_merged)\n", - " asim_mode_shares = c_asim.trip_mode.value_counts() / c_asim.shape[0]\n", - "\n", - " larch_mode_shares = larch_probs.loc[larch_probs.index == trip_id_to_check].T\n", - " larch_mode_shares.columns = [\"larch\"]\n", - "\n", - " comp = sim_mode_shares.to_frame(\"me\")\\\n", - " .merge(asim_mode_shares.to_frame(\"asim\"), left_index=True, right_index=True,how=\"outer\")\\\n", - " .merge(larch_mode_shares,left_index=True, right_index=True, how=\"outer\").fillna(0)\n", - "\n", - " return comp" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 124, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n23695361 0.000000 0.0 2.027363e-06 0.0 \n23695365 0.000000 0.0 2.943297e-09 0.0 \n23760961 0.999664 0.0 0.000000e+00 0.0 \n23760962 0.998934 0.0 0.000000e+00 0.0 \n23760963 0.997988 0.0 0.000000e+00 0.0 \n... ... ... ... ... \n2464724829 0.000000 0.0 3.268925e-05 0.0 \n2464847169 0.000000 0.0 2.321209e-03 0.0 \n2464847173 0.000000 0.0 1.269812e-03 0.0 \n2464980993 0.000000 0.0 8.280461e-05 0.0 \n2464980997 0.000000 0.0 6.298188e-05 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n23695361 1.061530e-06 0.0 0.111823 0.0 0.263130 0.625043 \n23695365 1.967559e-08 0.0 0.119603 0.0 0.272258 0.608139 \n23760961 0.000000e+00 0.0 0.000010 0.0 0.000000 0.000000 \n23760962 0.000000e+00 0.0 0.000579 0.0 0.000000 0.000000 \n23760963 0.000000e+00 0.0 0.001509 0.0 0.000000 0.000000 \n... ... ... ... ... ... ... \n2464724829 3.022226e-05 0.0 0.691838 0.0 0.308044 0.000000 \n2464847169 1.213314e-03 0.0 0.249261 0.0 0.745641 0.000000 \n2464847173 7.535248e-04 0.0 0.186394 0.0 0.810411 0.000000 \n2464980993 4.326905e-05 0.0 0.003217 0.0 0.996572 0.000000 \n2464980997 3.791076e-05 0.0 0.003749 0.0 0.996046 0.000000 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n23695361 0.0 0.0 0.0 0.0 0.0 0.0 \n23695365 0.0 0.0 0.0 0.0 0.0 0.0 \n23760961 0.0 0.0 0.0 0.0 0.0 0.0 \n23760962 0.0 0.0 0.0 0.0 0.0 0.0 \n23760963 0.0 0.0 0.0 0.0 0.0 0.0 \n... ... ... ... ... ... ... \n2464724829 0.0 0.0 0.0 0.0 0.0 0.0 \n2464847169 0.0 0.0 0.0 0.0 0.0 0.0 \n2464847173 0.0 0.0 0.0 0.0 0.0 0.0 \n2464980993 0.0 0.0 0.0 0.0 0.0 0.0 \n2464980997 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n23695361 0.0 0.0 2.489336e-08 4.965881e-08 4.072015e-07 \n23695365 0.0 0.0 2.435815e-08 4.996681e-08 4.311721e-07 \n23760961 0.0 0.0 4.724941e-05 1.754653e-04 1.035425e-04 \n23760962 0.0 0.0 1.066593e-04 2.780937e-04 1.021060e-04 \n23760963 0.0 0.0 1.039326e-04 2.850044e-04 1.143830e-04 \n... ... ... ... ... ... \n2464724829 0.0 0.0 3.606576e-05 4.836729e-07 1.874068e-05 \n2464847169 0.0 0.0 4.274883e-04 5.407950e-04 5.947334e-04 \n2464847173 0.0 0.0 3.063342e-04 4.098183e-04 4.563525e-04 \n2464980993 0.0 0.0 1.593300e-05 4.967854e-05 1.954461e-05 \n2464980997 0.0 0.0 2.239090e-05 6.033135e-05 2.206105e-05 \n\n[5428 rows x 21 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
236953610.0000000.02.027363e-060.01.061530e-060.00.1118230.00.2631300.6250430.00.00.00.00.00.00.00.02.489336e-084.965881e-084.072015e-07
236953650.0000000.02.943297e-090.01.967559e-080.00.1196030.00.2722580.6081390.00.00.00.00.00.00.00.02.435815e-084.996681e-084.311721e-07
237609610.9996640.00.000000e+000.00.000000e+000.00.0000100.00.0000000.0000000.00.00.00.00.00.00.00.04.724941e-051.754653e-041.035425e-04
237609620.9989340.00.000000e+000.00.000000e+000.00.0005790.00.0000000.0000000.00.00.00.00.00.00.00.01.066593e-042.780937e-041.021060e-04
237609630.9979880.00.000000e+000.00.000000e+000.00.0015090.00.0000000.0000000.00.00.00.00.00.00.00.01.039326e-042.850044e-041.143830e-04
..................................................................
24647248290.0000000.03.268925e-050.03.022226e-050.00.6918380.00.3080440.0000000.00.00.00.00.00.00.00.03.606576e-054.836729e-071.874068e-05
24648471690.0000000.02.321209e-030.01.213314e-030.00.2492610.00.7456410.0000000.00.00.00.00.00.00.00.04.274883e-045.407950e-045.947334e-04
24648471730.0000000.01.269812e-030.07.535248e-040.00.1863940.00.8104110.0000000.00.00.00.00.00.00.00.03.063342e-044.098183e-044.563525e-04
24649809930.0000000.08.280461e-050.04.326905e-050.00.0032170.00.9965720.0000000.00.00.00.00.00.00.00.01.593300e-054.967854e-051.954461e-05
24649809970.0000000.06.298188e-050.03.791076e-050.00.0037490.00.9960460.0000000.00.00.00.00.00.00.00.02.239090e-056.033135e-052.206105e-05
\n

5428 rows × 21 columns

\n
" - }, - "execution_count": 124, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "larch_probs" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 144, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'work' (100000 trips)\n", - "Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'work' (100000 trips)\n", - "Done\n", - "CPU times: user 1min 10s, sys: 3.98 s, total: 1min 14s\n", - "Wall time: 1min 9s\n" - ] - }, - { - "data": { - "text/plain": " me asim larch\nSHARED2FREE 0.395% 0.420% 0.359%\nSHARED3FREE 0.214% 0.226% 0.188%\nTNC_SINGLE 0.212% 0.231% 0.202%\nWALK 0.488% 0.466% 0.435%\nWALK_HVY 69.439% 69.422% 60.165%\nWALK_LOC 9.697% 9.596% 14.513%\nWALK_LRF 19.390% 19.469% 23.977%", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
measimlarch
SHARED2FREE0.395%0.420%0.359%
SHARED3FREE0.214%0.226%0.188%
TNC_SINGLE0.212%0.231%0.202%
WALK0.488%0.466%0.435%
WALK_HVY69.439%69.422%60.165%
WALK_LOC9.697%9.596%14.513%
WALK_LRF19.390%19.469%23.977%
\n
" - }, - "execution_count": 144, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "\n", - "trip_id_to_check = 24106017 # obvious results with 1e4 samples: 23695361 24106017\n", - "# better with 1e5 samples: 1932588125\n", - "# requires higher precision, check when time: 38280865\n", - "num_samples = 100000 # 1e5 takes about 80s\n", - "\n", - "comp = compare_mode_share_simulation(trip_id_to_check, num_samples, larch_trips_merged, larch_probs)\n", - "comp.loc[(comp > 1e-3).any(axis=1), :].applymap(lambda x: f\"{x:.3%}\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 139, - "outputs": [], - "source": [ - "#comp.loc[(comp > 0).any(axis=1), :]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 140, - "outputs": [ - { - "data": { - "text/plain": "0.252" - }, - "execution_count": 140, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "0.72 * 0.35" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "# check what happens if you multiply them - like Train at the lower level? but then 0\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
person_idhousehold_idtour_idprimary_purposetrip_numoutboundtrip_countpurposedestinationorigindestination_logsumdeparttrip_modemode_choice_logsum
trip_id
13724872141844130403617156090escort1True1escort710NaN7.0WALK11.435800
13724872541844130403617156090escort1False1Home107NaN7.0WALK11.480440
21138820164447638676126423525escort1True1escort1116NaN5.0WALK_LOC4.789158
21138820564447638676126423525escort1False1Home1611NaN6.0WALK_LOC5.050171
80638840124585011173905100798550escort1True1escort168NaN15.0WALK_LOC6.451457
80638840524585011173905100798550escort1False1Home816NaN16.0WALK_LOC6.446188
\n", - "" - ], - "text/plain": [ - " person_id household_id tour_id primary_purpose trip_num \\\n", - "trip_id \n", - "137248721 418441 304036 17156090 escort 1 \n", - "137248725 418441 304036 17156090 escort 1 \n", - "211388201 644476 386761 26423525 escort 1 \n", - "211388205 644476 386761 26423525 escort 1 \n", - "806388401 2458501 1173905 100798550 escort 1 \n", - "806388405 2458501 1173905 100798550 escort 1 \n", - "\n", - " outbound trip_count purpose destination origin \\\n", - "trip_id \n", - "137248721 True 1 escort 7 10 \n", - "137248725 False 1 Home 10 7 \n", - "211388201 True 1 escort 11 16 \n", - "211388205 False 1 Home 16 11 \n", - "806388401 True 1 escort 16 8 \n", - "806388405 False 1 Home 8 16 \n", - "\n", - " destination_logsum depart trip_mode mode_choice_logsum \n", - "trip_id \n", - "137248721 NaN 7.0 WALK 11.435800 \n", - "137248725 NaN 7.0 WALK 11.480440 \n", - "211388201 NaN 5.0 WALK_LOC 4.789158 \n", - "211388205 NaN 6.0 WALK_LOC 5.050171 \n", - "806388401 NaN 15.0 WALK_LOC 6.451457 \n", - "806388405 NaN 16.0 WALK_LOC 6.446188 " - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trips_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ca3206ce-df97-42d9-83ce-dc4067ea5675", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d6c2752e-68a8-4319-85c2-fe6bf1fd6d59", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3851df2c-bb8c-43b6-8a27-3b04a7f85a50", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b596af94-85d1-4d25-99b2-c19292397882", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "8eebea31-499c-41ca-8411-883a88ca800a", - "metadata": {}, - "source": [ - "## nested dev" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:01.113060Z", - "iopub.status.busy": "2021-08-31T10:31:01.112829Z", - "iopub.status.idle": "2021-08-31T10:31:01.359912Z", - "shell.execute_reply": "2021-08-31T10:31:01.358948Z", - "shell.execute_reply.started": "2021-08-31T10:31:01.113034Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "spec = simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - "nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "36c100b3-3e39-4950-a586-4d42be695eaa", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:01.775068Z", - "iopub.status.busy": "2021-08-31T10:31:01.774846Z", - "iopub.status.idle": "2021-08-31T10:31:01.975816Z", - "shell.execute_reply": "2021-08-31T10:31:01.974499Z", - "shell.execute_reply.started": "2021-08-31T10:31:01.775044Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'name': 'root',\n", - " 'coefficient': 1.0,\n", - " 'alternatives': [{'name': 'AUTO',\n", - " 'coefficient': 0.72,\n", - " 'alternatives': [{'name': 'DRIVEALONE',\n", - " 'coefficient': 0.35,\n", - " 'alternatives': ['DRIVEALONEFREE', 'DRIVEALONEPAY']},\n", - " {'name': 'SHAREDRIDE2',\n", - " 'coefficient': 0.35,\n", - " 'alternatives': ['SHARED2FREE', 'SHARED2PAY']},\n", - " {'name': 'SHAREDRIDE3',\n", - " 'coefficient': 0.35,\n", - " 'alternatives': ['SHARED3FREE', 'SHARED3PAY']}]},\n", - " {'name': 'NONMOTORIZED',\n", - " 'coefficient': 0.72,\n", - " 'alternatives': ['WALK', 'BIKE']},\n", - " {'name': 'TRANSIT',\n", - " 'coefficient': 0.72,\n", - " 'alternatives': [{'name': 'WALKACCESS',\n", - " 'coefficient': 0.5,\n", - " 'alternatives': ['WALK_LOC',\n", - " 'WALK_LRF',\n", - " 'WALK_EXP',\n", - " 'WALK_HVY',\n", - " 'WALK_COM']},\n", - " {'name': 'DRIVEACCESS',\n", - " 'coefficient': 0.5,\n", - " 'alternatives': ['DRIVE_LOC',\n", - " 'DRIVE_LRF',\n", - " 'DRIVE_EXP',\n", - " 'DRIVE_HVY',\n", - " 'DRIVE_COM']}]},\n", - " {'name': 'RIDEHAIL',\n", - " 'coefficient': 0.36,\n", - " 'alternatives': ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']}]}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nest_spec" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "f442cf50-e85c-4991-af39-68a6839b146e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:03.459794Z", - "iopub.status.busy": "2021-08-31T10:31:03.459560Z", - "iopub.status.idle": "2021-08-31T10:31:03.660759Z", - "shell.execute_reply": "2021-08-31T10:31:03.658857Z", - "shell.execute_reply.started": "2021-08-31T10:31:03.459767Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nest name: root level: 1 coefficient: 1.0 product_of_coefficients: 1.0 ancestors: ['root'] alternatives: ['AUTO', 'NONMOTORIZED', 'TRANSIT', 'RIDEHAIL']\n", - "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] alternatives: ['DRIVEALONE', 'SHAREDRIDE2', 'SHAREDRIDE3']\n", - "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] alternatives: ['DRIVEALONEFREE', 'DRIVEALONEPAY']\n", - "Nest name: DRIVEALONEFREE level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE'] alternatives: None\n", - "Nest name: DRIVEALONEPAY level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY'] alternatives: None\n", - "Nest name: SHAREDRIDE2 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2'] alternatives: ['SHARED2FREE', 'SHARED2PAY']\n", - "Nest name: SHARED2FREE level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2FREE'] alternatives: None\n", - "Nest name: SHARED2PAY level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE2', 'SHARED2PAY'] alternatives: None\n", - "Nest name: SHAREDRIDE3 level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3'] alternatives: ['SHARED3FREE', 'SHARED3PAY']\n", - "Nest name: SHARED3FREE level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3FREE'] alternatives: None\n", - "Nest name: SHARED3PAY level: 4 coefficient: 0 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'SHAREDRIDE3', 'SHARED3PAY'] alternatives: None\n", - "Nest name: NONMOTORIZED level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED'] alternatives: ['WALK', 'BIKE']\n", - "Nest name: WALK level: 3 coefficient: 0 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'WALK'] alternatives: None\n", - "Nest name: BIKE level: 3 coefficient: 0 product_of_coefficients: 0.72 ancestors: ['root', 'NONMOTORIZED', 'BIKE'] alternatives: None\n", - "Nest name: TRANSIT level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'TRANSIT'] alternatives: ['WALKACCESS', 'DRIVEACCESS']\n", - "Nest name: WALKACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS'] alternatives: ['WALK_LOC', 'WALK_LRF', 'WALK_EXP', 'WALK_HVY', 'WALK_COM']\n", - "Nest name: WALK_LOC level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LOC'] alternatives: None\n", - "Nest name: WALK_LRF level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_LRF'] alternatives: None\n", - "Nest name: WALK_EXP level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_EXP'] alternatives: None\n", - "Nest name: WALK_HVY level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_HVY'] alternatives: None\n", - "Nest name: WALK_COM level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'WALKACCESS', 'WALK_COM'] alternatives: None\n", - "Nest name: DRIVEACCESS level: 3 coefficient: 0.5 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS'] alternatives: ['DRIVE_LOC', 'DRIVE_LRF', 'DRIVE_EXP', 'DRIVE_HVY', 'DRIVE_COM']\n", - "Nest name: DRIVE_LOC level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LOC'] alternatives: None\n", - "Nest name: DRIVE_LRF level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_LRF'] alternatives: None\n", - "Nest name: DRIVE_EXP level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_EXP'] alternatives: None\n", - "Nest name: DRIVE_HVY level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_HVY'] alternatives: None\n", - "Nest name: DRIVE_COM level: 4 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'TRANSIT', 'DRIVEACCESS', 'DRIVE_COM'] alternatives: None\n", - "Nest name: RIDEHAIL level: 2 coefficient: 0.36 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL'] alternatives: ['TAXI', 'TNC_SINGLE', 'TNC_SHARED']\n", - "Nest name: TAXI level: 3 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TAXI'] alternatives: None\n", - "Nest name: TNC_SINGLE level: 3 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SINGLE'] alternatives: None\n", - "Nest name: TNC_SHARED level: 3 coefficient: 0 product_of_coefficients: 0.36 ancestors: ['root', 'RIDEHAIL', 'TNC_SHARED'] alternatives: None\n" - ] - } - ], - "source": [ - "for nest in logit.each_nest(nest_spec):\n", - " nest.print()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d509b34d-a755-48ca-8079-386c14023d69", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 349, - "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:59:35.109079Z", - "iopub.status.busy": "2021-09-01T01:59:35.108781Z", - "iopub.status.idle": "2021-09-01T01:59:35.319301Z", - "shell.execute_reply": "2021-09-01T01:59:35.318402Z", - "shell.execute_reply.started": "2021-09-01T01:59:35.109050Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_dev(choosers, spec, nest_spec,\n", - " skims=None, locals_d=None,\n", - " chunk_size=0, custom_chooser=None,\n", - " log_alt_losers=False,\n", - " want_logsums=False,\n", - " estimator=None,\n", - " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", - " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", - " assert len(choosers) > 0\n", - " result_list = []\n", - " # segment by person type and pick the right spec for each person type\n", - " for i, chooser_chunk, chunk_trace_label \\\n", - " in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", - " # the following replaces choices = _simple_simulate(...)\n", - " if skims is not None:\n", - " simulate.set_skim_wrapper_targets(choosers, skims)\n", - "\n", - " # only do this for nested, logit is straight forward\n", - " assert nest_spec is not None\n", - " choices = simulate.eval_nl(choosers, spec, nest_spec, locals_d, custom_chooser,\n", - " log_alt_losers=log_alt_losers,\n", - " want_logsums=want_logsums,\n", - " estimator=estimator,\n", - " trace_label=trace_label,\n", - " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", - "\n", - "\n", - " result_list.append(choices)\n", - " chunk.log_df(trace_label, f'result_list', result_list)\n", - "\n", - " if len(result_list) > 1:\n", - " choices = pd.concat(result_list)\n", - " assert len(choices.index == len(choosers.index))\n", - " return choices\n" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "05e06aa4-e943-4c34-ad1b-09c19c800bdc", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:35:06.100477Z", - "iopub.status.busy": "2021-08-31T10:35:06.100054Z", - "iopub.status.idle": "2021-08-31T10:35:06.974578Z", - "shell.execute_reply": "2021-08-31T10:35:06.973620Z", - "shell.execute_reply.started": "2021-08-31T10:35:06.100438Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "primary_purpose = 'escort'\n", - "trips_segment = trips_merged.loc[trips_merged.primary_purpose == primary_purpose].copy()\n", - "assert trips_segment.index.name == 'trip_id' # name index so tracing knows how to slice\n", - "coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "locals_dict = {}\n", - "locals_dict.update(constants)\n", - "locals_dict.update(coefficients)\n", - "segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - "locals_dict.update(skims)\n", - "\n", - "spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - "nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - "choices = simple_simulate_dev(\n", - " choosers=trips_segment,\n", - " spec=spec,\n", - " nest_spec=nest_spec,\n", - " skims=skims,\n", - " locals_d=locals_dict,\n", - " chunk_size=chunk_size,\n", - " want_logsums=logsum_column_name is not None,\n", - " trace_label=segment_trace_label,\n", - " trace_choice_name='trip_mode_choice',\n", - " estimator=estimator,\n", - " trace_column_names=None)\n", - "# for consistency, always return dataframe, whether or not logsums were requested\n", - "if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - "\n", - "choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - "\n", - "alts = spec.columns\n", - "choices[mode_column_name] = \\\n", - " choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "07ef8fbc-beb2-4e57-9d56-939719026d58", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "c60656db-075a-4eab-96b3-fc54eb785075", - "metadata": {}, - "source": [ - "## eval_nl dev" - ] - }, - { - "cell_type": "code", - "execution_count": 372, - "id": "56c8923c-e946-48d8-b964-87695c30163d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:27:40.290389Z", - "iopub.status.busy": "2021-09-01T02:27:40.290139Z", - "iopub.status.idle": "2021-09-01T02:27:40.977526Z", - "shell.execute_reply": "2021-09-01T02:27:40.976594Z", - "shell.execute_reply.started": "2021-09-01T02:27:40.290363Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# replaces simple_simulate_dev:\n", - "\n", - "want_logsums = True\n", - "trace_choice_name='trip_mode_choice'\n", - "trace_column_names=None\n", - "log_alt_losers=False\n", - "\n", - "tr_label = tracing.extend_trace_label(segment_trace_label, 'simple_simulate')\n", - "assert len(trips_segment) > 0\n", - "result_list = []\n", - "# segment by person type and pick the right spec for each person type\n", - "for i, chooser_chunk, chunk_trace_label \\\n", - " in chunk.adaptive_chunked_choosers(trips_segment, chunk_size, tr_label):\n", - " # the following replaces choices = _simple_simulate(...)\n", - " if skims is not None:\n", - " simulate.set_skim_wrapper_targets(trips_segment, skims)\n", - "\n", - " # only do this for nested, logit is straight forward\n", - " assert nest_spec is not None\n", - " choices = simulate.eval_nl(trips_segment, spec, nest_spec, locals_dict, custom_chooser=None,\n", - " log_alt_losers=log_alt_losers,\n", - " want_logsums=want_logsums,\n", - " estimator=estimator,\n", - " trace_label=tr_label,\n", - " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", - "\n", - "\n", - " result_list.append(choices)\n", - "\n", - "if len(result_list) > 1:\n", - " choices = pd.concat(result_list)\n", - "assert len(choices.index == len(trips_segment.index))" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "id": "cbc2dfde-1bd9-4b03-96d9-ca9b7222d36c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T12:52:34.544865Z", - "iopub.status.busy": "2021-08-31T12:52:34.544586Z", - "iopub.status.idle": "2021-08-31T12:52:35.159615Z", - "shell.execute_reply": "2021-08-31T12:52:35.158682Z", - "shell.execute_reply.started": "2021-08-31T12:52:34.544840Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "locals_d = locals_dict\n", - "custom_chooser=None\n", - "\n", - "trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - "logit.validate_nest_spec(nest_spec, trace_label)\n", - "\n", - "raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - "nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - "\n", - "nested_probabilities = \\\n", - " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", - "\n", - "logsums = pd.Series(np.log(nested_exp_utilities.root), index=trips_segment.index)\n", - "\n", - "# global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - "base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", - "\n", - "choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "id": "0cd9593b-b3fd-41f6-a7ca-4c5b73b63571", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T12:52:45.509057Z", - "iopub.status.busy": "2021-08-31T12:52:45.508819Z", - "iopub.status.idle": "2021-08-31T12:52:45.721956Z", - "shell.execute_reply": "2021-08-31T12:52:45.721064Z", - "shell.execute_reply.started": "2021-08-31T12:52:45.509021Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 0.990308\n", - "137248725 0.108433\n", - "211388201 0.806612\n", - "211388205 0.195994\n", - "806388401 0.854020\n", - "806388405 0.660550\n", - "dtype: float64" - ] - }, - "execution_count": 95, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rands" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "id": "b467330b-f1cc-43bc-9325-c6dfeaa9b364", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T12:50:51.735273Z", - "iopub.status.busy": "2021-08-31T12:50:51.734962Z", - "iopub.status.idle": "2021-08-31T12:50:52.023440Z", - "shell.execute_reply": "2021-08-31T12:50:52.022582Z", - "shell.execute_reply.started": "2021-08-31T12:50:51.735231Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.798480-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.360270-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.435800
137248725-1.732971-3966.018685-0.606540-0.324578-3964.610293-0.113602-3966.009714-7930.295428-inf0.26146615.945036-1368.16910311.480426-2737.292402-5512.292402-5512.292402-5512.292402-5512.292402-inf-2737.782457-5522.136798-5522.136798-5516.076156-5525.080218-inf-inf-24.341318-25.104291-24.140831-8.40687011.480440
211388201-3966.000225-7930.285939-inf-24.120135-3988.405850-8.442047-29.480718-3993.766432-10.318251-5.975663-4.216264-1381.397292-3.03571013.295720-2761.150907-2761.150907-2761.150907-2761.1509076.647860-2761.648763-5543.048131-5543.048131-2769.692185-2773.867648-inf4.786459-15.861515-3.864497-5.068360-1.2967574.789158
211388205-3967.791945-7932.077659-inf-25.129596-3989.415310-8.795359-30.177275-3994.462989-10.562046-6.219063-4.216264-1381.397292-3.03571014.022438-2760.977562-2760.977562-2760.977562-2760.9775627.011219-2761.294814-5546.132585-5546.132585-2764.829835-2774.132982-inf5.048078-15.776424-3.845077-5.005482-1.2860945.050171
806388401-3966.492870-7930.778584-inf-12.618378-3976.904092-4.416432-14.908422-3979.194136-5.217948-2.9129776.192499-1378.4807234.45859917.513379-2755.845870-2755.208370-2755.728926-2755.7930938.756690-2757.559519-5540.685990-5540.685990-5542.838278-5546.780761-inf6.304817-19.143696-19.495908-16.361074-5.8539176.451457
806388405-3969.835821-7934.121535-inf-14.515916-3978.801631-5.080571-16.227796-3980.513510-5.679728-3.3428055.998749-1378.5194734.31909917.553369-2755.987742-2755.350242-2755.870797-2755.9349648.776685-2757.571454-5542.673358-5542.673358-5537.799228-5547.515590-inf6.319213-18.886576-19.495133-16.300395-5.8285406.446188
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", - "trip_id \n", - "137248721 -1.798480 -3966.084195 -0.629468 -0.389611 \n", - "137248725 -1.732971 -3966.018685 -0.606540 -0.324578 \n", - "211388201 -3966.000225 -7930.285939 -inf -24.120135 \n", - "211388205 -3967.791945 -7932.077659 -inf -25.129596 \n", - "806388401 -3966.492870 -7930.778584 -inf -12.618378 \n", - "806388405 -3969.835821 -7934.121535 -inf -14.515916 \n", - "\n", - " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", - "trip_id \n", - "137248721 -3964.675326 -0.136364 -3966.074556 -7930.360270 -inf \n", - "137248725 -3964.610293 -0.113602 -3966.009714 -7930.295428 -inf \n", - "211388201 -3988.405850 -8.442047 -29.480718 -3993.766432 -10.318251 \n", - "211388205 -3989.415310 -8.795359 -30.177275 -3994.462989 -10.562046 \n", - "806388401 -3976.904092 -4.416432 -14.908422 -3979.194136 -5.217948 \n", - "806388405 -3978.801631 -5.080571 -16.227796 -3980.513510 -5.679728 \n", - "\n", - " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", - "trip_id \n", - "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", - "137248725 0.261466 15.945036 -1368.169103 11.480426 -2737.292402 \n", - "211388201 -5.975663 -4.216264 -1381.397292 -3.035710 13.295720 \n", - "211388205 -6.219063 -4.216264 -1381.397292 -3.035710 14.022438 \n", - "806388401 -2.912977 6.192499 -1378.480723 4.458599 17.513379 \n", - "806388405 -3.342805 5.998749 -1378.519473 4.319099 17.553369 \n", - "\n", - " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", - "trip_id \n", - "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", - "137248725 -5512.292402 -5512.292402 -5512.292402 -5512.292402 -inf \n", - "211388201 -2761.150907 -2761.150907 -2761.150907 -2761.150907 6.647860 \n", - "211388205 -2760.977562 -2760.977562 -2760.977562 -2760.977562 7.011219 \n", - "806388401 -2755.845870 -2755.208370 -2755.728926 -2755.793093 8.756690 \n", - "806388405 -2755.987742 -2755.350242 -2755.870797 -2755.934964 8.776685 \n", - "\n", - " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", - "trip_id \n", - "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", - "137248725 -2737.782457 -5522.136798 -5522.136798 -5516.076156 -5525.080218 \n", - "211388201 -2761.648763 -5543.048131 -5543.048131 -2769.692185 -2773.867648 \n", - "211388205 -2761.294814 -5546.132585 -5546.132585 -2764.829835 -2774.132982 \n", - "806388401 -2757.559519 -5540.685990 -5540.685990 -5542.838278 -5546.780761 \n", - "806388405 -2757.571454 -5542.673358 -5542.673358 -5537.799228 -5547.515590 \n", - "\n", - " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", - "trip_id \n", - "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", - "137248725 -inf -inf -24.341318 -25.104291 -24.140831 -8.406870 \n", - "211388201 -inf 4.786459 -15.861515 -3.864497 -5.068360 -1.296757 \n", - "211388205 -inf 5.048078 -15.776424 -3.845077 -5.005482 -1.286094 \n", - "806388401 -inf 6.304817 -19.143696 -19.495908 -16.361074 -5.853917 \n", - "806388405 -inf 6.319213 -18.886576 -19.495133 -16.300395 -5.828540 \n", - "\n", - " root \n", - "trip_id \n", - "137248721 11.435800 \n", - "137248725 11.480440 \n", - "211388201 4.789158 \n", - "211388205 5.050171 \n", - "806388401 6.451457 \n", - "806388405 6.446188 " - ] - }, - "execution_count": 92, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# root is logsum here, no need to take log again as for exponentiated utils\n", - "simulate.compute_nested_exp_utilities(raw_utilities, nest_spec, exponentiate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "5c9ba018-167d-497f-ac65-f7c49036ee58", - "metadata": {}, - "source": [ - "#### scale of utils, including emus" - ] - }, - { - "cell_type": "code", - "execution_count": 283, - "id": "9febf6ef-7c6b-4f3c-bdd9-aa3326a82e95", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:23:00.020038Z", - "iopub.status.busy": "2021-08-31T23:23:00.019377Z", - "iopub.status.idle": "2021-08-31T23:23:00.220889Z", - "shell.execute_reply": "2021-08-31T23:23:00.220154Z", - "shell.execute_reply.started": "2021-08-31T23:23:00.019997Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "nests_ = list(logit.each_nest(nest_spec))" - ] - }, - { - "cell_type": "code", - "execution_count": 284, - "id": "48ff920e-0c9b-4b69-9cee-eb3c3427bd4f", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:23:01.085713Z", - "iopub.status.busy": "2021-08-31T23:23:01.085486Z", - "iopub.status.idle": "2021-08-31T23:23:01.283509Z", - "shell.execute_reply": "2021-08-31T23:23:01.282444Z", - "shell.execute_reply.started": "2021-08-31T23:23:01.085685Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Nest name: AUTO level: 2 coefficient: 0.72 product_of_coefficients: 0.72 ancestors: ['root', 'AUTO'] alternatives: ['DRIVEALONE', 'SHAREDRIDE2', 'SHAREDRIDE3']\n", - "Nest name: DRIVEALONE level: 3 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE'] alternatives: ['DRIVEALONEFREE', 'DRIVEALONEPAY']\n", - "Nest name: DRIVEALONEFREE level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEFREE'] alternatives: None\n", - "Nest name: DRIVEALONEPAY level: 4 coefficient: 0.35 product_of_coefficients: 0.252 ancestors: ['root', 'AUTO', 'DRIVEALONE', 'DRIVEALONEPAY'] alternatives: None\n" - ] - } - ], - "source": [ - "#list(map(lambda x: x.print(), nests_));\n", - "# have a look at \n", - "nests_[1].print()\n", - "nests_[2].print()\n", - "nests_[3].print()\n", - "nests_[4].print()" - ] - }, - { - "cell_type": "code", - "execution_count": 312, - "id": "7b7e453d-da45-4751-8889-561a349bd373", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:22.371464Z", - "iopub.status.busy": "2021-09-01T01:11:22.371192Z", - "iopub.status.idle": "2021-09-01T01:11:22.593424Z", - "shell.execute_reply": "2021-09-01T01:11:22.592472Z", - "shell.execute_reply.started": "2021-09-01T01:11:22.371415Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation belowi s equivalent.\n", - "def compute_nested_utilities(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) / nest.product_of_coefficients #coefficient\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", - " return nested_utilities\n", - "\n", - "\n", - "utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", - "\n", - "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", - "seed = 9326543345\n", - "rng = default_rng(seed=seed)\n", - "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", - "rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", - "\n", - "probs_arr = utils_df - np.log(-np.log(rands))\n", - "\n", - "# Now replace the following with recursive nested choice\n", - "# choices = np.argmax(probs_arr, axis=1)\n", - "# choices = pd.Series(choices, index=utils_df.index)\n", - "# rands = pd.Series(np.asanyarray(rands).flatten(), index=utils_df.index)" - ] - }, - { - "cell_type": "code", - "execution_count": 313, - "id": "39b71df9-12ac-4d9c-8134-43d22f7e6a6f", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:23.703299Z", - "iopub.status.busy": "2021-09-01T01:11:23.703088Z", - "iopub.status.idle": "2021-09-01T01:11:23.937077Z", - "shell.execute_reply": "2021-09-01T01:11:23.935627Z", - "shell.execute_reply.started": "2021-09-01T01:11:23.703274Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.79848-3966.084195-0.629468-0.389611-3964.675326-0.136364-3966.074556-7930.36027-inf0.24503215.883036-1368.20010211.435786-2737.198627-5512.198627-5512.198627-5512.198627-5512.198627-inf-2737.365441-5518.612509-5518.612509-5520.374306-5521.789773-inf-inf-24.503675-25.143041-24.249471-8.44847311.4358
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", - "trip_id \n", - "137248721 -1.79848 -3966.084195 -0.629468 -0.389611 \n", - "\n", - " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", - "trip_id \n", - "137248721 -3964.675326 -0.136364 -3966.074556 -7930.36027 -inf \n", - "\n", - " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", - "trip_id \n", - "137248721 0.245032 15.883036 -1368.200102 11.435786 -2737.198627 \n", - "\n", - " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", - "trip_id \n", - "137248721 -5512.198627 -5512.198627 -5512.198627 -5512.198627 -inf \n", - "\n", - " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", - "trip_id \n", - "137248721 -2737.365441 -5518.612509 -5518.612509 -5520.374306 -5521.789773 \n", - "\n", - " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", - "trip_id \n", - "137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n", - "\n", - " root \n", - "trip_id \n", - "137248721 11.4358 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-0.717963-3964.2474012.7217010.230429-3964.716506-0.274584-3965.126773-7930.610173-inf1.73337615.473633-1368.52552614.717027-2736.517471-5510.738449-5512.038516-5512.559241-5511.584293-inf-2738.143234-5518.606969-5519.009587-5519.830735-5521.525966-inf-inf-25.519792-21.698963-25.267185-7.44465211.689443
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\n", - "trip_id \n", - "137248721 -0.717963 -3964.247401 2.721701 0.230429 \n", - "\n", - " SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\n", - "trip_id \n", - "137248721 -3964.716506 -0.274584 -3965.126773 -7930.610173 -inf \n", - "\n", - " AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\n", - "trip_id \n", - "137248721 1.733376 15.473633 -1368.525526 14.717027 -2736.517471 \n", - "\n", - " WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\n", - "trip_id \n", - "137248721 -5510.738449 -5512.038516 -5512.559241 -5511.584293 -inf \n", - "\n", - " DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\n", - "trip_id \n", - "137248721 -2738.143234 -5518.606969 -5519.009587 -5519.830735 -5521.525966 \n", - "\n", - " DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\n", - "trip_id \n", - "137248721 -inf -inf -25.519792 -21.698963 -25.267185 -7.444652 \n", - "\n", - " root \n", - "trip_id \n", - "137248721 11.689443 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "display(utils_df.head(1), probs_arr.head(1))" - ] - }, - { - "cell_type": "markdown", - "id": "2f080150-c15a-4059-8a26-42a8c0072606", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:16:40.014024Z", - "iopub.status.busy": "2021-08-31T23:16:40.013770Z", - "iopub.status.idle": "2021-08-31T23:16:40.236326Z", - "shell.execute_reply": "2021-08-31T23:16:40.235553Z", - "shell.execute_reply.started": "2021-08-31T23:16:40.013973Z" - }, - "tags": [] - }, - "source": [ - "### make choice at each level" - ] - }, - { - "cell_type": "code", - "execution_count": 315, - "id": "2ed71b54-67e4-4087-957a-75f4fa184144", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:43.778081Z", - "iopub.status.busy": "2021-09-01T01:11:43.777757Z", - "iopub.status.idle": "2021-09-01T01:11:43.975332Z", - "shell.execute_reply": "2021-09-01T01:11:43.974588Z", - "shell.execute_reply.started": "2021-09-01T01:11:43.778055Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def get_alternatives(nests, name):\n", - " alts = list(filter(lambda x: x.name == name, nests))\n", - " assert len(alts) == 1, f\"{len(alts)} not one\"\n", - " alts = alts[0].alternatives\n", - " return alts\n", - "\n", - "def recursive_choice(row, columns, nest_levels, nests):\n", - " choices = row[columns].idxmax() #axis=1).values[0]\n", - " next_level_columns = get_alternatives(nests, choices)\n", - " #print(f\"{choices} leads to columns {next_level_columns}\")\n", - " if next_level_columns is None:\n", - " return choices \n", - " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", - " return new_choice" - ] - }, - { - "cell_type": "code", - "execution_count": 316, - "id": "30424562-2627-446b-abd5-7c763c52060e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:49.444289Z", - "iopub.status.busy": "2021-09-01T01:11:49.444002Z", - "iopub.status.idle": "2021-09-01T01:11:49.664214Z", - "shell.execute_reply": "2021-09-01T01:11:49.663088Z", - "shell.execute_reply.started": "2021-09-01T01:11:49.444266Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4\n" - ] - } - ], - "source": [ - "# group nests by level:\n", - "depth = np.max([x.level for x in nests_])\n", - "print(depth)\n", - "nest_levels = {x: [] for x in range(1, depth+1)}\n", - "for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - "assert len(nest_levels[1]) == 1\n", - "assert nest_levels[1][0] == 'root'" - ] - }, - { - "cell_type": "code", - "execution_count": 326, - "id": "bae6e7a8-d7c3-42a2-a82f-874b1e59a0b6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:56:38.377472Z", - "iopub.status.busy": "2021-09-01T01:56:38.377253Z", - "iopub.status.idle": "2021-09-01T01:56:38.607858Z", - "shell.execute_reply": "2021-09-01T01:56:38.606913Z", - "shell.execute_reply.started": "2021-09-01T01:56:38.377447Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 WALK\n", - "137248725 WALK\n", - "211388201 WALK_LOC\n", - "211388205 WALK_LOC\n", - "806388401 WALK\n", - "806388405 WALK\n", - "dtype: object" - ] - }, - "execution_count": 326, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests_), axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 321, - "id": "fc315f63-542f-46fd-8d70-b7a1fcd492c6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:53:15.500933Z", - "iopub.status.busy": "2021-09-01T01:53:15.500709Z", - "iopub.status.idle": "2021-09-01T01:53:15.703201Z", - "shell.execute_reply": "2021-09-01T01:53:15.702357Z", - "shell.execute_reply.started": "2021-09-01T01:53:15.500909Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#display(utils_df, probs_arr)" - ] - }, - { - "cell_type": "markdown", - "id": "8a45e1c0-0a32-4f7a-8803-7559d71abefd", - "metadata": {}, - "source": [ - "### some tests\n", - "seeds, average?" - ] - }, - { - "cell_type": "code", - "execution_count": 333, - "id": "f0e00814-2a14-4da1-bc6c-89b52b7fb286", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:58:26.002425Z", - "iopub.status.busy": "2021-09-01T01:58:26.002200Z", - "iopub.status.idle": "2021-09-01T01:58:26.247954Z", - "shell.execute_reply": "2021-09-01T01:58:26.247054Z", - "shell.execute_reply.started": "2021-09-01T01:58:26.002401Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 WALK\n", - "137248725 WALK\n", - "211388201 WALK_LOC\n", - "211388205 WALK_LOC\n", - "806388401 WALK\n", - "806388405 WALK\n", - "dtype: object" - ] - }, - "execution_count": 333, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", - "seed = 9326543345\n", - "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", - "\n", - "def make_choice(utils_df, nests, nest_levels, seed=None):\n", - " rng = default_rng(seed=seed)\n", - " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", - " probs_arr = utils_df - np.log(-np.log(rands))\n", - " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", - " return choices\n", - " \n", - "make_choice(utils_df, nests_, nest_levels, seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 348, - "id": "bca659db-4c88-4287-8f5e-9c9516e3ef41", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:59:00.410062Z", - "iopub.status.busy": "2021-09-01T01:59:00.409811Z", - "iopub.status.idle": "2021-09-01T01:59:00.621940Z", - "shell.execute_reply": "2021-09-01T01:59:00.621156Z", - "shell.execute_reply.started": "2021-09-01T01:59:00.410037Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "137248721 WALK\n", - "137248725 WALK\n", - "211388201 WALK_LOC\n", - "211388205 WALK_LOC\n", - "806388401 WALK_LOC\n", - "806388405 WALK_LOC\n", - "dtype: object" - ] - }, - "execution_count": 348, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "make_choice(utils_df, nests_, nest_levels)" - ] - }, - { - "cell_type": "code", - "execution_count": 453, - "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:55:59.542669Z", - "iopub.status.busy": "2021-09-01T04:55:59.542360Z", - "iopub.status.idle": "2021-09-01T04:55:59.771722Z", - "shell.execute_reply": "2021-09-01T04:55:59.770801Z", - "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - " nested_probabilities = \\\n", - " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", - " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) \n", - " return base_probabilities\n", - "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" - ] - }, - { - "cell_type": "code", - "execution_count": 451, - "id": "054fd9b1-72fc-49cc-a790-48ef75bcbaed", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:55:08.260140Z", - "iopub.status.busy": "2021-09-01T04:55:08.259914Z", - "iopub.status.idle": "2021-09-01T04:55:09.140095Z", - "shell.execute_reply": "2021-09-01T04:55:09.139119Z", - "shell.execute_reply.started": "2021-09-01T04:55:08.260116Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "86627409 WALK_LRF\n", - "86627413 WALK_LRF\n", - "86673657 WALK_LOC\n", - "86673658 WALK\n", - "86673659 WALK\n", - " ... \n", - "2464446025 WALK\n", - "2464446029 WALK\n", - "2464449633 WALK\n", - "2464449634 WALK\n", - "2464449637 WALK\n", - "Length: 168, dtype: object" - ] - }, - "execution_count": 451, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def simple_simulate_rum(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", - "\n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices\n", - "\n", - "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" - ] - }, - { - "cell_type": "code", - "execution_count": 380, - "id": "2da81b5e-7992-4437-8e0f-905caf026804", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:33:10.886502Z", - "iopub.status.busy": "2021-09-01T02:33:10.886252Z", - "iopub.status.idle": "2021-09-01T02:33:11.092145Z", - "shell.execute_reply": "2021-09-01T02:33:11.090725Z", - "shell.execute_reply.started": "2021-09-01T02:33:10.886465Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'trip_mode_choice'" - ] - }, - "execution_count": 380, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trace_label" - ] - }, - { - "cell_type": "code", - "execution_count": 393, - "id": "f3c5c1ba-e9c3-4511-98bf-9ac98c987da7", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:39:52.539983Z", - "iopub.status.busy": "2021-09-01T02:39:52.539668Z", - "iopub.status.idle": "2021-09-01T02:39:52.796170Z", - "shell.execute_reply": "2021-09-01T02:39:52.795498Z", - "shell.execute_reply.started": "2021-09-01T02:39:52.539944Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " segment_trace_label, log_alt_losers=log_alt_losers, \n", - " trace_column_names=trace_column_names, seed=seed)\n", - "\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " choices_list.append(choices)\n", - "\n", - " choices_df = pd.concat(choices_list)\n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 454, - "id": "74eeedd3-4ade-4729-8170-79fa9dcf11f8", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:58:52.825042Z", - "iopub.status.busy": "2021-09-01T04:58:52.824768Z", - "iopub.status.idle": "2021-09-01T04:58:53.042884Z", - "shell.execute_reply": "2021-09-01T04:58:53.042061Z", - "shell.execute_reply.started": "2021-09-01T04:58:52.825004Z" - } - }, - "outputs": [], - "source": [ - "## caculate probabilities with Asim methodology, should be correct\n", - "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", - " full_probs = [] # analytical probs\n", - "\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " #choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " # segment_trace_label, log_alt_losers=log_alt_losers, \n", - " # trace_column_names=trace_column_names, seed=seed)\n", - " probs = simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, \n", - " estimator, segment_trace_label, log_alt_losers, \n", - " trace_column_names)\n", - " full_probs.append(probs)\n", - " probs_df = pd.concat(full_probs)\n", - " return probs_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": 455, - "id": "295a4fd1-8e85-47e2-af82-9f1beac31d63", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:59:46.126555Z", - "iopub.status.busy": "2021-09-01T04:59:46.126262Z", - "iopub.status.idle": "2021-09-01T04:59:52.979908Z", - "shell.execute_reply": "2021-09-01T04:59:52.979043Z", - "shell.execute_reply.started": "2021-09-01T04:59:46.126522Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2021-09-01 14:59:46.353007 Start\n", - "2021-09-01 14:59:52.976736 End\n" - ] - } - ], - "source": [ - "print(f\"{datetime.now()} Start\")\n", - "pipeline.open_pipeline(resume_after)\n", - "inject.get_injectable('preload_injectables', None)\n", - "model_name = \"trip_mode_choice\"\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "\n", - "step_name = model_name\n", - "args = {}\n", - "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "chunk_size = 0\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "trips_df = trips.to_frame()\n", - "#print(\"Running with %d trips\", trips_df.shape[0])\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - "orig_col = 'origin'\n", - "dest_col = 'destination'\n", - "\n", - "constants = {}\n", - "constants.update(config.get_model_constants(model_settings))\n", - "constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - "})\n", - "\n", - "skim_dict = network_los.get_default_skim_dict()\n", - "\n", - "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - "skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - "}\n", - "\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "\n", - "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - "\n", - "all_choices = []\n", - "for i in range(100):\n", - " if i % 10 == 0:\n", - " print(f\"{datetime.now()} iteration {i}\")\n", - " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=None)\n", - " all_choices.append(choices_df)\n", - "all_choices = pd.concat(all_choices, axis=1)\n", - "\n", - "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", - " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", - " log_alt_losers=None, trace_column_names=None)\n", - "\n", - "# update trips table with choices (and potionally logssums)\n", - "#trips_df = trips.to_frame()\n", - "#\n", - "#assign_in_place(trips_df, choices_df)\n", - "#assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - "print(f\"{datetime.now()} End\")" - ] - }, - { - "cell_type": "code", - "execution_count": 463, - "id": "dc87e904-16d1-44d4-81b7-cafbe4e223c3", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:01:40.901982Z", - "iopub.status.busy": "2021-09-01T05:01:40.901759Z", - "iopub.status.idle": "2021-09-01T05:01:41.337758Z", - "shell.execute_reply": "2021-09-01T05:01:41.336757Z", - "shell.execute_reply.started": "2021-09-01T05:01:40.901946Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#all_choices.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", - "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts = val_counts / all_choices.shape[1]\n", - "#val_counts = val_counts.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", - "#val_counts['prob_of_asim_choice'] = val_counts.apply(lambda x: x[x.asim], axis=1) # this is what our simulation says w.r.t. to asim choice\n", - "# for 100% and many samples should mostly agree" - ] - }, - { - "cell_type": "code", - "execution_count": 464, - "id": "09bf151c-a39a-4823-acfd-9c8ae747c338", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:01:46.119117Z", - "iopub.status.busy": "2021-09-01T05:01:46.118849Z", - "iopub.status.idle": "2021-09-01T05:01:46.354114Z", - "shell.execute_reply": "2021-09-01T05:01:46.353248Z", - "shell.execute_reply.started": "2021-09-01T05:01:46.119092Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#val_counts['prob_of_asim_choice'].hist(bins=100);" - ] - }, - { - "cell_type": "code", - "execution_count": 490, - "id": "f3b6ade9-d875-4104-b438-b53ef42e342c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:10:08.653227Z", - "iopub.status.busy": "2021-09-01T05:10:08.652978Z", - "iopub.status.idle": "2021-09-01T05:10:08.915620Z", - "shell.execute_reply": "2021-09-01T05:10:08.914513Z", - "shell.execute_reply.started": "2021-09-01T05:10:08.653189Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", - "analytical 3.152 0.852 0.0 0.0 0.0 \n", - "experiment 3.243 0.880 0.0 0.0 0.0 \n", - "\n", - " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", - "analytical 0.0 0.0 0.0 0.685 0.0 \n", - "experiment 0.0 0.0 0.0 0.772 0.0 \n", - "\n", - " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK \\\n", - "analytical 0.129 0.0 0.182 0.260 1.334 63.708 \n", - "experiment 0.156 0.0 0.174 0.259 1.434 63.243 \n", - "\n", - " WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", - "analytical 0.0 0.0 0.455 18.355 10.887 \n", - "experiment 0.0 0.0 0.394 18.639 10.807 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "mode_share_analytical = (probs_nl.sum(axis=0) / probs_nl.shape[0]).to_frame('analytical')\n", - "assert np.allclose(mode_share_analytical.sum(), 1)\n", - "mode_share_rum = (val_counts.sum(axis=0) / val_counts.shape[0]).to_frame('experiment')\n", - "assert np.allclose(mode_share_rum.sum(), 1)\n", - "full_share = mode_share_analytical.join(mode_share_rum, how='outer').fillna(0)\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * full_share).T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e48a56e-d613-4dfd-bef6-62bee524f12f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50914e1d-750d-440c-9617-f96bd2a46c56", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2d70e0e-520e-49a6-8024-ae31345f6ead", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "f9e8ef02-1d57-4b21-8aef-25a6e1095c02", - "metadata": {}, - "source": [ - "### try zenith normalisation of simple_simulate_rum" - ] - }, - { - "cell_type": "code", - "execution_count": 629, - "id": "db235e90-d2e5-4e90-b41c-6d90a670e41d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:24.045881Z", - "iopub.status.busy": "2021-09-01T11:37:24.045637Z", - "iopub.status.idle": "2021-09-01T11:37:24.276488Z", - "shell.execute_reply": "2021-09-01T11:37:24.274898Z", - "shell.execute_reply.started": "2021-09-01T11:37:24.045844Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation belowi s equivalent.\n", - "def compute_nested_utilities_zenith(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) # / nest.product_of_coefficients #coefficient\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", - " \n", - " # now go over all leaves and correct for scale\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] /= nest.coefficient\n", - " \n", - " return nested_utilities\n", - "\n", - "\n", - "def simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " utils_df = compute_nested_utilities_zenith(raw_utilities, nest_spec)\n", - "\n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices\n", - "\n", - "#simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" - ] - }, - { - "cell_type": "code", - "execution_count": 630, - "id": "f69bc3dc-84e4-4dd5-9a57-c053cadba4b5", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:24.828926Z", - "iopub.status.busy": "2021-09-01T11:37:24.828707Z", - "iopub.status.idle": "2021-09-01T11:37:25.046409Z", - "shell.execute_reply": "2021-09-01T11:37:25.045399Z", - "shell.execute_reply.started": "2021-09-01T11:37:24.828902Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " choices = simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " segment_trace_label, log_alt_losers=log_alt_losers, \n", - " trace_column_names=trace_column_names, seed=seed)\n", - "\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " choices_list.append(choices)\n", - "\n", - " choices_df = pd.concat(choices_list)\n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 631, - "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:32.250808Z", - "iopub.status.busy": "2021-09-01T11:37:32.250548Z", - "iopub.status.idle": "2021-09-01T20:20:35.257463Z", - "shell.execute_reply": "2021-09-01T20:20:35.255964Z", - "shell.execute_reply.started": "2021-09-01T11:37:32.250782Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "num_samples = 10\n", - "\n", - "rng_test = default_rng(23423)\n", - "\n", - "print(f\"{datetime.now()} Start\")\n", - "pipeline.open_pipeline(resume_after)\n", - "inject.get_injectable('preload_injectables', None)\n", - "model_name = \"trip_mode_choice\"\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "\n", - "step_name = model_name\n", - "args = {}\n", - "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "chunk_size = 0\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "trips_df = trips.to_frame()\n", - "#print(\"Running with %d trips\", trips_df.shape[0])\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - "orig_col = 'origin'\n", - "dest_col = 'destination'\n", - "\n", - "constants = {}\n", - "constants.update(config.get_model_constants(model_settings))\n", - "constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - "})\n", - "\n", - "skim_dict = network_los.get_default_skim_dict()\n", - "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - "}\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - "all_choices_zenith = []\n", - "all_choices = []\n", - "for i in range(num_samples):\n", - " \n", - " seed = rng_test.integers(0, 100000) #int(9.3 * (i+1)**3) # why not\n", - " \n", - " if i % 50 == 0:\n", - " print(f\"{datetime.now()} iteration {i}\")\n", - " choices_df_zenith = stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", - " all_choices_zenith.append(choices_df_zenith)\n", - "\n", - " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", - " all_choices.append(choices_df)\n", - " \n", - " t_ = choices_df_zenith.merge(choices_df, left_index=True, right_index=True, suffixes=['_zenith', '_asim'])\n", - " diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", - " \n", - " #print(f\"seed {seed} leads to {diffs.shape[0]} differences. tripids {diffs.index}\")\n", - "\n", - "all_choices_zenith = pd.concat(all_choices_zenith, axis=1)\n", - "all_choices = pd.concat(all_choices, axis=1)\n", - "\n", - "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", - " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", - " log_alt_losers=None, trace_column_names=None)\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - "print(f\"{datetime.now()} End\")" - ] - }, - { - "cell_type": "code", - "execution_count": 643, - "id": "3abd7949-dc9c-469b-845e-26c17741f70e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:33:21.533110Z", - "iopub.status.busy": "2021-09-01T20:33:21.531915Z", - "iopub.status.idle": "2021-09-01T20:33:21.740946Z", - "shell.execute_reply": "2021-09-01T20:33:21.739713Z", - "shell.execute_reply.started": "2021-09-01T20:33:21.533074Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Have 2038 samples\n" - ] - } - ], - "source": [ - "print(f\"Have {all_choices_zenith.shape[1]} samples\")" - ] - }, - { - "cell_type": "code", - "execution_count": 633, - "id": "8762bd6d-ac04-458c-869f-be86d8297351", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:21:27.778803Z", - "iopub.status.busy": "2021-09-01T20:21:27.778531Z", - "iopub.status.idle": "2021-09-01T20:21:28.675645Z", - "shell.execute_reply": "2021-09-01T20:21:28.674734Z", - "shell.execute_reply.started": "2021-09-01T20:21:27.778754Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "val_counts_zenith = all_choices_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts_zenith = val_counts_zenith / all_choices_zenith.shape[1]\n", - "\n", - "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts = val_counts / all_choices.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 634, - "id": "0f566105-9424-457b-86f5-d60c1d63aac5", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:21:30.954894Z", - "iopub.status.busy": "2021-09-01T20:21:30.954654Z", - "iopub.status.idle": "2021-09-01T20:21:31.179689Z", - "shell.execute_reply": "2021-09-01T20:21:31.178304Z", - "shell.execute_reply.started": "2021-09-01T20:21:30.954867Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
experiment_zenith3.1500.8740.00.00.00.00.00.00.6800.00.1360.00.1540.2361.28563.7670.00.00.51518.82410.379
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", - "analytical 3.152 0.852 0.0 0.0 0.0 \n", - "experiment 3.243 0.880 0.0 0.0 0.0 \n", - "experiment_zenith 3.150 0.874 0.0 0.0 0.0 \n", - "\n", - " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", - "analytical 0.0 0.0 0.0 0.685 0.0 \n", - "experiment 0.0 0.0 0.0 0.772 0.0 \n", - "experiment_zenith 0.0 0.0 0.0 0.680 0.0 \n", - "\n", - " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE \\\n", - "analytical 0.129 0.0 0.182 0.260 1.334 \n", - "experiment 0.156 0.0 0.174 0.259 1.434 \n", - "experiment_zenith 0.136 0.0 0.154 0.236 1.285 \n", - "\n", - " WALK WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", - "analytical 63.708 0.0 0.0 0.455 18.355 10.887 \n", - "experiment 63.243 0.0 0.0 0.394 18.639 10.807 \n", - "experiment_zenith 63.767 0.0 0.0 0.515 18.824 10.379 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "mode_share_rum_zenith = (val_counts_zenith.sum(axis=0) / val_counts_zenith.shape[0]).to_frame('experiment_zenith')\n", - "assert np.allclose(mode_share_rum_zenith.sum(), 1)\n", - "full_share_incl_zenith = full_share.merge(mode_share_rum_zenith, left_index=True, right_index=True, how='outer').fillna(0)\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * full_share_incl_zenith).T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c0d3081-b45b-4573-9246-e4a9d39591dd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "612b2f04-45e2-4eb2-b52a-9f753427877f", - "metadata": {}, - "source": [ - "## investigate diverging seed and look at diff in formulation" - ] - }, - { - "cell_type": "code", - "execution_count": 619, - "id": "ac09dc75-a92f-43fb-a0cd-fe7de1f8b66d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:12:02.988141Z", - "iopub.status.busy": "2021-09-01T11:12:02.987900Z", - "iopub.status.idle": "2021-09-01T11:12:03.212179Z", - "shell.execute_reply": "2021-09-01T11:12:03.210993Z", - "shell.execute_reply.started": "2021-09-01T11:12:02.988102Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation based on asim probabilities is equivalent\n", - "def compute_nested_utilities_asim(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) / nest.product_of_coefficients\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", - " return nested_utilities\n", - "\n", - "def compute_nested_utilities_zenith_check(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) # scale correction is below\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", - " \n", - " # now go over all leaves and correct for scale\n", - " for nest in logit.each_nest(nest_spec):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] /= nest.coefficient\n", - " \n", - " return nested_utilities" - ] - }, - { - "cell_type": "code", - "execution_count": 688, - "id": "4b7c6368-21ca-4442-969e-f2f8ff868842", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:49:56.958862Z", - "iopub.status.busy": "2021-09-01T20:49:56.958623Z", - "iopub.status.idle": "2021-09-01T20:49:57.191884Z", - "shell.execute_reply": "2021-09-01T20:49:57.190975Z", - "shell.execute_reply.started": "2021-09-01T20:49:56.958827Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None, use_zenith=True, raw_utilities=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " \n", - " if raw_utilities is None:\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " if use_zenith:\n", - " utils_df = compute_nested_utilities_zenith_check(raw_utilities, nest_spec)\n", - " else:\n", - " utils_df = compute_nested_utilities_asim(raw_utilities, nest_spec)\n", - "\n", - " # test\n", - " #return compute_nested_utilities_zenith_check(raw_utilities, nest_spec), compute_nested_utilities_asim(raw_utilities, nest_spec)\n", - " \n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices" - ] - }, - { - "cell_type": "code", - "execution_count": 626, - "id": "bd6cadc3-79f5-4824-b55e-639c39d5ab18", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:13:12.077886Z", - "iopub.status.busy": "2021-09-01T11:13:12.077652Z", - "iopub.status.idle": "2021-09-01T11:13:12.290222Z", - "shell.execute_reply": "2021-09-01T11:13:12.288861Z", - "shell.execute_reply.started": "2021-09-01T11:13:12.077863Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# t_z, t_a = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed)\n", - "# tr_id = 86673661\n", - "# display(t_z.loc[t_z.index==tr_id])\n", - "# display(t_a.loc[t_a.index==tr_id])" - ] - }, - { - "cell_type": "code", - "execution_count": 625, - "id": "42e6da62-58e8-4b16-9169-9faa1fc8a162", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:13:06.847583Z", - "iopub.status.busy": "2021-09-01T11:13:06.847204Z", - "iopub.status.idle": "2021-09-01T11:13:08.401700Z", - "shell.execute_reply": "2021-09-01T11:13:08.400711Z", - "shell.execute_reply.started": "2021-09-01T11:13:06.847554Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed 51006 leads to 34 differences out of 168. tripids Int64Index([ 86673657, 86673661, 106741681, 106741682, 106741685,\n", - " 106741686, 106741687, 106741688, 211327433, 211327437,\n", - " 444793574, 484173905, 484173909, 535170694, 535620053,\n", - " 708171014, 943749470, 943749471, 1060575853, 1091770617,\n", - " 1146472489, 1146472493, 1276281769, 1276281773, 1658748793,\n", - " 1658748797, 1767013726, 1767186577, 1767186578, 1768237161,\n", - " 1768237165, 1768237166, 2463663417, 2463663421],\n", - " dtype='int64', name='trip_id')\n" - ] - } - ], - "source": [ - "seed = 51006\n", - "x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True)\n", - "y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False)\n", - "t_ = x_.to_frame('trip_mode_zenith').merge(y_.to_frame('trip_mode_asim'), left_index=True, right_index=True)\n", - "diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", - "print(f\"seed {seed} leads to {diffs.shape[0]} differences out of {t_.shape[0]}. tripids {diffs.index}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2f3c0506-a79d-4a31-9d56-71b9966d0a92", - "metadata": {}, - "outputs": [], - "source": [ - "seed 51006 leads to 3 differences. tripids Int64Index([86673661, 535170689, 1060575849], dtype='int64', name='trip_id')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1a5f332a-b1d7-4a80-bb21-d37a1611cf9f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24c5e4a1-6142-49d4-8da8-a1e266b3cb9b", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 753, - "id": "2098907f-28af-4f53-af74-6aaee8a6c53c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T23:04:23.861048Z", - "iopub.status.busy": "2021-09-01T23:04:23.860748Z", - "iopub.status.idle": "2021-09-01T23:04:24.069307Z", - "shell.execute_reply": "2021-09-01T23:04:24.068434Z", - "shell.execute_reply.started": "2021-09-01T23:04:23.861024Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "rng_ = default_rng(seed=100)\n", - "new_utils = pd.DataFrame(0.1 * -np.log(-np.log(rng_.uniform(0,1,raw_utilities.shape))), columns=raw_utilities.columns)\n", - "new_utils.index = raw_utilities.index" - ] - }, - { - "cell_type": "code", - "execution_count": 767, - "id": "da48a178-82b2-44a5-b45a-e9de3bb5e5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T23:17:20.369675Z", - "iopub.status.busy": "2021-09-01T23:17:20.369424Z", - "iopub.status.idle": "2021-09-01T23:17:20.666506Z", - "shell.execute_reply": "2021-09-01T23:17:20.665626Z", - "shell.execute_reply.started": "2021-09-01T23:17:20.369637Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
1372487210.0566420.0373050.0427800.0295760.0861540.0265860.0883130.1003310.0280210.0185160.0643910.0200090.0263990.0226220.0230070.0217540.0871260.0211710.0364750.0276500.135171
1372487250.1019020.0228760.0312270.0699900.0569130.0466850.0884110.0877850.0254140.0381620.0239880.0288000.0393180.0201600.0483640.0230030.0339590.0345970.0377080.0995230.041213
2113882010.0375620.0700330.0559510.0403890.0200410.0959740.0946780.0943680.0360140.0250510.0348600.0349530.0354990.0280400.0436500.0247130.0313880.0259310.0669990.0526780.051228
2113882050.0354800.0562560.0439440.0480850.0420780.0622210.1068190.0872650.0351170.0303990.0351540.0264690.0506460.0357270.0242840.0329050.0394270.0410920.0547140.0484590.063458
8063884010.0549180.0404880.0331490.0542940.0434770.0679820.1524650.0741520.0334040.0287350.0592690.0243390.0369680.0277600.0235220.0609320.0234450.0239310.0446430.0337730.058353
8063884050.0327640.0573010.0184370.0770990.0469630.0635210.0949510.0890720.0308260.0378960.0357520.0198780.0495360.0796100.0269220.0248330.0248400.0266950.0799360.0401460.043022
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n", - "trip_id \n", - "137248721 0.056642 0.037305 0.042780 0.029576 \n", - "137248725 0.101902 0.022876 0.031227 0.069990 \n", - "211388201 0.037562 0.070033 0.055951 0.040389 \n", - "211388205 0.035480 0.056256 0.043944 0.048085 \n", - "806388401 0.054918 0.040488 0.033149 0.054294 \n", - "806388405 0.032764 0.057301 0.018437 0.077099 \n", - "\n", - " SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\n", - "trip_id \n", - "137248721 0.086154 0.026586 0.088313 0.100331 0.028021 0.018516 \n", - "137248725 0.056913 0.046685 0.088411 0.087785 0.025414 0.038162 \n", - "211388201 0.020041 0.095974 0.094678 0.094368 0.036014 0.025051 \n", - "211388205 0.042078 0.062221 0.106819 0.087265 0.035117 0.030399 \n", - "806388401 0.043477 0.067982 0.152465 0.074152 0.033404 0.028735 \n", - "806388405 0.046963 0.063521 0.094951 0.089072 0.030826 0.037896 \n", - "\n", - " WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\n", - "trip_id \n", - "137248721 0.064391 0.020009 0.026399 0.022622 0.023007 0.021754 \n", - "137248725 0.023988 0.028800 0.039318 0.020160 0.048364 0.023003 \n", - "211388201 0.034860 0.034953 0.035499 0.028040 0.043650 0.024713 \n", - "211388205 0.035154 0.026469 0.050646 0.035727 0.024284 0.032905 \n", - "806388401 0.059269 0.024339 0.036968 0.027760 0.023522 0.060932 \n", - "806388405 0.035752 0.019878 0.049536 0.079610 0.026922 0.024833 \n", - "\n", - " DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n", - "trip_id \n", - "137248721 0.087126 0.021171 0.036475 0.027650 0.135171 \n", - "137248725 0.033959 0.034597 0.037708 0.099523 0.041213 \n", - "211388201 0.031388 0.025931 0.066999 0.052678 0.051228 \n", - "211388205 0.039427 0.041092 0.054714 0.048459 0.063458 \n", - "806388401 0.023445 0.023931 0.044643 0.033773 0.058353 \n", - "806388405 0.024840 0.026695 0.079936 0.040146 0.043022 " - ] - }, - "execution_count": 767, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nested_exp_utilities_cf = simulate.compute_nested_exp_utilities(new_utils, nest_spec)\n", - "nested_probabilities_cf = simulate.compute_nested_probabilities(nested_exp_utilities_cf, nest_spec, trace_label=None)\n", - "base_probabilities_cf = simulate.compute_base_probabilities(nested_probabilities_cf, nest_spec, spec)\n", - "base_probabilities_cf" - ] - }, - { - "cell_type": "code", - "execution_count": 799, - "id": "50dfd1ae-10c3-475c-94be-de783c2fa5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:15:29.056902Z", - "iopub.status.busy": "2021-09-02T00:15:29.056683Z", - "iopub.status.idle": "2021-09-02T00:27:21.965206Z", - "shell.execute_reply": "2021-09-02T00:27:21.964340Z", - "shell.execute_reply.started": "2021-09-02T00:15:29.056877Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 11min 49s, sys: 766 ms, total: 11min 50s\n", - "Wall time: 11min 52s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "num_samples = 10000 # 7.5s per 100\n", - "\n", - "data_zenith = []\n", - "data_asim = []\n", - "for i in range(num_samples):\n", - " seed = rng_.integers(0, 100000)\n", - " x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True, raw_utilities=new_utils)\n", - " y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False, raw_utilities=new_utils)\n", - " data_zenith.append(x_)\n", - " data_asim.append(y_)\n", - " \n", - "data_asim = pd.concat(data_asim, axis=1)\n", - "data_zenith = pd.concat(data_zenith, axis=1)\n", - "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", - "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_asim = counts_asim / data_asim.shape[1]\n", - "\n", - "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", - "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", - "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", - "# assert np.allclose(mode_share_zenith.sum(), 1)\n", - "# assert np.allclose(mode_share_asim.sum(), 1)\n", - "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", - "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 796, - "id": "bb16f9d7-44f5-4316-b981-d65216a5e217", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:14:11.772592Z", - "iopub.status.busy": "2021-09-02T00:14:11.772347Z", - "iopub.status.idle": "2021-09-02T00:14:12.001077Z", - "shell.execute_reply": "2021-09-02T00:14:12.000166Z", - "shell.execute_reply.started": "2021-09-02T00:14:11.772556Z" - } - }, - "outputs": [], - "source": [ - "#temp_z = data_zenith.copy()\n", - "#temp_a = data_asim.copy()\n", - "#data_asim = data_asim.join(temp_a, lsuffix=\"_o\", rsuffix=\"_n\")\n", - "#data_zenith = data_zenith.join(temp_z, lsuffix=\"_o\", rsuffix=\"_n\")\n", - "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", - "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_asim = counts_asim / data_asim.shape[1]\n", - "\n", - "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", - "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", - "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", - "# assert np.allclose(mode_share_zenith.sum(), 1)\n", - "# assert np.allclose(mode_share_asim.sum(), 1)\n", - "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", - "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 805, - "id": "031d8340-0af0-4608-8a83-ed2af4aaa3b2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:41:01.948669Z", - "iopub.status.busy": "2021-09-02T00:41:01.948422Z", - "iopub.status.idle": "2021-09-02T00:41:02.175972Z", - "shell.execute_reply": "2021-09-02T00:41:02.174759Z", - "shell.execute_reply.started": "2021-09-02T00:41:01.948633Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
zenith8.0555.1674.6313.5023.6064.2663.9633.4983.8955.1764.7995.8514.9996.1214.6689.4864.2284.1252.9773.6713.317
asim6.0966.6984.9773.2023.5224.3344.1293.1843.2245.1496.2838.2964.7936.5844.5327.4844.1524.2732.6293.4333.023
probs8.8835.3214.7382.8903.1364.0033.5653.1623.7585.3244.9276.0495.3416.5415.03710.4273.9734.2242.5743.1472.979
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP DRIVE_HVY \\\n", - "zenith 8.055 5.167 4.631 3.502 3.606 4.266 \n", - "asim 6.096 6.698 4.977 3.202 3.522 4.334 \n", - "probs 8.883 5.321 4.738 2.890 3.136 4.003 \n", - "\n", - " DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY SHARED3FREE \\\n", - "zenith 3.963 3.498 3.895 5.176 4.799 \n", - "asim 4.129 3.184 3.224 5.149 6.283 \n", - "probs 3.565 3.162 3.758 5.324 4.927 \n", - "\n", - " SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK WALK_COM WALK_EXP \\\n", - "zenith 5.851 4.999 6.121 4.668 9.486 4.228 4.125 \n", - "asim 8.296 4.793 6.584 4.532 7.484 4.152 4.273 \n", - "probs 6.049 5.341 6.541 5.037 10.427 3.973 4.224 \n", - "\n", - " WALK_HVY WALK_LOC WALK_LRF \n", - "zenith 2.977 3.671 3.317 \n", - "asim 2.629 3.433 3.023 \n", - "probs 2.574 3.147 2.979 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# sample of 20000\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * mode_share_comp).T)" - ] - }, - { - "cell_type": "code", - "execution_count": 806, - "id": "aece448f-4055-4d2a-b90d-8530caec9cb9", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:43:32.971972Z", - "iopub.status.busy": "2021-09-02T00:43:32.971684Z", - "iopub.status.idle": "2021-09-02T00:43:33.223345Z", - "shell.execute_reply": "2021-09-02T00:43:33.222488Z", - "shell.execute_reply.started": "2021-09-02T00:43:32.971932Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
trip_id137248721_z137248721_a137248721
BIKE0.092700.054350.100331
DRIVEALONEFREE0.053650.050050.056642
DRIVEALONEPAY0.038800.031550.037305
DRIVE_COM0.028700.024700.021171
DRIVE_EXP0.028800.026050.021754
DRIVE_HVY0.078150.099700.087126
DRIVE_LOC0.028750.027000.022622
DRIVE_LRF0.027800.024450.023007
SHARED2FREE0.043250.023000.042780
SHARED2PAY0.033600.015850.029576
SHARED3FREE0.073650.119200.086154
SHARED3PAY0.032600.039600.026586
TAXI0.035100.050300.036475
TNC_SHARED0.126200.181150.135171
TNC_SINGLE0.026250.038450.027650
WALK0.079400.048250.088313
WALK_COM0.031850.024600.026399
WALK_EXP0.057050.058950.064391
WALK_HVY0.025200.018400.020009
WALK_LOC0.034550.027400.028021
WALK_LRF0.023950.017000.018516
\n", - "
" - ], - "text/plain": [ - "trip_id 137248721_z 137248721_a 137248721\n", - "BIKE 0.09270 0.05435 0.100331\n", - "DRIVEALONEFREE 0.05365 0.05005 0.056642\n", - "DRIVEALONEPAY 0.03880 0.03155 0.037305\n", - "DRIVE_COM 0.02870 0.02470 0.021171\n", - "DRIVE_EXP 0.02880 0.02605 0.021754\n", - "DRIVE_HVY 0.07815 0.09970 0.087126\n", - "DRIVE_LOC 0.02875 0.02700 0.022622\n", - "DRIVE_LRF 0.02780 0.02445 0.023007\n", - "SHARED2FREE 0.04325 0.02300 0.042780\n", - "SHARED2PAY 0.03360 0.01585 0.029576\n", - "SHARED3FREE 0.07365 0.11920 0.086154\n", - "SHARED3PAY 0.03260 0.03960 0.026586\n", - "TAXI 0.03510 0.05030 0.036475\n", - "TNC_SHARED 0.12620 0.18115 0.135171\n", - "TNC_SINGLE 0.02625 0.03845 0.027650\n", - "WALK 0.07940 0.04825 0.088313\n", - "WALK_COM 0.03185 0.02460 0.026399\n", - "WALK_EXP 0.05705 0.05895 0.064391\n", - "WALK_HVY 0.02520 0.01840 0.020009\n", - "WALK_LOC 0.03455 0.02740 0.028021\n", - "WALK_LRF 0.02395 0.01700 0.018516" - ] - }, - "execution_count": 806, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# TODO: look at probs per trip, i.e. do not sum across trips\n", - "trip_id = 137248721\n", - "\n", - "counts_zenith.loc[counts_zenith.index == trip_id].T.merge(\n", - " counts_asim.loc[counts_asim.index == trip_id].T, suffixes=['_z', '_a'], left_index=True, right_index=True, how='outer').merge(\n", - " base_probabilities_cf.loc[base_probabilities_cf.index == trip_id].T, suffixes=['', '_probs'], left_index=True, right_index=True, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ef84f10-d616-40e4-87e5-425499027ff3", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "431cee49-40a9-4dc7-93a5-75308fb302dd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1fcdc0ff-c510-4878-8672-8365e046442c", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/frozen_impl_dev.ipynb b/notebooks/frozen_impl_dev.ipynb deleted file mode 100644 index d4838d8b09..0000000000 --- a/notebooks/frozen_impl_dev.ipynb +++ /dev/null @@ -1,661 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", - "metadata": {}, - "source": [ - "# validate results\n", - "\n", - "## TODO\n", - "what happened to tracing when I fixed probability calcs - it seems like all trip ids are attached when I add one by\n", - "hand below - why?" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.385153Z", - "iopub.status.busy": "2022-05-01T10:17:57.384881Z", - "iopub.status.idle": "2022-05-01T10:17:57.534433Z", - "shell.execute_reply": "2022-05-01T10:17:57.533096Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%load_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "c3dba451-1e10-403e-8614-35d57e6577f4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.536623Z", - "iopub.status.busy": "2022-05-01T10:17:57.536012Z", - "iopub.status.idle": "2022-05-01T10:17:57.542755Z", - "shell.execute_reply": "2022-05-01T10:17:57.541685Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:00.350944Z", - "iopub.status.busy": "2022-05-01T10:18:00.350730Z", - "iopub.status.idle": "2022-05-01T10:18:12.760977Z", - "shell.execute_reply": "2022-05-01T10:18:12.760013Z", - "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import argparse\n", - "from datetime import datetime\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from numpy.random import default_rng\n", - "from statsmodels.stats.gof import powerdiscrepancy\n", - "from statsmodels.stats.proportion import multinomial_proportions_confint\n", - "import numba\n", - "import matplotlib.pyplot as plt\n", - "\n", - "from activitysim.cli import run\n", - "from activitysim.core import inject\n", - "\n", - "from fru_utils import run_trip_mode_choice, comp_mode_shares, mode_choice_for_trip" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:16.944537Z", - "iopub.status.busy": "2022-05-01T10:18:16.944291Z", - "iopub.status.idle": "2022-05-01T10:18:17.124764Z", - "shell.execute_reply": "2022-05-01T10:18:17.123725Z", - "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "pd.set_option(\"max_columns\", 500)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "outputs": [ - { - "data": { - "text/plain": "array([2, 1, 2])" - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.argmax(np.array([[1,2,3],[4,6,5],[7,8,9]]), axis=1)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 34, - "outputs": [ - { - "data": { - "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": "0 c\n1 b\n2 c\ndtype: object" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", - "display(df_)\n", - "ch_ = df_.idxmax(1)\n", - "display(ch_)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:17.839947Z", - "iopub.status.busy": "2022-05-01T10:18:17.839070Z", - "iopub.status.idle": "2022-05-01T10:18:18.019676Z", - "shell.execute_reply": "2022-05-01T10:18:18.018689Z", - "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_example_mtc\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:18.894533Z", - "iopub.status.busy": "2022-05-01T10:18:18.894303Z", - "iopub.status.idle": "2022-05-01T10:18:19.078807Z", - "shell.execute_reply": "2022-05-01T10:18:19.077951Z", - "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "os.chdir(example_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:22.111723Z", - "iopub.status.busy": "2022-05-01T10:18:22.111490Z", - "iopub.status.idle": "2022-05-01T10:18:22.297437Z", - "shell.execute_reply": "2022-05-01T10:18:22.296501Z", - "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "parser = argparse.ArgumentParser()\n", - "run.add_run_args(parser)\n", - "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", - "#run.run(args) # 2mins full example run\n", - "if not inject.is_injectable('preload_injectables'):\n", - " from activitysim import abm # register abm steps and other abm-specific injectables\n", - "run.handle_standard_args(args) # possibly update injectables" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "outputs": [], - "source": [ - "from activitysim.core import inject\n", - "from activitysim.core import pipeline\n", - "from activitysim.core import config\n", - "from activitysim.core import expressions\n", - "from activitysim.core import simulate\n", - "from activitysim.abm.models.util import estimation\n", - "\n", - "def run_mandatory_tour_frequency(choose_individual_max_utility=False):\n", - " resume_after = \"cdap_simulate\"\n", - " model_name = \"mandatory_tour_frequency\"\n", - " chunk_size = 0 # test_mtc means no chunking\n", - "\n", - " pipeline.open_pipeline(resume_after)\n", - " # preload any bulky injectables (e.g. skims) not in pipeline\n", - " inject.get_injectable('preload_injectables', None)\n", - " pipeline._PIPELINE.rng().begin_step(model_name)\n", - " #step_name = model_name\n", - " args = {}\n", - " #checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - " inject.set_step_args(args)\n", - "\n", - " #network_los = inject.get_injectable('network_los')\n", - " persons_merged = inject.get_table('persons_merged')\n", - "\n", - " trace_label = 'mandatory_tour_frequency'\n", - " model_settings_file_name = 'mandatory_tour_frequency.yaml'\n", - " model_settings = config.read_model_settings(model_settings_file_name)\n", - " choosers = persons_merged.to_frame()\n", - " # filter based on results of CDAP\n", - " choosers = choosers[choosers.cdap_activity == 'M']\n", - " print(\"Running mandatory_tour_frequency with %d persons\", len(choosers))\n", - " preprocessor_settings = model_settings.get('preprocessor', None)\n", - " if preprocessor_settings:\n", - " locals_dict = {}\n", - " expressions.assign_columns(\n", - " df=choosers,\n", - " model_settings=preprocessor_settings,\n", - " locals_dict=locals_dict,\n", - " trace_label=trace_label)\n", - "\n", - " estimator = estimation.manager.begin_estimation('mandatory_tour_frequency')\n", - " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - " coefficients_df = simulate.read_model_coefficients(model_settings)\n", - " model_spec = simulate.eval_coefficients(model_spec, coefficients_df, estimator)\n", - " nest_spec = config.get_logit_model_settings(model_settings)\n", - " constants = config.get_model_constants(model_settings)\n", - " choices = simulate.simple_simulate(\n", - " choosers=choosers,\n", - " spec=model_spec,\n", - " nest_spec=nest_spec,\n", - " locals_d=constants,\n", - " chunk_size=chunk_size,\n", - " trace_label=trace_label,\n", - " trace_choice_name='mandatory_tour_frequency',\n", - " estimator=estimator,\n", - " choose_individual_max_utility=choose_individual_max_utility)\n", - "\n", - " # convert indexes to alternative names\n", - " choices = pd.Series(model_spec.columns[choices.values], index=choices.index)\n", - "\n", - " finalise = True\n", - " if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - " return choices" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 70, - "outputs": [], - "source": [ - "#pipeline.close_pipeline()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 71, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle mandatory_tour_frequency not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running mandatory_tour_frequency with %d persons 89\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle mandatory_tour_frequency not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running mandatory_tour_frequency with %d persons 89\n" - ] - } - ], - "source": [ - "ch_prev = run_mandatory_tour_frequency(choose_individual_max_utility=False)\n", - "ch_new = run_mandatory_tour_frequency(choose_individual_max_utility=True)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 72, - "outputs": [ - { - "data": { - "text/plain": " count_prev count_new\nschool1 22 22.0\nwork1 61 65.0\nwork2 2 2.0\nwork_and_school 4 0.0", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
count_prevcount_new
school12222.0
work16165.0
work222.0
work_and_school40.0
\n
" - }, - "execution_count": 72, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ch_prev.value_counts().to_frame(\"count_prev\").merge(\n", - " ch_new.value_counts().to_frame(\"count_new\"), how=\"outer\", left_index=True, right_index=True\n", - ").fillna(0)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 73, - "outputs": [ - { - "data": { - "text/plain": "{'input_table_list': [{'tablename': 'households',\n 'filename': 'households.csv',\n 'index_col': 'household_id',\n 'rename_columns': {'HHID': 'household_id',\n 'PERSONS': 'hhsize',\n 'workers': 'num_workers',\n 'VEHICL': 'auto_ownership',\n 'TAZ': 'home_zone_id'},\n 'keep_columns': ['home_zone_id',\n 'income',\n 'hhsize',\n 'HHT',\n 'auto_ownership',\n 'num_workers']},\n {'tablename': 'persons',\n 'filename': 'persons.csv',\n 'index_col': 'person_id',\n 'rename_columns': {'PERID': 'person_id'},\n 'keep_columns': ['household_id',\n 'age',\n 'PNUM',\n 'sex',\n 'pemploy',\n 'pstudent',\n 'ptype']},\n {'tablename': 'land_use',\n 'filename': 'land_use.csv',\n 'index_col': 'zone_id',\n 'rename_columns': {'TAZ': 'zone_id', 'COUNTY': 'county_id'},\n 'keep_columns': ['DISTRICT',\n 'SD',\n 'county_id',\n 'TOTHH',\n 'TOTPOP',\n 'TOTACRE',\n 'RESACRE',\n 'CIACRE',\n 'TOTEMP',\n 'AGE0519',\n 'RETEMPN',\n 'FPSEMPN',\n 'HEREMPN',\n 'OTHEMPN',\n 'AGREMPN',\n 'MWTEMPN',\n 'PRKCST',\n 'OPRKCST',\n 'area_type',\n 'HSENROLL',\n 'COLLFTE',\n 'COLLPTE',\n 'TOPOLOGY',\n 'TERMINAL']}],\n 'households_sample_size': 100,\n 'check_for_variability': False,\n 'use_shadow_pricing': False,\n 'want_dest_choice_sample_tables': False,\n 'log_alt_losers': False,\n 'chunk_size': 0,\n 'min_available_chunk_ratio': 0.05,\n 'default_initial_rows_per_chunk': 500,\n 'chunk_method': 'hybrid_uss',\n 'chunk_training_mode': 'disabled',\n 'keep_chunk_logs': True,\n 'keep_mem_logs': True,\n 'trace_hh_id': [982875],\n 'trace_od': None,\n 'resume_after': None,\n 'checkpoints': True,\n 'models': ['initialize_landuse',\n 'initialize_households',\n 'compute_accessibility',\n 'school_location',\n 'workplace_location',\n 'auto_ownership_simulate',\n 'free_parking',\n 'cdap_simulate',\n 'mandatory_tour_frequency',\n 'mandatory_tour_scheduling',\n 'joint_tour_frequency',\n 'joint_tour_composition',\n 'joint_tour_participation',\n 'joint_tour_destination',\n 'joint_tour_scheduling',\n 'non_mandatory_tour_frequency',\n 'non_mandatory_tour_destination',\n 'non_mandatory_tour_scheduling',\n 'tour_mode_choice_simulate',\n 'atwork_subtour_frequency',\n 'atwork_subtour_destination',\n 'atwork_subtour_scheduling',\n 'atwork_subtour_mode_choice',\n 'stop_frequency',\n 'trip_purpose',\n 'trip_destination',\n 'trip_purpose_and_destination',\n 'trip_scheduling',\n 'trip_mode_choice',\n 'write_data_dictionary',\n 'track_skim_usage',\n 'write_trip_matrices',\n 'write_tables'],\n 'output_tables': {'h5_store': False,\n 'action': 'include',\n 'prefix': 'final_',\n 'tables': ['checkpoints',\n 'accessibility',\n 'land_use',\n 'households',\n 'persons',\n 'tours',\n 'trips',\n 'joint_tour_participants']},\n 'urban_threshold': 4,\n 'cbd_threshold': 2,\n 'rural_threshold': 6,\n 'min_value_of_time': 1,\n 'max_value_of_time': 50,\n 'distributed_vot_mu': 0.684,\n 'distributed_vot_sigma': 0.85,\n 'household_median_value_of_time': {1: 6.01, 2: 8.81, 3: 10.44, 4: 12.86},\n 'source_file_paths': ['configs/settings.yaml']}" - }, - "execution_count": 73, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "inject.get_injectable('settings')" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 74, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle mandatory_tour_frequency not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running mandatory_tour_frequency with %d persons 89\n" - ] - } - ], - "source": [ - "resume_after = \"cdap_simulate\"\n", - "model_name = \"mandatory_tour_frequency\"\n", - "chunk_size = 0 # test_mtc means no chunking\n", - "\n", - "pipeline.open_pipeline(resume_after)\n", - "# preload any bulky injectables (e.g. skims) not in pipeline\n", - "inject.get_injectable('preload_injectables', None)\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "#step_name = model_name\n", - "args = {}\n", - "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "#network_los = inject.get_injectable('network_los')\n", - "persons_merged = inject.get_table('persons_merged')\n", - "\n", - "trace_label = 'mandatory_tour_frequency'\n", - "model_settings_file_name = 'mandatory_tour_frequency.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "choosers = persons_merged.to_frame()\n", - "# filter based on results of CDAP\n", - "choosers = choosers[choosers.cdap_activity == 'M']\n", - "print(\"Running mandatory_tour_frequency with %d persons\", len(choosers))\n", - "preprocessor_settings = model_settings.get('preprocessor', None)\n", - "if preprocessor_settings:\n", - " locals_dict = {}\n", - " expressions.assign_columns(\n", - " df=choosers,\n", - " model_settings=preprocessor_settings,\n", - " locals_dict=locals_dict,\n", - " trace_label=trace_label)\n", - "\n", - "estimator = estimation.manager.begin_estimation('mandatory_tour_frequency')\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "coefficients_df = simulate.read_model_coefficients(model_settings)\n", - "model_spec = simulate.eval_coefficients(model_spec, coefficients_df, estimator)\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "constants = config.get_model_constants(model_settings)\n", - "choices = simulate.simple_simulate(\n", - " choosers=choosers,\n", - " spec=model_spec,\n", - " nest_spec=nest_spec,\n", - " locals_d=constants,\n", - " chunk_size=chunk_size,\n", - " trace_label=trace_label,\n", - " trace_choice_name='mandatory_tour_frequency',\n", - " estimator=estimator,\n", - " choose_individual_max_utility=True)\n", - "\n", - "# convert indexes to alternative names\n", - "choices = pd.Series(model_spec.columns[choices.values], index=choices.index)\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 58, - "outputs": [ - { - "data": { - "text/plain": " a b c\n0 1 2 3\n1 4 6 5\n2 7 8 9", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
abc
0123
1465
2789
\n
" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "df_ = pd.DataFrame([[1,2,3],[4,6,5],[7,8,9]], columns=[\"a\", \"b\", \"c\"])\n", - "display(df_)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 64, - "outputs": [ - { - "data": { - "text/plain": "{'a': 0, 'b': 1, 'c': 2}" - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "{v: k for k,v in enumerate(df_.columns)}" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/frozen_impl_interaction_sample_dev.ipynb b/notebooks/frozen_impl_interaction_sample_dev.ipynb deleted file mode 100644 index 79412c9265..0000000000 --- a/notebooks/frozen_impl_interaction_sample_dev.ipynb +++ /dev/null @@ -1,1660 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "%load_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "c3dba451-1e10-403e-8614-35d57e6577f4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import argparse\n", - "from datetime import datetime\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from activitysim.cli import run\n", - "from activitysim.core import inject" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "pd.set_option(\"display.max_columns\", 500)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "root_dir = \"/home/jan/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_runs\", \"seq_asim\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] No such file or directory: '/home/jan/code/activitysim/test_runs/seq_asim'", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mFileNotFoundError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/tmp/ipykernel_24910/2284515201.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mos\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchdir\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mexample_dir\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", - "\u001B[0;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: '/home/jan/code/activitysim/test_runs/seq_asim'" - ] - } - ], - "source": [ - "os.chdir(example_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "parser = argparse.ArgumentParser()\n", - "run.add_run_args(parser)\n", - "args = parser.parse_args(['-c', 'configs_lessmodes', '-o', 'output_fru', '-d', 'data'])\n", - "#run.run(args) # 2mins full example run\n", - "if not inject.is_injectable('preload_injectables'):\n", - " from activitysim import abm # register abm steps and other abm-specific injectables\n", - "run.handle_standard_args(args) # possibly update injectables" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9955cc9c", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "from activitysim.core import inject\n", - "from activitysim.core import pipeline\n", - "from activitysim.core import config\n", - "from activitysim.core import simulate\n", - "from activitysim.abm.models.util import estimation\n", - "from activitysim.abm.tables import shadow_pricing\n", - "from activitysim.core import interaction_simulate\n", - "from activitysim.core import logit\n", - "from activitysim.core.simulate import set_skim_wrapper_targets\n", - "from activitysim.core import chunk" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "3a923505", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "from activitysim.core.logit import inverse_ev1_cdf\n", - "\n", - "def hack_make_sample_choices(\n", - " choosers, probs,\n", - " alternatives,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs,\n", - " trace_label,\n", - " utilities=None,\n", - " choose_individual_max_utility=False\n", - "):\n", - " assert isinstance(probs, pd.DataFrame)\n", - " assert probs.shape == (len(choosers), alternative_count)\n", - " assert isinstance(alternatives, pd.DataFrame)\n", - " assert len(alternatives) == alternative_count\n", - "\n", - " if allow_zero_probs:\n", - " zero_probs = (probs.sum(axis=1) == 0)\n", - " if zero_probs.all():\n", - " return pd.DataFrame(columns=[alt_col_name, 'rand', 'prob', choosers.index.name])\n", - " if zero_probs.any():\n", - " # remove from sample\n", - " probs = probs[~zero_probs]\n", - " choosers = choosers[~zero_probs]\n", - " # TODO [janzill Jun2022]: do we want this for consistency?\n", - " # might need this in other places too?\n", - " if utilities is not None:\n", - " utilities = utilities[~zero_probs]\n", - "\n", - " if choose_individual_max_utility:\n", - " assert isinstance(utilities, pd.DataFrame)\n", - " #print(utilities.head(3))\n", - " assert utilities.shape == (len(choosers), alternative_count)\n", - " #print(utilities.shape)\n", - "\n", - " choice_dimension = (len(choosers), alternative_count, sample_size)\n", - " rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count*sample_size)\n", - " #print(f\"after generation rands shape = {rands.shape}\", flush=True)\n", - " rands = rands.reshape(choice_dimension)\n", - " #print(f\"before inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", - " rands = inverse_ev1_cdf(rands)\n", - " #print(f\"after inverse_ev1 rands shape = {rands.shape}\", flush=True)\n", - " utilities = utilities.to_numpy() # this should be much cleaner once xarray changes are implemented\n", - " utilities = np.repeat(utilities[:,:,None], sample_size, axis=2)\n", - " #print(f\"after utils reshape: {utilities.shape}\", flush=True)\n", - " utilities += rands\n", - " # this gives us len(choosers), sample_size dimensions, with values the chosen alternative\n", - " choices_array = np.argmax(utilities, axis=1)\n", - " print(choices_array.shape)\n", - "\n", - " choosers_index_rep = np.tile(np.arange(0,choices_array.shape[0]), sample_size)\n", - " #np.repeat(np.arange(0,choices_array.shape[0]), sample_size)\n", - " choices_flattened = choices_array.flatten(order='F')\n", - " #print(f\"choices flattened shape = {choices_flattened.shape}\")\n", - "\n", - " print(choosers_index_rep.shape, flush=True)\n", - " print(probs.shape, flush=True)\n", - " print(probs.head(3), flush=True)\n", - " probs_look_up = probs.to_numpy()[choosers_index_rep, choices_flattened]\n", - " #print(f\"probs_look_up shape = {probs_look_up.shape}\", flush=True)\n", - "\n", - " # choices_flattened are 0-based index into alternatives, need to map to alternative values given by\n", - " # alternatives.index.values (they are in this order by construction)\n", - " # explode to one row per chooser.index, alt_zone_id\n", - " choices_df = pd.DataFrame({\n", - " alt_col_name: alternatives.index.values[choices_flattened],\n", - " #'rand': rands.flatten(order='F'),\n", - " 'rand': np.zeros_like(choosers_index_rep), # TODO: zero out for now\n", - " 'prob': probs_look_up.flatten(order='F'),\n", - " # repeat is wrong here - we do not want 1,1,2,2,3,3, etc, but 1,2,3,1,2,3 by construction\n", - " #choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", - " choosers.index.name: np.tile(choosers.index.values, sample_size)\n", - " })\n", - "\n", - " else:\n", - " cum_probs_array = probs.values.cumsum(axis=1)\n", - " # alt probs in convenient layout to return prob of chose alternative\n", - " # (same layout as cum_probs_arr)\n", - " alt_probs_array = probs.values.flatten()\n", - " # get sample_size rands for each chooser\n", - " rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size)\n", - " # transform as we iterate over alternatives\n", - " # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr\n", - " # i.e rands[i] is a 2-D array of one alt choice rand for each chooser\n", - " rands = rands.T.reshape(sample_size, -1, 1)\n", - " # the alternative value chosen\n", - " choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype)\n", - " # chunk log these later after we populate them...\n", - " # the probability of the chosen alternative\n", - " choice_probs_array = np.empty([sample_size, len(choosers)])\n", - " # chunk log these later after we populate them...\n", - " alts = np.tile(alternatives.index.values, len(choosers))\n", - " # FIXME - do this all at once rather than iterate?\n", - " for i in range(sample_size):\n", - " # FIXME - do this in numpy, not pandas?\n", - " # rands for this alt in broadcastable shape\n", - " r = rands[i]\n", - "\n", - " # position of first occurrence of positive value\n", - " positions = np.argmax(cum_probs_array > r, axis=1)\n", - "\n", - " # FIXME - leave positions as numpy array, not pandas series?\n", - " # positions is series with the chosen alternative represented as a column index in probs\n", - " # which is an integer between zero and num alternatives in the alternative sample\n", - " positions = pd.Series(positions, index=probs.index)\n", - "\n", - " # need to get from an integer offset into the alternative sample to the alternative index\n", - " # that is, we want the index value of the row that is offset by rows into the\n", - " # tranche of this choosers alternatives created by cross join of alternatives and choosers\n", - "\n", - " # offsets is the offset into model_design df of first row of chooser alternatives\n", - " offsets = np.arange(len(positions)) * alternative_count\n", - "\n", - " # choices and choice_probs have one element per chooser and is in same order as choosers\n", - " choices_array[i] = np.take(alts, positions + offsets)\n", - " choice_probs_array[i] = np.take(alt_probs_array, positions + offsets)\n", - " del positions\n", - " del offsets\n", - "\n", - " del alts\n", - " del cum_probs_array\n", - " del alt_probs_array\n", - "\n", - " # explode to one row per chooser.index, alt_zone_id\n", - " choices_df = pd.DataFrame(\n", - " {alt_col_name: choices_array.flatten(order='F'),\n", - " 'rand': rands.flatten(order='F'),\n", - " 'prob': choice_probs_array.flatten(order='F'),\n", - " choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", - " })\n", - "\n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "7599d0d1-9063-48aa-95de-cef45e926a42", - "metadata": {}, - "outputs": [], - "source": [ - "def make_sample_choices_dev(\n", - " choosers, probs,\n", - " alternatives,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs,\n", - " trace_label,\n", - " utilities=None,\n", - " choose_individual_max_utility=False\n", - "):\n", - " assert isinstance(probs, pd.DataFrame)\n", - " assert probs.shape == (len(choosers), alternative_count)\n", - " assert isinstance(alternatives, pd.DataFrame)\n", - " assert len(alternatives) == alternative_count\n", - "\n", - " if allow_zero_probs:\n", - " zero_probs = (probs.sum(axis=1) == 0)\n", - " if zero_probs.all():\n", - " return pd.DataFrame(columns=[alt_col_name, 'rand', 'prob', choosers.index.name])\n", - " if zero_probs.any():\n", - " # remove from sample\n", - " probs = probs[~zero_probs]\n", - " choosers = choosers[~zero_probs]\n", - " # TODO [janzill Jun2022]: do we want this for consistency?\n", - " # might need this in other places too?\n", - " if utilities is not None:\n", - " utilities = utilities[~zero_probs]\n", - "\n", - "\n", - " if choose_individual_max_utility:\n", - " # Note [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension\n", - " # (len(choosers), alternative_count, sample_size) can get very large\n", - " choices_array = np.zeros_like(utilities, dtype=np.uint32)\n", - " zero_dim_index = np.arange(utilities.shape[0])\n", - "\n", - " utils_array = utilities.to_numpy() # TODO [janzill Jun2022]: once or for each?\n", - " for i in range(sample_size):\n", - " rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count)\n", - " choices_array[zero_dim_index, np.argmax(utils_array - np.log(-np.log(rands)), axis=1)] += 1\n", - "\n", - " i, j = np.nonzero(choices_array)\n", - " chunk.log_df(trace_label, 'i', i)\n", - " chunk.log_df(trace_label, 'j', j)\n", - "\n", - " #probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", - " # trace_label=trace_label, trace_choosers=choosers)\n", - " del utilities\n", - " choices_df = pd.DataFrame({\n", - " alt_col_name: alternatives.index.values[j],\n", - " \"pick_count\": choices_array[i, j],\n", - " \"prob\": probs.to_numpy()[i, j],\n", - " choosers.index.name: choosers.index.values[i]\n", - " })\n", - " del choices_array\n", - " del i\n", - " del j\n", - " # del probs\n", - " \n", - " else:\n", - " cum_probs_array = probs.values.cumsum(axis=1)\n", - " # alt probs in convenient layout to return prob of chose alternative\n", - " # (same layout as cum_probs_arr)\n", - " alt_probs_array = probs.values.flatten()\n", - " # get sample_size rands for each chooser\n", - " rands = pipeline.get_rn_generator().random_for_df(probs, n=sample_size)\n", - " # transform as we iterate over alternatives\n", - " # reshape so rands[i] is in broadcastable (2-D) shape for cum_probs_arr\n", - " # i.e rands[i] is a 2-D array of one alt choice rand for each chooser\n", - " rands = rands.T.reshape(sample_size, -1, 1)\n", - " # the alternative value chosen\n", - " choices_array = np.empty([sample_size, len(choosers)]).astype(alternatives.index.dtype)\n", - " # chunk log these later after we populate them...\n", - " # the probability of the chosen alternative\n", - " choice_probs_array = np.empty([sample_size, len(choosers)])\n", - " # chunk log these later after we populate them...\n", - " alts = np.tile(alternatives.index.values, len(choosers))\n", - " # FIXME - do this all at once rather than iterate?\n", - " for i in range(sample_size):\n", - " # FIXME - do this in numpy, not pandas?\n", - " # rands for this alt in broadcastable shape\n", - " r = rands[i]\n", - "\n", - " # position of first occurrence of positive value\n", - " positions = np.argmax(cum_probs_array > r, axis=1)\n", - "\n", - " # FIXME - leave positions as numpy array, not pandas series?\n", - " # positions is series with the chosen alternative represented as a column index in probs\n", - " # which is an integer between zero and num alternatives in the alternative sample\n", - " positions = pd.Series(positions, index=probs.index)\n", - "\n", - " # need to get from an integer offset into the alternative sample to the alternative index\n", - " # that is, we want the index value of the row that is offset by rows into the\n", - " # tranche of this choosers alternatives created by cross join of alternatives and choosers\n", - "\n", - " # offsets is the offset into model_design df of first row of chooser alternatives\n", - " offsets = np.arange(len(positions)) * alternative_count\n", - "\n", - " # choices and choice_probs have one element per chooser and is in same order as choosers\n", - " choices_array[i] = np.take(alts, positions + offsets)\n", - " choice_probs_array[i] = np.take(alt_probs_array, positions + offsets)\n", - " del positions\n", - " del offsets\n", - "\n", - " del alts\n", - " del cum_probs_array\n", - " del alt_probs_array\n", - "\n", - " # explode to one row per chooser.index, alt_zone_id\n", - " choices_df = pd.DataFrame(\n", - " {alt_col_name: choices_array.flatten(order='F'),\n", - " 'rand': rands.flatten(order='F'),\n", - " 'prob': choice_probs_array.flatten(order='F'),\n", - " choosers.index.name: np.repeat(np.asanyarray(choosers.index), sample_size)\n", - " })\n", - "\n", - " # pick_count and pick_dup\n", - " # pick_count is number of duplicate picks\n", - " # pick_dup flag is True for all but first of duplicates\n", - " pick_group = choices_df.groupby([choosers.index.name, alt_col_name])\n", - " # number each item in each group from 0 to the length of that group - 1.\n", - " choices_df['pick_count'] = pick_group.cumcount(ascending=True)\n", - " # flag duplicate rows after first\n", - " choices_df['pick_dup'] = choices_df['pick_count'] > 0\n", - " # add reverse cumcount to get total pick_count (conveniently faster than groupby.count + merge)\n", - " choices_df['pick_count'] += pick_group.cumcount(ascending=False) + 1\n", - " # drop the duplicates\n", - " choices_df = choices_df[~choices_df['pick_dup']]\n", - " del choices_df['pick_dup']\n", - " # set index after groupby so we can trace on it\n", - " choices_df.set_index(choosers.index.name, inplace=True)\n", - " # don't need this after tracing\n", - " del choices_df['rand']\n", - " \n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "1611fe26", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'pipeline' is not defined", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n", - "\u001B[0;31mNameError\u001B[0m: name 'pipeline' is not defined" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "choose_individual_max_utility = True\n", - "\n", - "resume_after = \"work_from_home\"\n", - "model_name = \"school_location\"\n", - "chunk_size = 0 # no chunking\n", - "\n", - "pipeline.open_pipeline(resume_after)\n", - "# preload any bulky injectables (e.g. skims) not in pipeline\n", - "inject.get_injectable('preload_injectables', None)\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "#step_name = model_name\n", - "args = {}\n", - "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "\n", - "persons_merged = inject.get_table('persons_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "households = inject.get_table('households')\n", - "persons = inject.get_table('persons')\n", - "locutor = inject.get_injectable('locutor')\n", - "\n", - "trace_label = model_name #'school_location'\n", - "model_settings_file_name = f\"{model_name}.yaml\" #'school_location.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "estimator = estimation.manager.begin_estimation(model_name)\n", - "# iterate_location_choice()\n", - "chunk_tag = trace_label\n", - "\n", - "# boolean to filter out persons not needing location modeling (e.g. is_worker, is_student)\n", - "chooser_filter_column = model_settings['CHOOSER_FILTER_COLUMN_NAME']\n", - "dest_choice_column_name = model_settings['DEST_CHOICE_COLUMN_NAME']\n", - "logsum_column_name = model_settings.get('DEST_CHOICE_LOGSUM_COLUMN_NAME')\n", - "sample_table_name = model_settings.get('DEST_CHOICE_SAMPLE_TABLE_NAME')\n", - "want_sample_table = config.setting('want_dest_choice_sample_tables') and sample_table_name is not None\n", - "persons_merged_df = persons_merged.to_frame()\n", - "persons_merged_df = persons_merged_df[persons_merged_df[chooser_filter_column]]\n", - "persons_merged_df.sort_index(inplace=True) # interaction_sample expects chooser index to be monotonic increasing\n", - "\n", - "# chooser segmentation allows different sets coefficients for e.g. different income_segments or tour_types\n", - "chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']\n", - "assert chooser_segment_column in persons_merged_df, f\"CHOOSER_SEGMENT_COLUMN '{chooser_segment_column}' not in \" \\\n", - " f\"persons_merged table.\"\n", - "shadow_price_calculator = shadow_pricing.load_shadow_price_calculator(model_settings)\n", - "chooser_segment_column = model_settings['CHOOSER_SEGMENT_COLUMN_NAME']\n", - "# maps segment names to compact (integer) ids\n", - "segment_ids = model_settings['SEGMENT_IDS']\n", - "\n", - "sample_list = []\n", - "for segment_name, segment_id in segment_ids.items():\n", - " print(f\"running {segment_name}, {segment_id}\")\n", - " choosers = persons_merged_df[persons_merged_df[chooser_segment_column] == segment_id]\n", - " # size_term and shadow price adjustment - one row per zone\n", - " dest_size_terms = shadow_price_calculator.dest_size_terms(segment_name)\n", - " assert dest_size_terms.index.is_monotonic_increasing, f\"shadow_price_calculator.dest_size_terms({segment_name}) \" \\\n", - " f\"not monotonic_increasing\"\n", - " if choosers.shape[0] == 0:\n", - " print(f\"{trace_label} skipping segment {segment_name}: no choosers\")\n", - " continue\n", - " print(f\"dropping {(~(dest_size_terms.size_term > 0)).sum()} \"\n", - " f\"of {len(dest_size_terms)} rows where size_term is zero\")\n", - " dest_size_terms = dest_size_terms[dest_size_terms.size_term > 0]\n", - " chooser_columns = model_settings['SIMULATE_CHOOSER_COLUMNS']\n", - " choosers_location_sample = choosers[chooser_columns]\n", - " skim_dict = network_los.get_default_skim_dict()\n", - " skims = skim_dict.wrap('home_zone_id', 'zone_id')\n", - " alt_dest_col_name = model_settings['ALT_DEST_COL_NAME']\n", - " assert not choosers_location_sample.empty\n", - " print(\"Running %s with %d persons\" % (trace_label, len(choosers_location_sample.index)))\n", - " sample_size = model_settings[\"SAMPLE_SIZE\"]\n", - " locals_d = {\n", - " 'skims': skims,\n", - " 'segment_size': segment_name\n", - " }\n", - " constants = config.get_model_constants(model_settings)\n", - " locals_d.update(constants)\n", - " spec = simulate.spec_for_segment(model_settings, spec_id='SAMPLE_SPEC',\n", - " segment_name=segment_name, estimator=estimator)\n", - " ### choices = interaction_sample()\n", - " alt_col_name=alt_dest_col_name\n", - " allow_zero_probs=False\n", - " log_alt_losers=False\n", - " # we return alternatives ordered in (index, alt_col_name)\n", - " # if choosers index is not ordered, it is probably a mistake, since the alts wont line up\n", - " assert alt_col_name is not None\n", - " assert choosers.index.is_monotonic_increasing\n", - "\n", - " # FIXME - legacy logic - not sure this is needed or even correct?\n", - " sample_size = min(sample_size, len(dest_size_terms.index))\n", - "\n", - " result_list = []\n", - " for i, chooser_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers(choosers_location_sample, chunk_size, trace_label,\n", - " chunk_tag):\n", - "\n", - " ### choices = hack_interaction_sample\n", - " # chooser = chooser_chunk\n", - " #alternatives = dest_size_terms\n", - " #trace_label=chunk_trace_label\n", - "\n", - " num_choosers = len(chooser_chunk.index)\n", - " assert num_choosers > 0\n", - " if len(spec.columns) > 1:\n", - " raise RuntimeError('spec must have only one column')\n", - " # if using skims, copy index into the dataframe, so it will be\n", - " # available as the \"destination\" for set_skim_wrapper_targets\n", - " if skims is not None and dest_size_terms.index.name not in dest_size_terms:\n", - " dest_size_terms = dest_size_terms.copy()\n", - " dest_size_terms[dest_size_terms.index.name] = dest_size_terms.index\n", - "\n", - " chooser_index_id = interaction_simulate.ALT_CHOOSER_ID if log_alt_losers else None\n", - "\n", - " # - cross join choosers and alternatives (cartesian product)\n", - " # for every chooser, there will be a row for each alternative\n", - " # index values (non-unique) are from alternatives df\n", - " alternative_count = dest_size_terms.shape[0]\n", - " interaction_df =\\\n", - " logit.interaction_dataset(chooser_chunk, dest_size_terms, sample_size=alternative_count,\n", - " chooser_index_id=chooser_index_id)\n", - "\n", - " assert alternative_count == len(interaction_df.index) / len(chooser_chunk.index)\n", - "\n", - " if skims is not None:\n", - " set_skim_wrapper_targets(interaction_df, skims)\n", - "\n", - " # evaluate expressions from the spec multiply by coefficients and sum\n", - " # spec is df with one row per spec expression and one col with utility coefficient\n", - " # column names of interaction_df match spec index values\n", - " # utilities has utility value for element in the cross product of choosers and alternatives\n", - " # interaction_utilities is a df with one utility column and one row per row in interaction_df\n", - " trace_rows = trace_ids = None\n", - "\n", - " # interaction_utilities is a df with one utility column and one row per interaction_df row\n", - " interaction_utilities, trace_eval_results = interaction_simulate.eval_interaction_utilities(\n", - " spec, interaction_df, locals_d, chunk_trace_label, trace_rows, estimator=None,\n", - " log_alt_losers=log_alt_losers\n", - " )\n", - " # ########### HWM - high water mark (point of max observed memory usage)\n", - " #del interaction_df\n", - "\n", - " # reshape utilities (one utility column and one row per row in interaction_utilities)\n", - " # to a dataframe with one row per chooser and one column per alternative\n", - " utilities = pd.DataFrame(\n", - " interaction_utilities.values.reshape(len(chooser_chunk), alternative_count),\n", - " index=chooser_chunk.index)\n", - " #del interaction_utilities\n", - "\n", - " # convert to probabilities (utilities exponentiated and normalized to probs)\n", - " # probs is same shape as utilities, one row per chooser and one column for alternative\n", - " probs = logit.utils_to_probs(utilities, allow_zero_probs=allow_zero_probs,\n", - " trace_label=chunk_trace_label, trace_choosers=chooser_chunk)\n", - " #del utilities\n", - "\n", - " choices_df = make_sample_choices_dev(\n", - " chooser_chunk, probs, dest_size_terms,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs=allow_zero_probs,\n", - " trace_label=chunk_trace_label,\n", - " utilities=utilities,\n", - " choose_individual_max_utility=choose_individual_max_utility\n", - " )\n", - " # - NARROW\n", - " choices_df['prob'] = choices_df['prob'].astype(np.float32)\n", - " assert (choices_df['pick_count'].max() < 4294967295) or (choices_df.empty)\n", - " choices_df['pick_count'] = choices_df['pick_count'].astype(np.uint32)\n", - "\n", - " if choices_df.shape[0] > 0:\n", - " result_list.append(choices_df)\n", - "\n", - " if len(result_list) > 1:\n", - " choices_df = pd.concat(result_list)\n", - " \n", - " # TODO: why does this fail\n", - " #assert allow_zero_probs or (len(choosers_location_sample.index) == len(np.unique(choices_df.index.values))), \\\n", - " # \"what is this\"\n", - " \n", - " # keep alts in canonical order so choices based on their probs are stable across runs\n", - " choices_df = choices_df.sort_values(by=alt_col_name).sort_index(kind='mergesort')\n", - "\n", - " sample_list.append(choices_df)\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0db07776-70ab-41d1-8dff-84cd7c30065a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "87f5534d-ae21-4f7b-9c85-cd910cc375ae", - "metadata": {}, - "outputs": [], - "source": [ - "def run_fru():\n", - " test_df = make_sample_choices_dev(\n", - " chooser_chunk, probs, dest_size_terms,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs=allow_zero_probs,\n", - " trace_label=chunk_trace_label,\n", - " utilities=utilities,\n", - " choose_individual_max_utility=True\n", - " )\n", - " \n", - "\n", - "def run_previous():\n", - " test_df = make_sample_choices_dev(\n", - " chooser_chunk, probs, dest_size_terms,\n", - " sample_size, alternative_count, alt_col_name,\n", - " allow_zero_probs=allow_zero_probs,\n", - " trace_label=chunk_trace_label,\n", - " utilities=None,\n", - " choose_individual_max_utility=False\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "22315306-6ec6-4daf-9176-ba397493c945", - "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " " - ] - }, - { - "data": { - "text/plain": [ - " 37642 function calls (37233 primitive calls) in 3.187 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 3.187 3.187 {built-in method builtins.exec}\n", - " 1 0.003 0.003 3.187 3.187 :1()\n", - " 1 0.000 0.000 3.185 3.185 2292568830.py:12(run_previous)\n", - " 1 2.165 2.165 3.185 3.185 3493973527.py:3(make_sample_choices_dev)\n", - " 170/148 0.006 0.000 0.344 0.002 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", - " 3 0.332 0.111 0.332 0.111 {method 'cumsum' of 'numpy.ndarray' objects}\n", - " 4 0.241 0.060 0.241 0.060 {method 'repeat' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.236 0.236 <__array_function__ internals>:177(tile)\n", - " 1 0.000 0.000 0.236 0.236 shape_base.py:1171(tile)\n", - " 4 0.221 0.055 0.221 0.055 {method 'flatten' of 'numpy.ndarray' objects}\n", - " 98 0.000 0.000 0.101 0.001 fromnumeric.py:51(_wrapfunc)\n", - " 30 0.000 0.000 0.077 0.003 <__array_function__ internals>:177(argmax)\n", - " 30 0.000 0.000 0.077 0.003 fromnumeric.py:1127(argmax)\n", - " 30 0.076 0.003 0.076 0.003 {method 'argmax' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.065 0.033 groupby.py:3040(cumcount)\n", - " 2 0.004 0.002 0.058 0.029 groupby.py:1699(_cumcount_array)\n", - " 1 0.000 0.000 0.042 0.042 ops.py:832(group_info)\n", - " 1 0.000 0.000 0.042 0.042 ops.py:854(_get_compressed_codes)\n", - " 1 0.000 0.000 0.030 0.030 sorting.py:661(compress_group_index)\n", - " 73 0.023 0.000 0.023 0.000 {method 'take' of 'numpy.ndarray' objects}\n", - " 1 0.001 0.001 0.022 0.022 random.py:562(random_for_df)\n", - " 60 0.000 0.000 0.019 0.000 <__array_function__ internals>:177(take)\n", - " 60 0.000 0.000 0.019 0.000 fromnumeric.py:93(take)\n", - " 1 0.000 0.000 0.018 0.018 sorting.py:683(_reorder_by_uniques)\n", - " 1 0.005 0.005 0.018 0.018 random.py:598()\n", - " 63 0.000 0.000 0.016 0.000 common.py:55(new_method)\n", - " 4 0.000 0.000 0.015 0.004 frame.py:587(__init__)\n", - " 1 0.000 0.000 0.015 0.015 construction.py:425(dict_to_mgr)\n", - " 1 0.000 0.000 0.015 0.015 construction.py:102(arrays_to_mgr)\n", - " 62 0.000 0.000 0.015 0.000 arraylike.py:98(__add__)\n", - " 1 0.000 0.000 0.015 0.015 managers.py:1937(create_block_manager_from_column_arrays)\n", - " 62 0.000 0.000 0.015 0.000 series.py:5637(_arith_method)\n", - " 62 0.001 0.000 0.014 0.000 base.py:1286(_arith_method)\n", - " 9 0.000 0.000 0.013 0.001 frame.py:3463(__getitem__)\n", - " 13471 0.013 0.000 0.013 0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n", - " 101 0.001 0.000 0.012 0.000 series.py:323(__init__)\n", - " 1 0.011 0.011 0.011 0.011 {method 'get_labels_groupby' of 'pandas._libs.hashtable.Int64HashTable' objects}\n", - " 3 0.000 0.000 0.011 0.004 generic.py:3708(_take_with_is_copy)\n", - " 3 0.000 0.000 0.011 0.004 generic.py:3609(take)\n", - " 1 0.000 0.000 0.011 0.011 ops.py:790(codes)\n", - " 1 0.000 0.000 0.011 0.011 ops.py:793()\n", - " 2 0.000 0.000 0.011 0.005 grouper.py:616(codes)\n", - " 2 0.000 0.000 0.011 0.005 grouper.py:659(_codes_and_uniques)\n", - " 2 0.000 0.000 0.011 0.005 algorithms.py:568(factorize)\n", - " 3 0.000 0.000 0.011 0.004 managers.py:1683(_consolidate_inplace)\n", - " 3 0.000 0.000 0.010 0.003 managers.py:2074(_consolidate)\n", - " 8 0.005 0.001 0.010 0.001 managers.py:2091(_merge_blocks)\n", - " 2 0.000 0.000 0.009 0.005 algorithms.py:524(factorize_array)\n", - " 2 0.009 0.004 0.009 0.004 {method 'factorize' of 'pandas._libs.hashtable.Int64HashTable' objects}\n", - " 63 0.000 0.000 0.009 0.000 series.py:2988(_construct_result)\n", - " 8 0.009 0.001 0.009 0.001 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 3 0.000 0.000 0.008 0.003 managers.py:875(take)\n", - " 1 0.000 0.000 0.007 0.007 managers.py:2008(_form_blocks)\n", - " 3 0.007 0.002 0.007 0.002 managers.py:2060(_stack_arrays)\n", - " 1 0.000 0.000 0.007 0.007 frame.py:3530(_getitem_bool_array)\n", - " 19 0.000 0.000 0.006 0.000 take.py:57(take_nd)\n", - " 19 0.000 0.000 0.006 0.000 take.py:120(_take_nd_ndarray)\n", - " 59/44 0.000 0.000 0.006 0.000 groupby.py:908(__getattribute__)\n", - " 3 0.000 0.000 0.006 0.002 managers.py:634(reindex_indexer)\n", - " 2 0.000 0.000 0.006 0.003 groupby.py:675(_selected_obj)\n", - " 7 0.000 0.000 0.006 0.001 blocks.py:1114(take_nd)\n", - " 17 0.000 0.000 0.005 0.000 <__array_function__ internals>:177(concatenate)\n", - " 3 0.000 0.000 0.005 0.002 <__array_function__ internals>:177(vstack)\n", - " 3 0.000 0.000 0.005 0.002 shape_base.py:222(vstack)\n", - " 3 0.000 0.000 0.005 0.002 <__array_function__ internals>:177(repeat)\n", - " 3 0.000 0.000 0.005 0.002 fromnumeric.py:436(repeat)\n", - " 1 0.004 0.004 0.004 0.004 {method 'put' of 'numpy.ndarray' objects}\n", - " 38 0.004 0.000 0.004 0.000 {built-in method numpy.arange}\n", - " 4861 0.003 0.000 0.004 0.000 {built-in method builtins.isinstance}\n", - " 1 0.000 0.000 0.003 0.003 managers.py:692()\n", - " 5 0.000 0.000 0.003 0.001 generic.py:5646(_consolidate_inplace)\n", - " 5 0.000 0.000 0.003 0.001 generic.py:5632(_protect_consolidate)\n", - " 5 0.000 0.000 0.003 0.001 generic.py:5650(f)\n", - " 5 0.000 0.000 0.003 0.001 managers.py:618(consolidate)\n", - " 102 0.001 0.000 0.003 0.000 construction.py:470(sanitize_array)\n", - " 5 0.000 0.000 0.003 0.001 managers.py:713(_slice_take_blocks_ax0)\n", - " 21 0.003 0.000 0.003 0.000 {built-in method numpy.asanyarray}\n", - " 62 0.000 0.000 0.002 0.000 array_ops.py:183(arithmetic_op)\n", - " 95 0.000 0.000 0.002 0.000 managers.py:1731(from_array)\n", - " 2 0.000 0.000 0.002 0.001 sorting.py:618(get_group_index_sorter)\n", - " 2 0.002 0.001 0.002 0.001 {pandas._libs.algos.groupsort_indexer}\n", - " 63 0.000 0.000 0.002 0.000 array_ops.py:134(_na_arithmetic_op)\n", - " 181 0.000 0.000 0.002 0.000 generic.py:5577(__setattr__)\n", - " 3 0.000 0.000 0.002 0.001 frame.py:3630(__setitem__)\n", - " 3 0.000 0.000 0.002 0.001 frame.py:3822(_set_item)\n", - " 96 0.000 0.000 0.002 0.000 blocks.py:2041(new_block)\n", - " 63 0.000 0.000 0.002 0.000 expressions.py:223(evaluate)\n", - " 63 0.000 0.000 0.002 0.000 expressions.py:92(_evaluate_numexpr)\n", - " 96 0.000 0.000 0.002 0.000 config.py:255(__call__)\n", - " 2 0.000 0.000 0.002 0.001 take.py:350(wrapper)\n", - " 1354 0.001 0.000 0.002 0.000 generic.py:43(_check)\n", - " 223 0.000 0.000 0.001 0.000 series.py:640(name)\n", - " 15/12 0.000 0.000 0.001 0.000 base.py:397(__new__)\n", - " 96 0.000 0.000 0.001 0.000 config.py:127(_get_option)\n", - " 63 0.000 0.000 0.001 0.000 expressions.py:63(_evaluate_standard)\n", - " 3 0.000 0.000 0.001 0.000 frame.py:3797(_set_item_mgr)\n", - " 2 0.000 0.000 0.001 0.001 base.py:5768(_get_indexer_strict)\n", - " 72 0.001 0.000 0.001 0.000 generic.py:5517(__finalize__)\n", - "1525/1157 0.001 0.000 0.001 0.000 {built-in method builtins.len}\n", - " 2 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis0_float64_float64}\n", - " 10 0.001 0.000 0.001 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.001 0.000 {built-in method builtins.next}\n", - " 4 0.000 0.000 0.001 0.000 groupby.py:998(_group_selection_context)\n", - " 171 0.000 0.000 0.001 0.000 construction.py:379(extract_array)\n", - " 62 0.001 0.000 0.001 0.000 {built-in method _operator.add}\n", - " 2 0.000 0.000 0.001 0.001 algorithms.py:1663(safe_sort)\n", - " 2 0.000 0.000 0.001 0.001 contextlib.py:108(__enter__)\n", - " 2 0.000 0.000 0.001 0.001 groupby.py:958(_set_group_selection)\n", - " 6 0.000 0.000 0.001 0.000 base.py:3706(get_indexer)\n", - " 5 0.000 0.000 0.001 0.000 base.py:1098(take)\n", - " 223 0.000 0.000 0.001 0.000 common.py:1721(validate_all_hashable)\n", - " 8 0.000 0.000 0.001 0.000 base.py:672(_with_infer)\n", - " 1 0.001 0.001 0.001 0.001 sorting.py:100(get_group_index)\n", - " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis1_int64_int64}\n", - " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis1_float64_float64}\n", - " 128 0.000 0.000 0.001 0.000 _ufunc_config.py:32(seterr)\n", - " 96 0.000 0.000 0.001 0.000 config.py:109(_get_single_key)\n", - " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis0_bool_bool}\n", - " 4 0.000 0.000 0.001 0.000 base.py:5744(get_indexer_for)\n", - " 228 0.000 0.000 0.001 0.000 {built-in method builtins.all}\n", - " 5 0.000 0.000 0.001 0.000 algorithms.py:1352(take)\n", - " 2 0.000 0.000 0.001 0.000 <__array_function__ internals>:177(nonzero)\n", - " 2 0.000 0.000 0.001 0.000 fromnumeric.py:1866(nonzero)\n", - " 1 0.000 0.000 0.001 0.001 generic.py:11363(__iadd__)\n", - " 1 0.000 0.000 0.001 0.001 generic.py:11337(_inplace_method)\n", - " 106 0.000 0.000 0.001 0.000 base.py:6987(ensure_index)\n", - " 1709 0.001 0.000 0.001 0.000 {built-in method builtins.getattr}\n", - " 2 0.000 0.000 0.001 0.000 managers.py:1219(insert)\n", - " 64 0.000 0.000 0.001 0.000 _ufunc_config.py:429(__enter__)\n", - " 2 0.000 0.000 0.001 0.000 base.py:3400(difference)\n", - " 110 0.000 0.000 0.001 0.000 base.py:7082(maybe_extract_name)\n", - " 3 0.000 0.000 0.001 0.000 generic.py:4020(__delitem__)\n", - " 2 0.000 0.000 0.001 0.000 base.py:4109(reindex)\n", - " 1 0.000 0.000 0.001 0.001 frame.py:3790(_iset_item_mgr)\n", - " 1 0.001 0.001 0.001 0.001 {pandas._libs.algos.take_2d_axis1_bool_bool}\n", - " 1 0.000 0.000 0.001 0.001 managers.py:1061(iset)\n", - " 12 0.001 0.000 0.001 0.000 {pandas._libs.algos.take_1d_int64_int64}\n", - " 102 0.000 0.000 0.001 0.000 construction.py:695(_try_cast)\n", - " 6 0.000 0.000 0.001 0.000 utils.py:249(maybe_convert_indices)\n", - " 5 0.000 0.000 0.001 0.000 index_tricks.py:322(__getitem__)\n", - " 3 0.000 0.000 0.001 0.000 managers.py:1299(idelete)\n", - " 446 0.000 0.000 0.001 0.000 common.py:1740()\n", - " 102 0.000 0.000 0.001 0.000 blocks.py:1960(maybe_coerce_values)\n", - " 105 0.000 0.000 0.001 0.000 generic.py:239(__init__)\n", - " 134 0.000 0.000 0.001 0.000 series.py:687(_values)\n", - " 1 0.000 0.000 0.001 0.001 blocks.py:378(delete)\n", - " 2 0.000 0.000 0.000 0.000 base.py:6561(insert)\n", - " 1 0.000 0.000 0.000 0.000 _decorators.py:302(wrapper)\n", - " 1 0.000 0.000 0.000 0.000 frame.py:5365(set_index)\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(delete)\n", - " 3 0.000 0.000 0.000 0.000 frame.py:4515(_sanitize_column)\n", - " 2 0.000 0.000 0.000 0.000 function_base.py:4958(delete)\n", - " 64 0.000 0.000 0.000 0.000 _ufunc_config.py:434(__exit__)\n", - " 99 0.000 0.000 0.000 0.000 common.py:552(require_length_match)\n", - " 3 0.000 0.000 0.000 0.000 frame.py:10986(_reindex_for_setitem)\n", - " 96 0.000 0.000 0.000 0.000 blocks.py:2055(check_ndim)\n", - " 2 0.000 0.000 0.000 0.000 base.py:3451(_difference)\n", - " 344 0.000 0.000 0.000 0.000 inference.py:321(is_hashable)\n", - " 6 0.000 0.000 0.000 0.000 {method 'copy' of 'numpy.ndarray' objects}\n", - " 105 0.000 0.000 0.000 0.000 blocks.py:1989(get_block_type)\n", - " 293 0.000 0.000 0.000 0.000 series.py:590(name)\n", - " 1 0.000 0.000 0.000 0.000 arraylike.py:54(__gt__)\n", - " 1 0.000 0.000 0.000 0.000 series.py:5613(_cmp_method)\n", - " 1 0.000 0.000 0.000 0.000 frame.py:7595(groupby)\n", - " 67 0.000 0.000 0.000 0.000 common.py:75(get_op_result_name)\n", - " 6 0.000 0.000 0.000 0.000 frame.py:3923(_get_item_cache)\n", - " 1 0.000 0.000 0.000 0.000 groupby.py:839(__init__)\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(diff)\n", - " 2 0.000 0.000 0.000 0.000 function_base.py:1295(diff)\n", - " 1 0.000 0.000 0.000 0.000 grouper.py:700(get_grouper)\n", - " 15 0.000 0.000 0.000 0.000 {pandas._libs.lib.maybe_convert_objects}\n", - " 61 0.000 0.000 0.000 0.000 series.py:825(__array__)\n", - " 13 0.000 0.000 0.000 0.000 base.py:5178(equals)\n", - " 134 0.000 0.000 0.000 0.000 managers.py:1848(internal_values)\n", - " 16 0.000 0.000 0.000 0.000 {method 'astype' of 'numpy.ndarray' objects}\n", - " 261 0.000 0.000 0.000 0.000 base.py:884(__len__)\n", - " 19 0.000 0.000 0.000 0.000 take.py:554(_take_preprocess_indexer_and_fill_value)\n", - " 63 0.000 0.000 0.000 0.000 dispatch.py:11(should_extension_dispatch)\n", - " 102 0.000 0.000 0.000 0.000 construction.py:627(_sanitize_ndim)\n", - " 9 0.000 0.000 0.000 0.000 base.py:7106(_maybe_cast_data_without_dtype)\n", - " 6 0.000 0.000 0.000 0.000 base.py:6293(_maybe_cast_listlike_indexer)\n", - " 1 0.000 0.000 0.000 0.000 {pandas._libs.algos.take_2d_axis0_int64_int64}\n", - " 128 0.000 0.000 0.000 0.000 _ufunc_config.py:131(geterr)\n", - " 95 0.000 0.000 0.000 0.000 construction.py:802(is_empty_data)\n", - " 96 0.000 0.000 0.000 0.000 config.py:589(_get_root)\n", - " 90 0.000 0.000 0.000 0.000 common.py:1587(_is_dtype_type)\n", - " 96 0.000 0.000 0.000 0.000 config.py:642(_warn_if_deprecated)\n", - " 178 0.000 0.000 0.000 0.000 construction.py:438(ensure_wrapped_if_datetimelike)\n", - " 5 0.000 0.000 0.000 0.000 frame.py:3411(_ixs)\n", - " 38 0.000 0.000 0.000 0.000 base.py:286(is_dtype)\n", - " 218 0.000 0.000 0.000 0.000 generic.py:5561(__getattr__)\n", - " 108 0.000 0.000 0.000 0.000 base.py:4820(_values)\n", - " 1 0.000 0.000 0.000 0.000 array_ops.py:227(comparison_op)\n", - " 28 0.000 0.000 0.000 0.000 common.py:497(is_categorical_dtype)\n", - " 12 0.000 0.000 0.000 0.000 base.py:554(_dtype_to_subclass)\n", - " 51 0.000 0.000 0.000 0.000 {built-in method numpy.empty}\n", - " 192 0.000 0.000 0.000 0.000 config.py:603(_get_deprecated_option)\n", - " 1 0.000 0.000 0.000 0.000 ops.py:684(shape)\n", - " 31 0.000 0.000 0.000 0.000 series.py:743(__len__)\n", - " 3 0.000 0.000 0.000 0.000 ops.py:686()\n", - " 8 0.000 0.000 0.000 0.000 base.py:6004(_should_compare)\n", - " 23 0.000 0.000 0.000 0.000 _dtype.py:328(_name_get)\n", - " 11 0.000 0.000 0.000 0.000 base.py:3577(get_loc)\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:603(ngroups)\n", - " 62 0.000 0.000 0.000 0.000 array_ops.py:441(maybe_prepare_scalar_for_op)\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:650(group_index)\n", - " 40 0.000 0.000 0.000 0.000 common.py:161(is_object_dtype)\n", - " 13 0.000 0.000 0.000 0.000 {pandas._libs.lib.infer_dtype}\n", - " 1 0.000 0.000 0.000 0.000 range.py:189(_data)\n", - " 4 0.000 0.000 0.000 0.000 algorithms.py:115(_ensure_data)\n", - " 62 0.000 0.000 0.000 0.000 __init__.py:152(align_method_SERIES)\n", - " 11 0.000 0.000 0.000 0.000 base.py:2596(inferred_type)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method _operator.gt}\n", - " 322 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_list_like}\n", - " 30 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects}\n", - " 95 0.000 0.000 0.000 0.000 series.py:542(_set_axis)\n", - " 396 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass}\n", - " 17 0.000 0.000 0.000 0.000 base.py:53(shape)\n", - " 102 0.000 0.000 0.000 0.000 construction.py:664(_sanitize_str_dtypes)\n", - " 5 0.000 0.000 0.000 0.000 {built-in method builtins.sorted}\n", - " 128 0.000 0.000 0.000 0.000 {built-in method numpy.seterrobj}\n", - " 359 0.000 0.000 0.000 0.000 {built-in method builtins.hash}\n", - " 8 0.000 0.000 0.000 0.000 warnings.py:130(filterwarnings)\n", - " 2 0.000 0.000 0.000 0.000 algorithms.py:299(_get_data_algo)\n", - " 22 0.000 0.000 0.000 0.000 managers.py:2079()\n", - " 11 0.000 0.000 0.000 0.000 base.py:6284(_maybe_cast_indexer)\n", - " 14 0.000 0.000 0.000 0.000 common.py:229(asarray_tuplesafe)\n", - " 14 0.000 0.000 0.000 0.000 {method 'any' of 'numpy.ndarray' objects}\n", - " 104 0.000 0.000 0.000 0.000 common.py:1416(is_1d_only_ea_dtype)\n", - " 31 0.000 0.000 0.000 0.000 base.py:45(__len__)\n", - " 105 0.000 0.000 0.000 0.000 flags.py:47(__init__)\n", - " 256 0.000 0.000 0.000 0.000 {built-in method numpy.geterrobj}\n", - " 6 0.000 0.000 0.000 0.000 base.py:3823(_check_indexing_method)\n", - " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(putmask)\n", - " 15 0.000 0.000 0.000 0.000 numeric.py:289(full)\n", - " 1 0.000 0.000 0.000 0.000 generic.py:1516(__invert__)\n", - " 102 0.000 0.000 0.000 0.000 construction.py:684(_maybe_repeat)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:596(_homogenize)\n", - " 8 0.000 0.000 0.000 0.000 blocks.py:166(_consolidate_key)\n", - " 96 0.000 0.000 0.000 0.000 config.py:630(_translate_key)\n", - " 23 0.000 0.000 0.000 0.000 common.py:581(is_dtype_equal)\n", - " 101 0.000 0.000 0.000 0.000 managers.py:1700(__init__)\n", - " 5 0.000 0.000 0.000 0.000 frame.py:3906(_box_col_values)\n", - " 13 0.000 0.000 0.000 0.000 base.py:5023(__getitem__)\n", - " 14 0.000 0.000 0.000 0.000 _methods.py:54(_any)\n", - " 167/166 0.000 0.000 0.000 0.000 {built-in method numpy.asarray}\n", - " 172 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 24 0.000 0.000 0.000 0.000 base.py:654(_simple_new)\n", - " 101 0.000 0.000 0.000 0.000 cast.py:1711(sanitize_to_nanoseconds)\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(insert)\n", - " 51 0.000 0.000 0.000 0.000 base.py:55()\n", - " 19 0.000 0.000 0.000 0.000 common.py:1240(is_float_dtype)\n", - " 16 0.000 0.000 0.000 0.000 base.py:2272(is_boolean)\n", - " 2 0.000 0.000 0.000 0.000 algorithms.py:289(_get_values_for_rank)\n", - " 2 0.000 0.000 0.000 0.000 function_base.py:5143(insert)\n", - " 11 0.000 0.000 0.000 0.000 base.py:2344(is_floating)\n", - " 72 0.000 0.000 0.000 0.000 flags.py:83(allows_duplicate_labels)\n", - " 96 0.000 0.000 0.000 0.000 config.py:571(_select_options)\n", - " 5 0.000 0.000 0.000 0.000 {method '_rebuild_blknos_and_blklocs' of 'pandas._libs.internals.BlockManager' objects}\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(setdiff1d)\n", - " 23 0.000 0.000 0.000 0.000 _dtype.py:314(_name_includes_bit_suffix)\n", - " 1 0.000 0.000 0.000 0.000 contextlib.py:72(inner)\n", - " 12 0.000 0.000 0.000 0.000 common.py:1274(is_bool_dtype)\n", - " 6 0.000 0.000 0.000 0.000 base.py:3786(_get_indexer)\n", - " 63 0.000 0.000 0.000 0.000 missing.py:138(dispatch_fill_zeros)\n", - " 6 0.000 0.000 0.000 0.000 common.py:105(is_bool_indexer)\n", - " 2 0.000 0.000 0.000 0.000 arraysetops.py:784(setdiff1d)\n", - " 5 0.000 0.000 0.000 0.000 managers.py:2137(_preprocess_slice_or_indexer)\n", - " 2 0.000 0.000 0.000 0.000 managers.py:1277(_insert_update_blklocs_and_blknos)\n", - " 4 0.000 0.000 0.000 0.000 blocks.py:238(fill_value)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:253(apply)\n", - " 2 0.000 0.000 0.000 0.000 frame.py:10808(values)\n", - " 30 0.000 0.000 0.000 0.000 common.py:1483(is_ea_or_datetimelike_dtype)\n", - " 19 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(copyto)\n", - " 15 0.000 0.000 0.000 0.000 base.py:4973(__contains__)\n", - " 209 0.000 0.000 0.000 0.000 typing.py:1149(cast)\n", - " 8 0.000 0.000 0.000 0.000 base.py:7168(unpack_nested_dtype)\n", - " 6 0.000 0.000 0.000 0.000 blocks.py:2030(new_block_2d)\n", - " 1 0.000 0.000 0.000 0.000 base.py:6947(ensure_index_from_sequences)\n", - " 96 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}\n", - " 4 0.000 0.000 0.000 0.000 missing.py:571(na_value_for_dtype)\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(in1d)\n", - " 4 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(append)\n", - " 1 0.000 0.000 0.000 0.000 indexing.py:2352(check_bool_indexer)\n", - " 6 0.000 0.000 0.000 0.000 common.py:459(is_interval_dtype)\n", - " 58 0.000 0.000 0.000 0.000 common.py:1552(get_dtype)\n", - " 13 0.000 0.000 0.000 0.000 {method 'max' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:396(apply)\n", - " 11 0.000 0.000 0.000 0.000 numerictypes.py:356(issubdtype)\n", - " 5 0.000 0.000 0.000 0.000 managers.py:1016(iget)\n", - " 59 0.000 0.000 0.000 0.000 common.py:147()\n", - " 2 0.000 0.000 0.000 0.000 arraysetops.py:519(in1d)\n", - " 31 0.000 0.000 0.000 0.000 common.py:155()\n", - " 7 0.000 0.000 0.000 0.000 common.py:680(is_integer_dtype)\n", - " 101 0.000 0.000 0.000 0.000 managers.py:1792(_block)\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(moveaxis)\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:468(__init__)\n", - " 4 0.000 0.000 0.000 0.000 function_base.py:5337(append)\n", - " 8 0.000 0.000 0.000 0.000 managers.py:1665(is_consolidated)\n", - " 5 0.000 0.000 0.000 0.000 numerictypes.py:597(find_common_type)\n", - " 6 0.000 0.000 0.000 0.000 dtypes.py:1206(is_dtype)\n", - " 1 0.000 0.000 0.000 0.000 {function SeedSequence.generate_state at 0x7fb5e410ac10}\n", - " 12 0.000 0.000 0.000 0.000 common.py:786(is_unsigned_integer_dtype)\n", - " 6 0.000 0.000 0.000 0.000 base.py:845(_engine)\n", - " 63 0.000 0.000 0.000 0.000 expressions.py:72(_can_use_numexpr)\n", - " 145 0.000 0.000 0.000 0.000 generic.py:349(flags)\n", - " 12 0.000 0.000 0.000 0.000 common.py:732(is_signed_integer_dtype)\n", - " 6 0.000 0.000 0.000 0.000 base.py:5917(_maybe_promote)\n", - " 13 0.000 0.000 0.000 0.000 _methods.py:38(_amax)\n", - " 2 0.000 0.000 0.000 0.000 numeric.py:1404(moveaxis)\n", - " 2 0.000 0.000 0.000 0.000 contextlib.py:117(__exit__)\n", - " 5 0.000 0.000 0.000 0.000 managers.py:1673(_consolidate_check)\n", - " 63 0.000 0.000 0.000 0.000 _ufunc_config.py:425(__init__)\n", - " 17 0.000 0.000 0.000 0.000 common.py:1429(is_extension_array_dtype)\n", - " 8 0.000 0.000 0.000 0.000 warnings.py:181(_add_filter)\n", - " 8 0.000 0.000 0.000 0.000 re.py:250(compile)\n", - " 34 0.000 0.000 0.000 0.000 range.py:909(__len__)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:635(_extract_index)\n", - " 2 0.000 0.000 0.000 0.000 base.py:3465(_wrap_difference_result)\n", - " 2 0.000 0.000 0.000 0.000 base.py:5799(_raise_if_missing)\n", - " 1 0.000 0.000 0.000 0.000 utils.py:457(check_array_indexer)\n", - " 6 0.000 0.000 0.000 0.000 {method 'get_indexer' of 'pandas._libs.index.IndexEngine' objects}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method _operator.invert}\n", - " 2 0.000 0.000 0.000 0.000 base.py:3258(_wrap_setop_result)\n", - " 10 0.000 0.000 0.000 0.000 frame.py:1413(__len__)\n", - " 95 0.000 0.000 0.000 0.000 numeric.py:331(_is_all_dates)\n", - " 72 0.000 0.000 0.000 0.000 generic.py:328(attrs)\n", - " 62 0.000 0.000 0.000 0.000 array_ops.py:510(_bool_arith_check)\n", - " 3 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(argsort)\n", - " 75 0.000 0.000 0.000 0.000 {pandas._libs.lib.item_from_zerodim}\n", - " 2 0.000 0.000 0.000 0.000 algorithms.py:308(_check_object_for_strings)\n", - " 2 0.000 0.000 0.000 0.000 algorithms.py:193(_reconstruct_data)\n", - " 8 0.000 0.000 0.000 0.000 re.py:289(_compile)\n", - " 22 0.000 0.000 0.000 0.000 numerictypes.py:282(issubclass_)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:487()\n", - " 15 0.000 0.000 0.000 0.000 :1017(_handle_fromlist)\n", - " 3 0.000 0.000 0.000 0.000 numeric.py:139(_ensure_array)\n", - " 4 0.000 0.000 0.000 0.000 numeric.py:149(ones)\n", - " 15 0.000 0.000 0.000 0.000 managers.py:156(blknos)\n", - " 5 0.000 0.000 0.000 0.000 common.py:97(_maybe_match_name)\n", - " 3 0.000 0.000 0.000 0.000 _asarray.py:22(require)\n", - " 4 0.000 0.000 0.000 0.000 base.py:118(_reset_cache)\n", - " 6 0.000 0.000 0.000 0.000 indexing.py:2318(convert_to_index_sliceable)\n", - " 2 0.000 0.000 0.000 0.000 generic.py:1991(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 managers.py:1541(as_array)\n", - " 4 0.000 0.000 0.000 0.000 common.py:348(is_datetime64tz_dtype)\n", - " 12 0.000 0.000 0.000 0.000 base.py:4846(_get_engine_target)\n", - " 2 0.000 0.000 0.000 0.000 managers.py:1267(_insert_update_mgr_locs)\n", - " 13 0.000 0.000 0.000 0.000 missing.py:390(array_equivalent)\n", - " 2 0.000 0.000 0.000 0.000 groupby.py:985(_reset_group_selection)\n", - " 73 0.000 0.000 0.000 0.000 flags.py:51(allows_duplicate_labels)\n", - " 11 0.000 0.000 0.000 0.000 inference.py:184(is_array_like)\n", - " 18 0.000 0.000 0.000 0.000 {built-in method numpy.array}\n", - " 1 0.000 0.000 0.000 0.000 construction.py:483()\n", - " 4 0.000 0.000 0.000 0.000 base.py:2232(is_unique)\n", - " 21 0.000 0.000 0.000 0.000 common.py:160(cast_scalar_indexer)\n", - " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(prod)\n", - " 9 0.000 0.000 0.000 0.000 indexing.py:2481(check_deprecated_indexers)\n", - " 9 0.000 0.000 0.000 0.000 blocks.py:2119(extend_blocks)\n", - " 64 0.000 0.000 0.000 0.000 series.py:523(_constructor)\n", - " 60 0.000 0.000 0.000 0.000 fromnumeric.py:89(_take_dispatcher)\n", - " 11 0.000 0.000 0.000 0.000 {method 'get_loc' of 'pandas._libs.index.IndexEngine' objects}\n", - " 4 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(ravel)\n", - " 2 0.000 0.000 0.000 0.000 base.py:3560(_convert_can_do_setop)\n", - " 3 0.000 0.000 0.000 0.000 fromnumeric.py:1012(argsort)\n", - " 4 0.000 0.000 0.000 0.000 common.py:287(maybe_iterable_to_list)\n", - " 59 0.000 0.000 0.000 0.000 common.py:145(classes)\n", - " 4 0.000 0.000 0.000 0.000 numeric.py:1341(normalize_axis_tuple)\n", - " 9 0.000 0.000 0.000 0.000 base.py:2624(_is_multi)\n", - " 30 0.000 0.000 0.000 0.000 fromnumeric.py:1123(_argmax_dispatcher)\n", - " 3 0.000 0.000 0.000 0.000 cast.py:1960(construct_1d_object_array_from_listlike)\n", - " 74 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'delete' of 'pandas._libs.internals.BlockPlacement' objects}\n", - " 3 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(atleast_2d)\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:2970(prod)\n", - " 8 0.000 0.000 0.000 0.000 warnings.py:458(__enter__)\n", - " 5 0.000 0.000 0.000 0.000 managers.py:1679()\n", - " 2 0.000 0.000 0.000 0.000 {method 'sum' of 'numpy.ndarray' objects}\n", - " 5 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(ndim)\n", - " 9 0.000 0.000 0.000 0.000 series.py:575(dtype)\n", - " 5 0.000 0.000 0.000 0.000 series.py:1238(_set_as_cached)\n", - " 11 0.000 0.000 0.000 0.000 generic.py:560(_get_axis)\n", - " 37 0.000 0.000 0.000 0.000 managers.py:222(items)\n", - " 11 0.000 0.000 0.000 0.000 abc.py:96(__instancecheck__)\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:69(_wrapreduction)\n", - " 9 0.000 0.000 0.000 0.000 blocks.py:332(getitem_block_columns)\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:427(_split_op_result)\n", - " 2 0.000 0.000 0.000 0.000 generic.py:1752(_check_label_or_level_ambiguity)\n", - " 2 0.000 0.000 0.000 0.000 _methods.py:46(_sum)\n", - " 13 0.000 0.000 0.000 0.000 base.py:803(is_)\n", - " 46 0.000 0.000 0.000 0.000 {pandas._libs.algos.ensure_platform_int}\n", - " 19 0.000 0.000 0.000 0.000 take.py:326(_get_take_nd_function)\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:925(_convert_grouper)\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:252(make_block)\n", - " 1 0.000 0.000 0.000 0.000 generic.py:767(_set_axis)\n", - " 3 0.000 0.000 0.000 0.000 blocks.py:2135(ensure_block_shape)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method builtins.any}\n", - " 25 0.000 0.000 0.000 0.000 base.py:834(_reset_identity)\n", - " 2 0.000 0.000 0.000 0.000 managers.py:2128(_fast_count_smallints)\n", - " 3 0.000 0.000 0.000 0.000 shape_base.py:81(atleast_2d)\n", - " 26 0.000 0.000 0.000 0.000 {built-in method __new__ of type object at 0x908780}\n", - " 7 0.000 0.000 0.000 0.000 blocks.py:267(make_block_same_class)\n", - " 12 0.000 0.000 0.000 0.000 common.py:346(apply_if_callable)\n", - " 4 0.000 0.000 0.000 0.000 fromnumeric.py:1755(ravel)\n", - " 2 0.000 0.000 0.000 0.000 missing.py:625(is_valid_na_for_dtype)\n", - " 8 0.000 0.000 0.000 0.000 warnings.py:477(__exit__)\n", - " 36 0.000 0.000 0.000 0.000 base.py:1650(name)\n", - " 9 0.000 0.000 0.000 0.000 cast.py:468(maybe_promote)\n", - " 2 0.000 0.000 0.000 0.000 base.py:2844(unique)\n", - " 3 0.000 0.000 0.000 0.000 missing.py:66(isna)\n", - " 9 0.000 0.000 0.000 0.000 managers.py:1837(dtype)\n", - " 1 0.000 0.000 0.000 0.000 frame.py:821(shape)\n", - " 1 0.000 0.000 0.000 0.000 random.py:721(getrandbits)\n", - " 23 0.000 0.000 0.000 0.000 _dtype.py:24(_kind_name)\n", - " 8 0.000 0.000 0.000 0.000 {method 'remove' of 'list' objects}\n", - " 8 0.000 0.000 0.000 0.000 types.py:171(__get__)\n", - " 9 0.000 0.000 0.000 0.000 generic.py:636(_info_axis)\n", - " 2 0.000 0.000 0.000 0.000 groupby.py:1734(_obj_1d_constructor)\n", - " 9 0.000 0.000 0.000 0.000 range.py:522(equals)\n", - " 6 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 generic.py:1399(_indexed_same)\n", - " 10 0.000 0.000 0.000 0.000 numerictypes.py:573(_can_coerce_all)\n", - " 11 0.000 0.000 0.000 0.000 {method 'format' of 'str' objects}\n", - " 4 0.000 0.000 0.000 0.000 managers.py:1992(_grouping_func)\n", - " 3 0.000 0.000 0.000 0.000 managers.py:2102()\n", - " 31 0.000 0.000 0.000 0.000 common.py:150(classes_and_not_datetimelike)\n", - " 2 0.000 0.000 0.000 0.000 base.py:4931(_validate_fill_value)\n", - " 18 0.000 0.000 0.000 0.000 generic.py:546(_get_axis_number)\n", - " 2 0.000 0.000 0.000 0.000 base.py:785(_view)\n", - " 3 0.000 0.000 0.000 0.000 missing.py:149(_isna)\n", - " 13 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", - " 1 0.000 0.000 0.000 0.000 base.py:841(_cleanup)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:212(set_axis)\n", - " 2 0.000 0.000 0.000 0.000 base.py:4226(_wrap_reindex_result)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 contextlib.py:238(helper)\n", - " 3 0.000 0.000 0.000 0.000 base.py:358(size)\n", - " 6 0.000 0.000 0.000 0.000 missing.py:911(clean_reindex_fill_method)\n", - " 8 0.000 0.000 0.000 0.000 warnings.py:437(__init__)\n", - " 9 0.000 0.000 0.000 0.000 {pandas._libs.lib.maybe_indices_to_slice}\n", - " 24 0.000 0.000 0.000 0.000 {built-in method _warnings._filters_mutated}\n", - " 7 0.000 0.000 0.000 0.000 managers.py:217(is_single_block)\n", - " 3 0.000 0.000 0.000 0.000 managers.py:973(from_blocks)\n", - " 3 0.000 0.000 0.000 0.000 frame.py:3883(_ensure_valid_index)\n", - " 2 0.000 0.000 0.000 0.000 base.py:768(_shallow_copy)\n", - " 2 0.000 0.000 0.000 0.000 groupby.py:977()\n", - " 4 0.000 0.000 0.000 0.000 common.py:1747(pandas_dtype)\n", - " 11 0.000 0.000 0.000 0.000 {built-in method _abc._abc_instancecheck}\n", - " 24 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_float}\n", - " 2 0.000 0.000 0.000 0.000 generic.py:1401()\n", - " 1 0.000 0.000 0.000 0.000 api.py:322(default_index)\n", - " 4 0.000 0.000 0.000 0.000 frame.py:3920(_clear_item_cache)\n", - " 3 0.000 0.000 0.000 0.000 generic.py:3925(_set_is_copy)\n", - " 4 0.000 0.000 0.000 0.000 common.py:1148(needs_i8_conversion)\n", - " 3 0.000 0.000 0.000 0.000 _asarray.py:111()\n", - " 3 0.000 0.000 0.000 0.000 shape_base.py:218(_vhstack_dispatcher)\n", - " 19 0.000 0.000 0.000 0.000 multiarray.py:1071(copyto)\n", - " 1 0.000 0.000 0.000 0.000 base.py:57(_validate_set_axis)\n", - " 3 0.000 0.000 0.000 0.000 base.py:2642(_na_value)\n", - " 16 0.000 0.000 0.000 0.000 base.py:937(dtype)\n", - " 2 0.000 0.000 0.000 0.000 base.py:4230(_maybe_preserve_names)\n", - " 13 0.000 0.000 0.000 0.000 blocks.py:354(dtype)\n", - " 2 0.000 0.000 0.000 0.000 algorithms.py:237(_ensure_arraylike)\n", - " 2 0.000 0.000 0.000 0.000 cast.py:2147(can_hold_element)\n", - " 6 0.000 0.000 0.000 0.000 {pandas._libs.internals.get_blkno_placements}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method posix.urandom}\n", - " 15 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_scalar}\n", - " 17 0.000 0.000 0.000 0.000 multiarray.py:148(concatenate)\n", - " 9 0.000 0.000 0.000 0.000 blocks.py:310(_slice)\n", - " 2 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(bincount)\n", - " 2 0.000 0.000 0.000 0.000 base.py:3554(_assert_can_do_setop)\n", - " 19 0.000 0.000 0.000 0.000 base.py:326(ndim)\n", - " 3 0.000 0.000 0.000 0.000 generic.py:566(_get_block_manager_axis)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:494()\n", - " 14 0.000 0.000 0.000 0.000 base.py:5905(_index_as_unique)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method numpy.zeros}\n", - " 2 0.000 0.000 0.000 0.000 contextlib.py:82(__init__)\n", - " 14 0.000 0.000 0.000 0.000 managers.py:172(blklocs)\n", - " 5 0.000 0.000 0.000 0.000 numerictypes.py:649()\n", - " 10 0.000 0.000 0.000 0.000 {method 'insert' of 'list' objects}\n", - " 2 0.000 0.000 0.000 0.000 base.py:723(tolist)\n", - " 2 0.000 0.000 0.000 0.000 blocks.py:222(get_values)\n", - " 4 0.000 0.000 0.000 0.000 {method 'transpose' of 'numpy.ndarray' objects}\n", - " 15 0.000 0.000 0.000 0.000 {built-in method builtins.callable}\n", - " 1 0.000 0.000 0.000 0.000 range.py:167(_simple_new)\n", - " 5 0.000 0.000 0.000 0.000 blocks.py:358(iget)\n", - " 4 0.000 0.000 0.000 0.000 numeric.py:1391()\n", - " 6 0.000 0.000 0.000 0.000 base.py:518()\n", - " 14 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_integer}\n", - " 3 0.000 0.000 0.000 0.000 grouper.py:802()\n", - " 15 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_iterator}\n", - " 4 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:570(name)\n", - " 10 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 9 0.000 0.000 0.000 0.000 base.py:540(_ensure_array)\n", - " 2 0.000 0.000 0.000 0.000 groupby.py:898(__getattr__)\n", - " 10 0.000 0.000 0.000 0.000 {method 'index' of 'list' objects}\n", - " 12 0.000 0.000 0.000 0.000 base.py:229(disallow_kwargs)\n", - " 9 0.000 0.000 0.000 0.000 blocks.py:244(mgr_locs)\n", - " 1 0.000 0.000 0.000 0.000 typing.py:868(__new__)\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:838(is_in_axis)\n", - " 3 0.000 0.000 0.000 0.000 shape_base.py:207(_arrays_for_stack_dispatcher)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method numpy.core._multiarray_umath.normalize_axis_index}\n", - " 1 0.000 0.000 0.000 0.000 managers.py:1722(from_blocks)\n", - " 2 0.000 0.000 0.000 0.000 base.py:1701(_get_names)\n", - " 9 0.000 0.000 0.000 0.000 managers.py:919(__init__)\n", - " 9 0.000 0.000 0.000 0.000 managers.py:1257()\n", - " 8 0.000 0.000 0.000 0.000 enum.py:753(value)\n", - " 8 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 pipeline.py:140(get_rn_generator)\n", - " 2 0.000 0.000 0.000 0.000 range.py:347(dtype)\n", - " 5 0.000 0.000 0.000 0.000 frame.py:804(axes)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method pandas._libs.missing.checknull}\n", - " 1 0.000 0.000 0.000 0.000 common.py:1042(is_numeric_v_string_like)\n", - " 5 0.000 0.000 0.000 0.000 {method 'clear' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 sorting.py:148()\n", - " 1 0.000 0.000 0.000 0.000 construction.py:486()\n", - " 3 0.000 0.000 0.000 0.000 grouper.py:804()\n", - " 5 0.000 0.000 0.000 0.000 generic.py:660(ndim)\n", - " 7 0.000 0.000 0.000 0.000 ops.py:680(groupings)\n", - " 5 0.000 0.000 0.000 0.000 fromnumeric.py:3164(ndim)\n", - " 3 0.000 0.000 0.000 0.000 grouper.py:803()\n", - " 1 0.000 0.000 0.000 0.000 ops.py:660(__init__)\n", - " 6 0.000 0.000 0.000 0.000 missing.py:107(clean_fill_method)\n", - " 1 0.000 0.000 0.000 0.000 sorting.py:135(_int64_cut_off)\n", - " 5 0.000 0.000 0.000 0.000 {pandas._libs.algos.ensure_int64}\n", - " 1 0.000 0.000 0.000 0.000 generic.py:4060(_check_inplace_and_allows_duplicate_labels)\n", - " 8 0.000 0.000 0.000 0.000 base.py:6022(_is_comparable_dtype)\n", - " 2 0.000 0.000 0.000 0.000 base.py:7046(ensure_has_len)\n", - " 5 0.000 0.000 0.000 0.000 base.py:692(_constructor)\n", - " 2 0.000 0.000 0.000 0.000 common.py:315(is_datetime64_dtype)\n", - " 5 0.000 0.000 0.000 0.000 numerictypes.py:650()\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:921(_is_label_like)\n", - " 5 0.000 0.000 0.000 0.000 base.py:1124(_maybe_disallow_fill)\n", - " 1 0.000 0.000 0.000 0.000 _decorators.py:214(_format_argument_list)\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:851(is_in_obj)\n", - " 2 0.000 0.000 0.000 0.000 common.py:389(is_timedelta64_dtype)\n", - " 1 0.000 0.000 0.000 0.000 base.py:890(__array__)\n", - " 5 0.000 0.000 0.000 0.000 fromnumeric.py:3160(_ndim_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 _validators.py:218(validate_bool_kwarg)\n", - " 3 0.000 0.000 0.000 0.000 generic.py:3948(_check_setitem_copy)\n", - " 3 0.000 0.000 0.000 0.000 fromnumeric.py:432(_repeat_dispatcher)\n", - " 3 0.000 0.000 0.000 0.000 {method 'upper' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 construction.py:233(mgr_to_mgr)\n", - " 3 0.000 0.000 0.000 0.000 managers.py:2111()\n", - " 4 0.000 0.000 0.000 0.000 fromnumeric.py:1751(_ravel_dispatcher)\n", - " 4 0.000 0.000 0.000 0.000 {built-in method _operator.index}\n", - " 3 0.000 0.000 0.000 0.000 function.py:49(__call__)\n", - " 3 0.000 0.000 0.000 0.000 frame.py:578(_constructor)\n", - " 2 0.000 0.000 0.000 0.000 cast.py:2185(np_can_hold_element)\n", - " 4 0.000 0.000 0.000 0.000 function_base.py:5333(_append_dispatcher)\n", - " 3 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 3 0.000 0.000 0.000 0.000 numeric.py:199(_ensure_dtype)\n", - " 1 0.000 0.000 0.000 0.000 pipeline.py:72(rng)\n", - " 3 0.000 0.000 0.000 0.000 numeric.py:188(_validate_dtype)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method pandas._libs.missing.is_matching_na}\n", - " 3 0.000 0.000 0.000 0.000 fromnumeric.py:1008(_argsort_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 base.py:3050(_validate_sort_keyword)\n", - " 3 0.000 0.000 0.000 0.000 shape_base.py:77(_atleast_2d_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:306(__len__)\n", - " 2 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 shape_base.py:1243()\n", - " 2 0.000 0.000 0.000 0.000 shape_base.py:1253()\n", - " 2 0.000 0.000 0.000 0.000 grouper.py:588(_ilevel)\n", - " 2 0.000 0.000 0.000 0.000 function_base.py:1291(_diff_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 numeric.py:1466()\n", - " 1 0.000 0.000 0.000 0.000 managers.py:1118(value_getitem)\n", - " 1 0.000 0.000 0.000 0.000 range.py:241(start)\n", - " 2 0.000 0.000 0.000 0.000 generic.py:1772()\n", - " 2 0.000 0.000 0.000 0.000 base.py:7191(_maybe_try_sort)\n", - " 2 0.000 0.000 0.000 0.000 function_base.py:5139(_insert_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 fromnumeric.py:1862(_nonzero_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 multiarray.py:883(bincount)\n", - " 1 0.000 0.000 0.000 0.000 range.py:287(step)\n", - " 2 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_bool_list}\n", - " 2 0.000 0.000 0.000 0.000 arraysetops.py:515(_in1d_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 arraysetops.py:780(_setdiff1d_dispatcher)\n", - " 2 0.000 0.000 0.000 0.000 function_base.py:4954(_delete_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:70()\n", - " 2 0.000 0.000 0.000 0.000 {method 'add' of 'set' objects}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method from_bytes}\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:248(mgr_locs)\n", - " 2 0.000 0.000 0.000 0.000 base.py:1898(nlevels)\n", - " 2 0.000 0.000 0.000 0.000 numeric.py:1400(_moveaxis_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 multiarray.py:1106(putmask)\n", - " 1 0.000 0.000 0.000 0.000 {method 'keys' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 contextlib.py:59(_recreate_cm)\n", - " 1 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'append' of 'pandas._libs.internals.BlockPlacement' objects}\n", - " 1 0.000 0.000 0.000 0.000 range.py:183(_constructor)\n", - " 1 0.000 0.000 0.000 0.000 range.py:264(stop)\n", - " 1 0.000 0.000 0.000 0.000 shape_base.py:1167(_tile_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:2965(_prod_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 base.py:4786(values)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 1 0.000 0.000 0.000 0.000 managers.py:282()\n", - " 1 0.000 0.000 0.000 0.000 {method 'clear_mapping' of 'pandas._libs.index.IndexEngine' objects}\n", - " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_bool}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%prun -s cumulative run_previous()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35a9df01-2b32-41c3-b9e2-259b21f214c6", - "metadata": {}, - "outputs": [], - "source": [ - "37642 function calls (37233 primitive calls) in 3.187 seconds\n", - "406159 function calls (406079 primitive calls) in 60.697 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 60.697 60.697 {built-in method builtins.exec}\n", - " 1 0.000 0.000 60.697 60.697 :1()\n", - " 1 0.033 0.033 60.697 60.697 2292568830.py:1(run_fru)\n", - " 1 37.147 37.147 60.664 60.664 3636660809.py:1(make_sample_choices_dev)\n", - " 30 0.745 0.025 22.155 0.738 random.py:562(random_for_df)\n", - " 30 0.192 0.006 13.381 0.446 random.py:598()\n", - " 404130 13.189 0.000 13.189 0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n", - " 32 8.025 0.251 8.025 0.251 {built-in method numpy.asanyarray}\n", - " 70/66 0.128 0.002 1.361 0.021 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", - " 32 0.000 0.000 1.232 0.039 fromnumeric.py:51(_wrapfunc)\n", - " 30 0.000 0.000 1.104 0.037 <__array_function__ internals>:177(argmax)\n", - " 30 0.000 0.000 1.103 0.037 fromnumeric.py:1127(argmax)\n", - " 30 1.103 0.037 1.103 0.037 {method 'argmax' of 'numpy.ndarray' objects}\n", - " \n", - "w/o random calls (test where we simply take max(obs_utlils) but everything else is identical)\n", - " 1039 function calls (1019 primitive calls) in 1.403 seconds\n", - " Ordered by: cumulative time\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 1.403 1.403 {built-in method builtins.exec}\n", - " 1 0.000 0.000 1.403 1.403 :1()\n", - " 1 0.000 0.000 1.403 1.403 2292568830.py:1(run_fru)\n", - " 1 0.021 0.021 1.403 1.403 691721383.py:1(make_sample_choices_dev)\n", - " 40/36 0.120 0.003 1.380 0.038 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", - " 32 0.000 0.000 1.260 0.039 fromnumeric.py:51(_wrapfunc)\n", - " 30 0.000 0.000 1.134 0.038 <__array_function__ internals>:177(argmax)\n", - " 30 0.000 0.000 1.133 0.038 fromnumeric.py:1127(argmax)\n", - " 30 1.133 0.038 1.133 0.038 {method 'argmax' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.127 0.127 <__array_function__ internals>:177(nonzero)\n", - " 1 0.000 0.000 0.127 0.127 fromnumeric.py:1866(nonzero)\n", - " 1 0.127 0.127 0.127 0.127 {method 'nonzero' of 'numpy.ndarray' objects}" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "082ecf8f-bf23-4a61-be4f-a0d914469f18", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " " - ] - }, - { - "data": { - "text/plain": [ - " 406159 function calls (406079 primitive calls) in 61.192 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 61.192 61.192 {built-in method builtins.exec}\n", - " 1 0.000 0.000 61.192 61.192 :1()\n", - " 1 0.027 0.027 61.192 61.192 2292568830.py:1(run_fru)\n", - " 1 37.470 37.470 61.165 61.165 1111942506.py:1(make_sample_choices_dev)\n", - " 30 0.778 0.026 22.305 0.744 random.py:562(random_for_df)\n", - " 30 0.195 0.006 13.419 0.447 random.py:598()\n", - " 404130 13.225 0.000 13.225 0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n", - " 32 8.104 0.253 8.104 0.253 {built-in method numpy.asanyarray}\n", - " 70/66 0.122 0.002 1.388 0.021 {built-in method numpy.core._multiarray_umath.implement_array_function}\n", - " 32 0.000 0.000 1.265 0.040 fromnumeric.py:51(_wrapfunc)\n", - " 30 0.000 0.000 1.129 0.038 <__array_function__ internals>:177(argmax)\n", - " 30 0.000 0.000 1.128 0.038 fromnumeric.py:1127(argmax)\n", - " 30 1.128 0.038 1.128 0.038 {method 'argmax' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.137 0.137 <__array_function__ internals>:177(nonzero)\n", - " 1 0.000 0.000 0.137 0.137 fromnumeric.py:1866(nonzero)\n", - " 1 0.137 0.137 0.137 0.137 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.122 0.122 <__array_function__ internals>:177(zeros_like)\n", - " 1 0.000 0.000 0.122 0.122 numeric.py:76(zeros_like)\n", - " 2 0.000 0.000 0.122 0.061 <__array_function__ internals>:177(copyto)\n", - " 30 0.000 0.000 0.003 0.000 contextlib.py:72(inner)\n", - " 30 0.002 0.000 0.002 0.000 {function SeedSequence.generate_state at 0x7fb5e410ac10}\n", - " 1 0.000 0.000 0.001 0.001 frame.py:587(__init__)\n", - " 1 0.000 0.000 0.001 0.001 construction.py:425(dict_to_mgr)\n", - " 1 0.000 0.000 0.001 0.001 construction.py:102(arrays_to_mgr)\n", - " 60 0.000 0.000 0.001 0.000 _ufunc_config.py:32(seterr)\n", - " 30 0.000 0.000 0.000 0.000 random.py:721(getrandbits)\n", - " 30 0.000 0.000 0.000 0.000 _ufunc_config.py:429(__enter__)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:1937(create_block_manager_from_column_arrays)\n", - " 158/83 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", - " 32 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(concatenate)\n", - " 30 0.000 0.000 0.000 0.000 {built-in method posix.urandom}\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:397(__new__)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:1683(_consolidate_inplace)\n", - " 32 0.000 0.000 0.000 0.000 frame.py:1413(__len__)\n", - " 30 0.000 0.000 0.000 0.000 _ufunc_config.py:434(__exit__)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:2074(_consolidate)\n", - " 60 0.000 0.000 0.000 0.000 _ufunc_config.py:131(geterr)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:2008(_form_blocks)\n", - " 190 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", - " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(empty_like)\n", - " 5 0.000 0.000 0.000 0.000 construction.py:470(sanitize_array)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:596(_homogenize)\n", - " 38 0.000 0.000 0.000 0.000 abc.py:96(__instancecheck__)\n", - " 30 0.000 0.000 0.000 0.000 pipeline.py:140(get_rn_generator)\n", - " 3 0.000 0.000 0.000 0.000 managers.py:2091(_merge_blocks)\n", - " 60 0.000 0.000 0.000 0.000 {built-in method numpy.seterrobj}\n", - " 2 0.000 0.000 0.000 0.000 frame.py:1693(to_numpy)\n", - " 36 0.000 0.000 0.000 0.000 base.py:884(__len__)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method builtins.sorted}\n", - " 8 0.000 0.000 0.000 0.000 managers.py:2079()\n", - " 38 0.000 0.000 0.000 0.000 {built-in method _abc._abc_instancecheck}\n", - " 120 0.000 0.000 0.000 0.000 {built-in method numpy.geterrobj}\n", - " 1 0.000 0.000 0.000 0.000 base.py:7106(_maybe_cast_data_without_dtype)\n", - " 4 0.000 0.000 0.000 0.000 blocks.py:166(_consolidate_key)\n", - " 88 0.000 0.000 0.000 0.000 {built-in method builtins.getattr}\n", - " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.maybe_convert_objects}\n", - " 4 0.000 0.000 0.000 0.000 managers.py:2060(_stack_arrays)\n", - " 3 0.000 0.000 0.000 0.000 generic.py:5646(_consolidate_inplace)\n", - " 4 0.000 0.000 0.000 0.000 _dtype.py:328(_name_get)\n", - " 3 0.000 0.000 0.000 0.000 generic.py:5632(_protect_consolidate)\n", - " 41 0.000 0.000 0.000 0.000 generic.py:43(_check)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:635(_extract_index)\n", - " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(vstack)\n", - " 2 0.000 0.000 0.000 0.000 chunk.py:599(log_df)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:487()\n", - " 2 0.000 0.000 0.000 0.000 chunk.py:140(chunk_training_mode)\n", - " 1 0.000 0.000 0.000 0.000 generic.py:2063(__array__)\n", - " 4 0.000 0.000 0.000 0.000 common.py:348(is_datetime64tz_dtype)\n", - " 2 0.000 0.000 0.000 0.000 frame.py:821(shape)\n", - " 1 0.000 0.000 0.000 0.000 shape_base.py:222(vstack)\n", - " 5 0.000 0.000 0.000 0.000 construction.py:379(extract_array)\n", - " 2 0.000 0.000 0.000 0.000 managers.py:1541(as_array)\n", - " 4 0.000 0.000 0.000 0.000 _asarray.py:22(require)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:483()\n", - " 2 0.000 0.000 0.000 0.000 common.py:229(asarray_tuplesafe)\n", - " 1 0.000 0.000 0.000 0.000 frame.py:898(_values)\n", - " 5 0.000 0.000 0.000 0.000 construction.py:695(_try_cast)\n", - " 4 0.000 0.000 0.000 0.000 _dtype.py:314(_name_includes_bit_suffix)\n", - " 2 0.000 0.000 0.000 0.000 config.py:108(setting)\n", - " 4 0.000 0.000 0.000 0.000 base.py:286(is_dtype)\n", - " 4 0.000 0.000 0.000 0.000 common.py:287(maybe_iterable_to_list)\n", - " 1 0.000 0.000 0.000 0.000 config.py:255(__call__)\n", - " 3 0.000 0.000 0.000 0.000 generic.py:5650(f)\n", - " 1 0.000 0.000 0.000 0.000 config.py:127(_get_option)\n", - " 30 0.000 0.000 0.000 0.000 {built-in method from_bytes}\n", - " 30 0.000 0.000 0.000 0.000 fromnumeric.py:1123(_argmax_dispatcher)\n", - " 30 0.000 0.000 0.000 0.000 pipeline.py:72(rng)\n", - " 4 0.000 0.000 0.000 0.000 managers.py:1665(is_consolidated)\n", - " 4 0.000 0.000 0.000 0.000 numerictypes.py:356(issubdtype)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method numpy.arange}\n", - " 1 0.000 0.000 0.000 0.000 {method '_rebuild_blknos_and_blklocs' of 'pandas._libs.internals.BlockManager' objects}\n", - " 1 0.000 0.000 0.000 0.000 base.py:554(_dtype_to_subclass)\n", - " 5 0.000 0.000 0.000 0.000 blocks.py:1989(get_block_type)\n", - " 32 0.000 0.000 0.000 0.000 multiarray.py:148(concatenate)\n", - " 1 0.000 0.000 0.000 0.000 numeric.py:289(full)\n", - " 2 0.000 0.000 0.000 0.000 inject.py:121(get_injectable)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:1673(_consolidate_check)\n", - " 2 0.000 0.000 0.000 0.000 base.py:7082(maybe_extract_name)\n", - " 5 0.000 0.000 0.000 0.000 common.py:1587(_is_dtype_type)\n", - " 1 0.000 0.000 0.000 0.000 config.py:109(_get_single_key)\n", - " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(argsort)\n", - " 4 0.000 0.000 0.000 0.000 common.py:552(require_length_match)\n", - " 1 0.000 0.000 0.000 0.000 base.py:654(_simple_new)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method numpy.empty}\n", - " 5 0.000 0.000 0.000 0.000 construction.py:627(_sanitize_ndim)\n", - " 2 0.000 0.000 0.000 0.000 common.py:1240(is_float_dtype)\n", - " 7 0.000 0.000 0.000 0.000 range.py:909(__len__)\n", - " 32 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass}\n", - " 1 0.000 0.000 0.000 0.000 cast.py:1960(construct_1d_object_array_from_listlike)\n", - " 30 0.000 0.000 0.000 0.000 contextlib.py:59(_recreate_cm)\n", - " 4 0.000 0.000 0.000 0.000 managers.py:1992(_grouping_func)\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:2030(new_block_2d)\n", - " 8 0.000 0.000 0.000 0.000 numerictypes.py:282(issubclass_)\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:1012(argsort)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:1679()\n", - " 1 0.000 0.000 0.000 0.000 base.py:53(shape)\n", - " 1 0.000 0.000 0.000 0.000 <__array_function__ internals>:177(atleast_2d)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method numpy.array}\n", - " 2 0.000 0.000 0.000 0.000 orca.py:1610(get_injectable)\n", - " 3 0.000 0.000 0.000 0.000 managers.py:618(consolidate)\n", - " 3 0.000 0.000 0.000 0.000 blocks.py:2119(extend_blocks)\n", - " 1 0.000 0.000 0.000 0.000 api.py:322(default_index)\n", - " 1 0.000 0.000 0.000 0.000 generic.py:239(__init__)\n", - " 4 0.000 0.000 0.000 0.000 common.py:1483(is_ea_or_datetimelike_dtype)\n", - " 4 0.000 0.000 0.000 0.000 _asarray.py:111()\n", - " 3 0.000 0.000 0.000 0.000 generic.py:5577(__setattr__)\n", - " 3 0.000 0.000 0.000 0.000 base.py:55()\n", - " 1 0.000 0.000 0.000 0.000 construction.py:494()\n", - " 2 0.000 0.000 0.000 0.000 :1017(_handle_fromlist)\n", - " 2 0.000 0.000 0.000 0.000 blocks.py:222(get_values)\n", - " 5 0.000 0.000 0.000 0.000 construction.py:664(_sanitize_str_dtypes)\n", - " 5 0.000 0.000 0.000 0.000 {built-in method numpy.asarray}\n", - " 17 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}\n", - " 1 0.000 0.000 0.000 0.000 config.py:589(_get_root)\n", - " 1 0.000 0.000 0.000 0.000 common.py:786(is_unsigned_integer_dtype)\n", - " 3 0.000 0.000 0.000 0.000 construction.py:438(ensure_wrapped_if_datetimelike)\n", - " 1 0.000 0.000 0.000 0.000 blocks.py:1960(maybe_coerce_values)\n", - " 1 0.000 0.000 0.000 0.000 shape_base.py:81(atleast_2d)\n", - " 1 0.000 0.000 0.000 0.000 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 range.py:167(_simple_new)\n", - " 5 0.000 0.000 0.000 0.000 construction.py:684(_maybe_repeat)\n", - " 14 0.000 0.000 0.000 0.000 typing.py:1149(cast)\n", - " 1 0.000 0.000 0.000 0.000 common.py:161(is_object_dtype)\n", - " 1 0.000 0.000 0.000 0.000 config.py:642(_warn_if_deprecated)\n", - " 1 0.000 0.000 0.000 0.000 common.py:732(is_signed_integer_dtype)\n", - " 3 0.000 0.000 0.000 0.000 common.py:147()\n", - " 10 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_list_like}\n", - " 4 0.000 0.000 0.000 0.000 common.py:1416(is_1d_only_ea_dtype)\n", - " 2 0.000 0.000 0.000 0.000 inject.py:116(is_injectable)\n", - " 9 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 2 0.000 0.000 0.000 0.000 inference.py:321(is_hashable)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method numpy.zeros}\n", - " 1 0.000 0.000 0.000 0.000 managers.py:2102()\n", - " 1 0.000 0.000 0.000 0.000 construction.py:486()\n", - " 2 0.000 0.000 0.000 0.000 generic.py:5561(__getattr__)\n", - " 2 0.000 0.000 0.000 0.000 config.py:603(_get_deprecated_option)\n", - " 4 0.000 0.000 0.000 0.000 {method 'format' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 orca.py:1568(get_raw_injectable)\n", - " 1 0.000 0.000 0.000 0.000 shape_base.py:218(_vhstack_dispatcher)\n", - " 4 0.000 0.000 0.000 0.000 cast.py:1711(sanitize_to_nanoseconds)\n", - " 2 0.000 0.000 0.000 0.000 {method 'transpose' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 managers.py:217(is_single_block)\n", - " 1 0.000 0.000 0.000 0.000 common.py:1747(pandas_dtype)\n", - " 2 0.000 0.000 0.000 0.000 base.py:6987(ensure_index)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 4 0.000 0.000 0.000 0.000 orca.py:1560(is_injectable)\n", - " 1 0.000 0.000 0.000 0.000 flags.py:47(__init__)\n", - " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_scalar}\n", - " 2 0.000 0.000 0.000 0.000 common.py:155()\n", - " 1 0.000 0.000 0.000 0.000 config.py:571(_select_options)\n", - " 4 0.000 0.000 0.000 0.000 blocks.py:354(dtype)\n", - " 4 0.000 0.000 0.000 0.000 _dtype.py:24(_kind_name)\n", - " 4 0.000 0.000 0.000 0.000 {method 'upper' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 base.py:4786(values)\n", - " 1 0.000 0.000 0.000 0.000 construction.py:233(mgr_to_mgr)\n", - " 2 0.000 0.000 0.000 0.000 base.py:518()\n", - " 2 0.000 0.000 0.000 0.000 base.py:834(_reset_identity)\n", - " 1 0.000 0.000 0.000 0.000 config.py:630(_translate_key)\n", - " 1 0.000 0.000 0.000 0.000 shape_base.py:207(_arrays_for_stack_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 base.py:1650(name)\n", - " 2 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 3 0.000 0.000 0.000 0.000 common.py:145(classes)\n", - " 2 0.000 0.000 0.000 0.000 common.py:150(classes_and_not_datetimelike)\n", - " 2 0.000 0.000 0.000 0.000 blocks.py:244(mgr_locs)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method __new__ of type object at 0x908780}\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.hash}\n", - " 2 0.000 0.000 0.000 0.000 multiarray.py:1071(copyto)\n", - " 1 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'setdefault' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:1862(_nonzero_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 {method 'astype' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 {pandas._libs.lib.is_iterator}\n", - " 1 0.000 0.000 0.000 0.000 base.py:540(_ensure_array)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'keys' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 managers.py:2111()\n", - " 1 0.000 0.000 0.000 0.000 multiarray.py:80(empty_like)\n", - " 1 0.000 0.000 0.000 0.000 managers.py:919(__init__)\n", - " 1 0.000 0.000 0.000 0.000 base.py:229(disallow_kwargs)\n", - " 1 0.000 0.000 0.000 0.000 numeric.py:72(_zeros_like_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 shape_base.py:77(_atleast_2d_dispatcher)\n", - " 1 0.000 0.000 0.000 0.000 fromnumeric.py:1008(_argsort_dispatcher)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%prun -s cumulative run_fru()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "66c833e5-2ad3-4052-ae11-2aa887ad9a4f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e0f3f75-1150-4712-bfb4-435e85af8b52", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1fc21256-564e-4842-82fb-4ba5358fb644", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ab332607-5f2e-4c7f-bfe4-5407c32644fb", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/frozen_rand_mode_choice_dev.ipynb b/notebooks/frozen_rand_mode_choice_dev.ipynb deleted file mode 100644 index 101953e516..0000000000 --- a/notebooks/frozen_rand_mode_choice_dev.ipynb +++ /dev/null @@ -1,3418 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", - "metadata": {}, - "source": [ - "# Start work here, clean up as you go\n", - "\n", - "Problem below: scale of error term on lower levels needs to be given by nest, not the case atm\n", - "\n", - "Do I remember this correctly and probabilities are calculated as products of marginal and conditional probabilities?\n", - "if so, the corresponding utilities at leaf and node levels would need to be calculated, and I would be able to use\n", - "these directly, right? CHECK, would make it much easier!" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.385153Z", - "iopub.status.busy": "2022-05-01T10:17:57.384881Z", - "iopub.status.idle": "2022-05-01T10:17:57.534433Z", - "shell.execute_reply": "2022-05-01T10:17:57.533096Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%load_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "c3dba451-1e10-403e-8614-35d57e6577f4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.536623Z", - "iopub.status.busy": "2022-05-01T10:17:57.536012Z", - "iopub.status.idle": "2022-05-01T10:17:57.542755Z", - "shell.execute_reply": "2022-05-01T10:17:57.541685Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:00.350944Z", - "iopub.status.busy": "2022-05-01T10:18:00.350730Z", - "iopub.status.idle": "2022-05-01T10:18:12.760977Z", - "shell.execute_reply": "2022-05-01T10:18:12.760013Z", - "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import argparse\n", - "from datetime import datetime\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from numpy.random import default_rng\n", - "\n", - "\n", - "from activitysim.cli import run\n", - "from activitysim.core import inject\n", - "from activitysim.core import tracing\n", - "from activitysim.core import config\n", - "from activitysim.core import pipeline\n", - "from activitysim.core import mem\n", - "from activitysim.core import chunk\n", - "from activitysim.core import simulate\n", - "from activitysim.core import logit\n", - "from activitysim.abm.models.util.mode import mode_choice_simulate\n", - "from activitysim.abm.models.util import estimation\n", - "from activitysim.core import expressions\n", - "from activitysim.core.util import assign_in_place" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:16.944537Z", - "iopub.status.busy": "2022-05-01T10:18:16.944291Z", - "iopub.status.idle": "2022-05-01T10:18:17.124764Z", - "shell.execute_reply": "2022-05-01T10:18:17.123725Z", - "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "pd.set_option(\"max_columns\", 500)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:17.839947Z", - "iopub.status.busy": "2022-05-01T10:18:17.839070Z", - "iopub.status.idle": "2022-05-01T10:18:18.019676Z", - "shell.execute_reply": "2022-05-01T10:18:18.018689Z", - "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_example_mtc\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:18.894533Z", - "iopub.status.busy": "2022-05-01T10:18:18.894303Z", - "iopub.status.idle": "2022-05-01T10:18:19.078807Z", - "shell.execute_reply": "2022-05-01T10:18:19.077951Z", - "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "os.chdir(example_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:22.111723Z", - "iopub.status.busy": "2022-05-01T10:18:22.111490Z", - "iopub.status.idle": "2022-05-01T10:18:22.297437Z", - "shell.execute_reply": "2022-05-01T10:18:22.296501Z", - "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "parser = argparse.ArgumentParser()\n", - "run.add_run_args(parser)\n", - "# args = parser.parse_args()\n", - "# parser.parse_args(['--sum', '7', '-1', '42'])\n", - "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", - "#run.run(args) # 2mins full example run\n", - "\n", - "\n", - "if not inject.is_injectable('preload_injectables'):\n", - " from activitysim import abm # register abm steps and other abm-specific injectables\n", - "run.handle_standard_args(args) # possibly update injectables" - ] - }, - { - "cell_type": "markdown", - "id": "6045438f-d1eb-4b5c-9737-185798b7f97f", - "metadata": {}, - "source": [ - "## trip mode choice by hand" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "600ae244-3e6c-4b66-8d39-aa9f6f60b378", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:26.399145Z", - "iopub.status.busy": "2022-05-01T10:18:26.398687Z", - "iopub.status.idle": "2022-05-01T10:18:26.583256Z", - "shell.execute_reply": "2022-05-01T10:18:26.582233Z", - "shell.execute_reply.started": "2022-05-01T10:18:26.399105Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#inject.get_table('trips').to_frame() #pipeline.orca.get_raw_table('trips').to_frame()\n", - "#inject.get_table('tours_merged').to_frame() #pipeline.orca.get_raw_table('tours_merged').to_frame()\n", - "#inject.get_injectable('network_los')" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "outputs": [], - "source": [ - "def run_trip_mode_choice(do_these_purposes=None, simulate_function=simulate.simple_simulate):\n", - " #do_these_purposes=['escort']\n", - " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", - " resume_after = \"trip_scheduling\"\n", - " model_name = \"trip_mode_choice\"\n", - " chunk_size = 0 # test_mtc means no chunking\n", - "\n", - " pipeline.open_pipeline(resume_after)\n", - " # preload any bulky injectables (e.g. skims) not in pipeline\n", - " inject.get_injectable('preload_injectables', None)\n", - " pipeline._PIPELINE.rng().begin_step(model_name)\n", - " step_name = model_name\n", - " args = {}\n", - " checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - " inject.set_step_args(args)\n", - "\n", - " trips = inject.get_table('trips')\n", - " tours_merged = inject.get_table('tours_merged')\n", - " network_los = inject.get_injectable('network_los')\n", - "\n", - " trace_label = 'trip_mode_choice'\n", - " model_settings_file_name = 'trip_mode_choice.yaml'\n", - " model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - " mode_column_name = 'trip_mode'\n", - "\n", - " trips_df = trips.to_frame()\n", - " print(\"Running with %d trips\", trips_df.shape[0])\n", - "\n", - " tours_merged = tours_merged.to_frame()\n", - " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "\n", - " # - trips_merged - merge trips and tours_merged\n", - " trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - " assert trips_merged.index.equals(trips.index)\n", - "\n", - " # setup skim keys\n", - " assert ('trip_period' not in trips_merged)\n", - " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - " orig_col = 'origin'\n", - " dest_col = 'destination'\n", - "\n", - " constants = {}\n", - " constants.update(config.get_model_constants(model_settings))\n", - " constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - " })\n", - "\n", - " skim_dict = network_los.get_default_skim_dict()\n", - "\n", - " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - " skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - " }\n", - "\n", - " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - " nest_specs = config.get_logit_model_settings(model_settings)\n", - "\n", - " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - "\n", - " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", - " continue\n", - "\n", - " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " (primary_purpose, len(trips_segment.index), ))\n", - "\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " ################ Replace wrapper function\n", - " # choices = mode_choice_simulate(...)\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", - " choices = simulate_function(\n", - " choosers=trips_segment,\n", - " spec=spec,\n", - " nest_spec=nest_spec,\n", - " skims=skims,\n", - " locals_d=locals_dict,\n", - " chunk_size=chunk_size,\n", - " want_logsums=logsum_column_name is not None,\n", - " trace_label=segment_trace_label,\n", - " trace_choice_name='trip_mode_choice',\n", - " estimator=estimator,\n", - " trace_column_names=None)\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " alts = spec.columns\n", - " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", - " ################\n", - " choices_list.append(choices)\n", - " choices_df_asim = pd.concat(choices_list)\n", - "\n", - " # update trips table with choices (and potionally logssums)\n", - " trips_df = trips.to_frame()\n", - "\n", - " if (do_these_purposes is not None):\n", - " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", - "\n", - " assign_in_place(trips_df, choices_df_asim)\n", - " assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - " finalise = True\n", - " if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - " print(\"Done\")\n", - "\n", - " return trips_df" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:28:53.643404Z", - "iopub.status.busy": "2021-09-01T02:28:53.643055Z", - "iopub.status.idle": "2021-09-01T02:29:01.223951Z", - "shell.execute_reply": "2021-09-01T02:29:01.222865Z", - "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'atwork' (27 trips)\n", - "trip_mode_choice tour_type 'eatout' (33 trips)\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", - "trip_mode_choice tour_type 'othmaint' (46 trips)\n", - "trip_mode_choice tour_type 'school' (37 trips)\n", - "trip_mode_choice tour_type 'shopping' (77 trips)\n", - "trip_mode_choice tour_type 'social' (19 trips)\n", - "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n" - ] - } - ], - "source": [ - "trips_df = run_trip_mode_choice()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "7eae2854-38bd-4da0-a161-3a15bbd17177", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:30:57.562848Z", - "iopub.status.busy": "2021-08-31T10:30:57.562615Z", - "iopub.status.idle": "2021-08-31T10:30:57.806655Z", - "shell.execute_reply": "2021-08-31T10:30:57.805135Z", - "shell.execute_reply.started": "2021-08-31T10:30:57.562824Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n8684833 26478 26478 eatout 1 True \n8684837 26478 26478 eatout 1 False \n8685009 26478 26478 othmaint 1 True \n8685013 26478 26478 othmaint 1 False \n8753057 26686 26686 eatout 1 True \n... ... ... ... ... ... \n2472945113 7539466 2848131 shopping 1 True \n2472945117 7539466 2848131 shopping 1 False \n2472945118 7539466 2848131 shopping 2 False \n2473024473 7539708 2848373 univ 1 True \n2473024477 7539708 2848373 univ 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n8684833 1 13 8 1085604 eatout \n8684837 1 8 13 1085604 home \n8685009 1 10 8 1085626 othmaint \n8685013 1 8 10 1085626 home \n8753057 1 5 8 1094132 eatout \n... ... ... ... ... ... \n2472945113 1 8 3 309118139 shopping \n2472945117 2 25 8 309118139 shopping \n2472945118 2 3 25 309118139 home \n2473024473 1 13 18 309128059 univ \n2473024477 1 18 13 309128059 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n8684833 NaN 11.0 WALK -1.171760 \n8684837 NaN 11.0 WALK -1.238719 \n8685009 NaN 12.0 BIKE 6.198626 \n8685013 NaN 13.0 BIKE 6.175681 \n8753057 NaN 19.0 WALK 4.457539 \n... ... ... ... ... \n2472945113 NaN 18.0 WALK_LOC 12.537675 \n2472945117 56.842247 21.0 WALK_LOC 11.880804 \n2472945118 NaN 22.0 WALK 13.710030 \n2473024473 NaN 16.0 WALK_LOC -0.530696 \n2473024477 NaN 23.0 WALK_LRF 0.624304 \n\n[482 rows x 14 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
86848332647826478eatout1True11381085604eatoutNaN11.0WALK-1.171760
86848372647826478eatout1False18131085604homeNaN11.0WALK-1.238719
86850092647826478othmaint1True11081085626othmaintNaN12.0BIKE6.198626
86850132647826478othmaint1False18101085626homeNaN13.0BIKE6.175681
87530572668626686eatout1True1581094132eatoutNaN19.0WALK4.457539
.............................................
247294511375394662848131shopping1True183309118139shoppingNaN18.0WALK_LOC12.537675
247294511775394662848131shopping1False2258309118139shopping56.84224721.0WALK_LOC11.880804
247294511875394662848131shopping2False2325309118139homeNaN22.0WALK13.710030
247302447375397082848373univ1True11318309128059univNaN16.0WALK_LOC-0.530696
247302447775397082848373univ1False11813309128059homeNaN23.0WALK_LRF0.624304
\n

482 rows × 14 columns

\n
" - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trips_df" - ] - }, - { - "cell_type": "markdown", - "id": "8eebea31-499c-41ca-8411-883a88ca800a", - "metadata": {}, - "source": [ - "## nested dev" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "3e2513e9-c03f-4891-9a1c-4053a4440a10", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:01.113060Z", - "iopub.status.busy": "2021-08-31T10:31:01.112829Z", - "iopub.status.idle": "2021-08-31T10:31:01.359912Z", - "shell.execute_reply": "2021-08-31T10:31:01.358948Z", - "shell.execute_reply.started": "2021-08-31T10:31:01.113034Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# see fct above - return if necessary\n", - "#spec = simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - "#nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "36c100b3-3e39-4950-a586-4d42be695eaa", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T10:31:01.775068Z", - "iopub.status.busy": "2021-08-31T10:31:01.774846Z", - "iopub.status.idle": "2021-08-31T10:31:01.975816Z", - "shell.execute_reply": "2021-08-31T10:31:01.974499Z", - "shell.execute_reply.started": "2021-08-31T10:31:01.775044Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#print(nest_spec)\n", - "#for nest in logit.each_nest(nest_spec):\n", - "# nest.print()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "e8d11f6a-f82a-40bd-8eef-fc28bcca8252", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:59:35.109079Z", - "iopub.status.busy": "2021-09-01T01:59:35.108781Z", - "iopub.status.idle": "2021-09-01T01:59:35.319301Z", - "shell.execute_reply": "2021-09-01T01:59:35.318402Z", - "shell.execute_reply.started": "2021-09-01T01:59:35.109050Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator,\n", - " log_alt_losers=False,\n", - " want_logsums=False, trace_label=None,\n", - " trace_choice_name=None, trace_column_names=None):\n", - "\n", - " trace_label = tracing.extend_trace_label(trace_label, 'eval_nl')\n", - " assert trace_label\n", - " have_trace_targets = tracing.has_trace_targets(choosers)\n", - "\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - "\n", - " if have_trace_targets:\n", - " tracing.trace_df(choosers, '%s.choosers' % trace_label)\n", - "\n", - " raw_utilities = simulate.eval_utilities(spec, choosers, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=have_trace_targets,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - " chunk.log_df(trace_label, \"raw_utilities\", raw_utilities)\n", - "\n", - " if have_trace_targets:\n", - " tracing.trace_df(raw_utilities, '%s.raw_utilities' % trace_label,\n", - " column_labels=['alternative', 'utility'])\n", - "\n", - " # exponentiated utilities of leaves and nests\n", - " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - " chunk.log_df(trace_label, \"nested_exp_utilities\", nested_exp_utilities)\n", - "\n", - " del raw_utilities\n", - " chunk.log_df(trace_label, 'raw_utilities', None)\n", - "\n", - " if have_trace_targets:\n", - " tracing.trace_df(nested_exp_utilities, '%s.nested_exp_utilities' % trace_label,\n", - " column_labels=['alternative', 'utility'])\n", - "\n", - " # probabilities of alternatives relative to siblings sharing the same nest\n", - " nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec,\n", - " trace_label=trace_label)\n", - " chunk.log_df(trace_label, \"nested_probabilities\", nested_probabilities)\n", - "\n", - " if want_logsums:\n", - " # logsum of nest root\n", - " logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index)\n", - " chunk.log_df(trace_label, \"logsums\", logsums)\n", - "\n", - " del nested_exp_utilities\n", - " chunk.log_df(trace_label, 'nested_exp_utilities', None)\n", - "\n", - " if have_trace_targets:\n", - " tracing.trace_df(nested_probabilities, '%s.nested_probabilities' % trace_label,\n", - " column_labels=['alternative', 'probability'])\n", - "\n", - " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", - " chunk.log_df(trace_label, \"base_probabilities\", base_probabilities)\n", - "\n", - " del nested_probabilities\n", - " chunk.log_df(trace_label, 'nested_probabilities', None)\n", - "\n", - " if have_trace_targets:\n", - " tracing.trace_df(base_probabilities, '%s.base_probabilities' % trace_label,\n", - " column_labels=['alternative', 'probability'])\n", - "\n", - " # note base_probabilities could all be zero since we allowed all probs for nests to be zero\n", - " # check here to print a clear message but make_choices will raise error if probs don't sum to 1\n", - " BAD_PROB_THRESHOLD = 0.001\n", - " no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD\n", - "\n", - " if no_choices.any():\n", - "\n", - " logit.report_bad_choices(\n", - " no_choices, base_probabilities,\n", - " trace_label=tracing.extend_trace_label(trace_label, 'bad_probs'),\n", - " trace_choosers=choosers,\n", - " msg=\"base_probabilities do not sum to one\")\n", - "\n", - " if custom_chooser:\n", - " choices, rands = custom_chooser(probs=base_probabilities, choosers=choosers, spec=spec,\n", - " trace_label=trace_label)\n", - " else:\n", - " choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)\n", - "\n", - " del base_probabilities\n", - " chunk.log_df(trace_label, 'base_probabilities', None)\n", - "\n", - " if have_trace_targets:\n", - " tracing.trace_df(choices, '%s.choices' % trace_label,\n", - " columns=[None, trace_choice_name])\n", - " tracing.trace_df(rands, '%s.rands' % trace_label,\n", - " columns=[None, 'rand'])\n", - " if want_logsums:\n", - " tracing.trace_df(logsums, '%s.logsums' % trace_label,\n", - " columns=[None, 'logsum'])\n", - "\n", - " if want_logsums:\n", - " choices = choices.to_frame('choice')\n", - " choices['logsum'] = logsums\n", - "\n", - " return choices\n", - "\n", - "\n", - "def simple_simulate_dev(choosers, spec, nest_spec,\n", - " skims=None, locals_d=None,\n", - " chunk_size=0, custom_chooser=None,\n", - " log_alt_losers=False,\n", - " want_logsums=False,\n", - " estimator=None,\n", - " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", - " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", - " assert len(choosers) > 0\n", - " result_list = []\n", - " # segment by person type and pick the right spec for each person type\n", - " for i, chooser_chunk, chunk_trace_label \\\n", - " in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", - " # the following replaces choices = _simple_simulate(...)\n", - " if skims is not None:\n", - " simulate.set_skim_wrapper_targets(choosers, skims)\n", - "\n", - " # only do this for nested, logit is straight forward\n", - " assert nest_spec is not None\n", - " choices = eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser,\n", - " log_alt_losers=log_alt_losers,\n", - " want_logsums=want_logsums,\n", - " estimator=estimator,\n", - " trace_label=trace_label,\n", - " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", - "\n", - "\n", - " result_list.append(choices)\n", - " chunk.log_df(trace_label, f'result_list', result_list)\n", - "\n", - " if len(result_list) > 1:\n", - " choices = pd.concat(result_list)\n", - " assert len(choices.index == len(choosers.index))\n", - " return choices\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "ec4eb1ff-2f35-4919-96b4-5acffa01f597", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "Done\n" - ] - }, - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n137248721 418441 304036 escort 1 True \n137248725 418441 304036 escort 1 False \n211388201 644476 386761 escort 1 True \n211388205 644476 386761 escort 1 False \n806388401 2458501 1173905 escort 1 True \n806388405 2458501 1173905 escort 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n137248721 1 7 10 17156090 escort \n137248725 1 10 7 17156090 home \n211388201 1 11 16 26423525 escort \n211388205 1 16 11 26423525 home \n806388401 1 16 8 100798550 escort \n806388405 1 8 16 100798550 home \n\n destination_logsum depart trip_mode mode_choice_logsum \ntrip_id \n137248721 NaN 7.0 WALK 11.435800 \n137248725 NaN 7.0 WALK 11.480440 \n211388201 NaN 5.0 WALK_LOC 4.789158 \n211388205 NaN 6.0 WALK_LOC 5.050171 \n806388401 NaN 15.0 WALK_LOC 6.451457 \n806388405 NaN 16.0 WALK_LOC 6.446188 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdeparttrip_modemode_choice_logsum
trip_id
137248721418441304036escort1True171017156090escortNaN7.0WALK11.435800
137248725418441304036escort1False110717156090homeNaN7.0WALK11.480440
211388201644476386761escort1True1111626423525escortNaN5.0WALK_LOC4.789158
211388205644476386761escort1False1161126423525homeNaN6.0WALK_LOC5.050171
80638840124585011173905escort1True1168100798550escortNaN15.0WALK_LOC6.451457
80638840524585011173905escort1False1816100798550homeNaN16.0WALK_LOC6.446188
\n
" - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "test_trips = run_trip_mode_choice(do_these_purposes=[\"escort\"], simulate_function=simple_simulate_dev)\n", - "test_trips" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Get raw utilities, etc" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 61, - "outputs": [], - "source": [ - "def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator,\n", - " log_alt_losers=False,\n", - " want_logsums=False, trace_label=None,\n", - " trace_choice_name=None, trace_column_names=None):\n", - "\n", - " trace_label = tracing.extend_trace_label(trace_label, 'eval_nl')\n", - " assert trace_label\n", - " have_trace_targets = tracing.has_trace_targets(choosers)\n", - "\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, choosers, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=have_trace_targets,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - " # exponentiated utilities of leaves and nests\n", - " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - " nested_utils = simulate.compute_nested_utilities(raw_utilities, nest_spec)\n", - " # probabilities of alternatives relative to siblings sharing the same nest\n", - " nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec,\n", - " trace_label=trace_label)\n", - " if want_logsums:\n", - " # logsum of nest root\n", - " logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index)\n", - " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec)\n", - " # note base_probabilities could all be zero since we allowed all probs for nests to be zero\n", - " # check here to print a clear message but make_choices will raise error if probs don't sum to 1\n", - " BAD_PROB_THRESHOLD = 0.001\n", - " no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD\n", - " if no_choices.any():\n", - " print(\"BAD\")\n", - " choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label)\n", - " if want_logsums:\n", - " choices = choices.to_frame('choice')\n", - " choices['logsum'] = logsums\n", - " return choices, raw_utilities, nested_exp_utilities, nested_utils\n", - "\n", - "\n", - "def simple_simulate_dev(choosers, spec, nest_spec,\n", - " skims=None, locals_d=None,\n", - " chunk_size=0, custom_chooser=None,\n", - " log_alt_losers=False,\n", - " want_logsums=False,\n", - " estimator=None,\n", - " trace_label=None, trace_choice_name=None, trace_column_names=None):\n", - " trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate')\n", - " assert len(choosers) > 0\n", - " result_list = []\n", - " # segment by person type and pick the right spec for each person type\n", - " for i, chooser_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label):\n", - " # the following replaces choices = _simple_simulate(...)\n", - " if skims is not None:\n", - " simulate.set_skim_wrapper_targets(choosers, skims)\n", - "\n", - " # only do this for nested, logit is straight forward\n", - " assert nest_spec is not None\n", - " choices, raw_utilities, nested_exp_utilities, nested_utils = eval_nl_dev(choosers, spec, nest_spec, locals_d,\n", - " custom_chooser,\n", - " log_alt_losers=log_alt_losers,\n", - " want_logsums=want_logsums,\n", - " estimator=estimator,\n", - " trace_label=trace_label,\n", - " trace_choice_name=trace_choice_name, trace_column_names=trace_column_names)\n", - "\n", - "\n", - " result_list.append(choices)\n", - " chunk.log_df(trace_label, f'result_list', result_list)\n", - "\n", - " if len(result_list) > 1:\n", - " choices = pd.concat(result_list)\n", - " assert len(choices.index == len(choosers.index))\n", - " return choices, raw_utilities, nested_exp_utilities, nested_utils\n", - "\n", - "\n", - "def get_stuff(do_these_purposes=None):\n", - " #do_these_purposes=['escort']\n", - " \"\"\"open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists\"\"\"\n", - " resume_after = \"trip_scheduling\"\n", - " model_name = \"trip_mode_choice\"\n", - " chunk_size = 0 # test_mtc means no chunking\n", - "\n", - " pipeline.open_pipeline(resume_after)\n", - " # preload any bulky injectables (e.g. skims) not in pipeline\n", - " inject.get_injectable('preload_injectables', None)\n", - " pipeline._PIPELINE.rng().begin_step(model_name)\n", - " step_name = model_name\n", - " args = {}\n", - " checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - " inject.set_step_args(args)\n", - "\n", - " trips = inject.get_table('trips')\n", - " tours_merged = inject.get_table('tours_merged')\n", - " network_los = inject.get_injectable('network_los')\n", - "\n", - " trace_label = 'trip_mode_choice'\n", - " model_settings_file_name = 'trip_mode_choice.yaml'\n", - " model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - " logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - " mode_column_name = 'trip_mode'\n", - "\n", - " trips_df = trips.to_frame()\n", - " print(\"Running with %d trips\", trips_df.shape[0])\n", - "\n", - " tours_merged = tours_merged.to_frame()\n", - " tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "\n", - " # - trips_merged - merge trips and tours_merged\n", - " trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - " assert trips_merged.index.equals(trips.index)\n", - "\n", - " # setup skim keys\n", - " assert ('trip_period' not in trips_merged)\n", - " trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - " orig_col = 'origin'\n", - " dest_col = 'destination'\n", - "\n", - " constants = {}\n", - " constants.update(config.get_model_constants(model_settings))\n", - " constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - " })\n", - "\n", - " skim_dict = network_los.get_default_skim_dict()\n", - "\n", - " odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - " dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - " od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - " skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - " }\n", - "\n", - " model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - " nest_specs = config.get_logit_model_settings(model_settings)\n", - "\n", - " estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - " choices_list = []\n", - " raw_util_list = []\n", - " nest_list = []\n", - " nu_list = []\n", - " nest_spec_list = []\n", - "\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - "\n", - " if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes):\n", - " continue\n", - "\n", - " print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " (primary_purpose, len(trips_segment.index), ))\n", - "\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " ################ Replace wrapper function\n", - " # choices = mode_choice_simulate(...)\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label)\n", - " choices, raw_utilities, nested_exp_utilities, nested_utils = simple_simulate_dev(\n", - " choosers=trips_segment,\n", - " spec=spec,\n", - " nest_spec=nest_spec,\n", - " skims=skims,\n", - " locals_d=locals_dict,\n", - " chunk_size=chunk_size,\n", - " want_logsums=logsum_column_name is not None,\n", - " trace_label=segment_trace_label,\n", - " trace_choice_name='trip_mode_choice',\n", - " estimator=estimator,\n", - " trace_column_names=None)\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " alts = spec.columns\n", - " choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts))))\n", - " ################\n", - " choices_list.append(choices)\n", - " raw_util_list.append(raw_utilities)\n", - " nest_list.append(nested_exp_utilities)\n", - " nu_list.append(nested_utils)\n", - " nest_spec_list.append(nest_spec)\n", - "\n", - " choices_df_asim = pd.concat(choices_list)\n", - "\n", - " # update trips table with choices (and potionally logssums)\n", - " trips_df = trips.to_frame()\n", - "\n", - " if (do_these_purposes is not None):\n", - " trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)]\n", - "\n", - " assign_in_place(trips_df, choices_df_asim)\n", - " assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - " finalise = True\n", - " if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - " print(\"Done\")\n", - "\n", - " return trips_df, raw_util_list, nest_list, nu_list, nest_spec_list" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 62, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "Done\n" - ] - } - ], - "source": [ - "t, ru, neu, nu, ns = get_stuff(do_these_purposes=[\"escort\"])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 63, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.294906 -2855.580620 -0.453217 -0.280520 \n137248725 -1.247739 -2855.533453 -0.436709 -0.233697 \n211388201 -2855.520162 -5709.805876 -inf -17.366498 \n211388205 -2856.810200 -5711.095914 -inf -18.093309 \n806388401 -2855.874866 -5710.160580 -inf -9.085232 \n806388405 -2858.281791 -5712.567505 -inf -10.451460 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.566235 -0.098182 -2855.573680 -5709.859395 -inf \n137248725 -2854.519411 -0.081794 -2855.526994 -5709.812708 -inf \n211388201 -2871.652212 -6.078274 -21.226117 -2875.511831 -7.429141 \n211388205 -2872.379023 -6.332658 -21.727638 -2876.013352 -7.604673 \n806388401 -2863.370946 -3.179831 -10.734064 -2865.019778 -3.756922 \n806388405 -2864.737174 -3.658011 -11.684013 -2865.969727 -4.089405 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.311848 15.883036 -1368.200102 11.435786 -1970.783011 \n137248725 0.323683 15.945036 -1368.169103 11.480426 -1970.850529 \n211388201 -4.210520 -4.216264 -1381.397292 -3.035710 9.572919 \n211388205 -4.381625 -4.216264 -1381.397292 -3.035710 10.096155 \n806388401 -1.968599 6.192499 -1378.480723 4.458599 12.609633 \n806388405 -2.273383 5.998749 -1378.519473 4.319099 12.638426 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.783011 -3968.783011 -3968.783011 -3968.783011 -inf \n137248725 -3968.850529 -3968.850529 -3968.850529 -3968.850529 -inf \n211388201 -1988.028653 -1988.028653 -1988.028653 -1988.028653 4.786459 \n211388205 -1987.903845 -1987.903845 -1987.903845 -1987.903845 5.048078 \n806388401 -1984.209027 -1983.750027 -1984.124827 -1984.171027 6.304817 \n806388405 -1984.311174 -1983.852174 -1984.226974 -1984.273174 6.319213 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.903118 -3973.401006 -3973.401006 -3974.669500 -3975.688637 \n137248725 -1971.203369 -3975.938495 -3975.938495 -3971.574832 -3978.057757 \n211388201 -1988.387109 -3990.994654 -3990.994654 -1994.178373 -1997.184706 \n211388205 -1988.132266 -3993.215462 -3993.215462 -1990.677481 -1997.375747 \n806388401 -1985.442854 -3989.293913 -3989.293913 -3990.843560 -3993.682148 \n806388405 -1985.451447 -3990.724818 -3990.724818 -3987.215444 -3994.211225 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.503675 -25.143041 -24.249471 -8.448473 \n137248725 -inf -inf -24.341318 -25.104291 -24.140831 -8.406870 \n211388201 -inf 3.446251 -15.861515 -3.864497 -5.068360 -1.296757 \n211388205 -inf 3.634616 -15.776424 -3.845077 -5.005482 -1.286094 \n806388401 -inf 4.539468 -19.143696 -19.495908 -16.361074 -5.853917 \n806388405 -inf 4.549833 -18.886576 -19.495133 -16.300395 -5.828540 \n\n root \ntrip_id \n137248721 11.435801 \n137248725 11.480441 \n211388201 3.456910 \n211388205 3.643469 \n806388401 5.193789 \n806388405 5.134877 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.294906-2855.580620-0.453217-0.280520-2854.566235-0.098182-2855.573680-5709.859395-inf0.31184815.883036-1368.20010211.435786-1970.783011-3968.783011-3968.783011-3968.783011-3968.783011-inf-1970.903118-3973.401006-3973.401006-3974.669500-3975.688637-inf-inf-24.503675-25.143041-24.249471-8.44847311.435801
137248725-1.247739-2855.533453-0.436709-0.233697-2854.519411-0.081794-2855.526994-5709.812708-inf0.32368315.945036-1368.16910311.480426-1970.850529-3968.850529-3968.850529-3968.850529-3968.850529-inf-1971.203369-3975.938495-3975.938495-3971.574832-3978.057757-inf-inf-24.341318-25.104291-24.140831-8.40687011.480441
211388201-2855.520162-5709.805876-inf-17.366498-2871.652212-6.078274-21.226117-2875.511831-7.429141-4.210520-4.216264-1381.397292-3.0357109.572919-1988.028653-1988.028653-1988.028653-1988.0286534.786459-1988.387109-3990.994654-3990.994654-1994.178373-1997.184706-inf3.446251-15.861515-3.864497-5.068360-1.2967573.456910
211388205-2856.810200-5711.095914-inf-18.093309-2872.379023-6.332658-21.727638-2876.013352-7.604673-4.381625-4.216264-1381.397292-3.03571010.096155-1987.903845-1987.903845-1987.903845-1987.9038455.048078-1988.132266-3993.215462-3993.215462-1990.677481-1997.375747-inf3.634616-15.776424-3.845077-5.005482-1.2860943.643469
806388401-2855.874866-5710.160580-inf-9.085232-2863.370946-3.179831-10.734064-2865.019778-3.756922-1.9685996.192499-1378.4807234.45859912.609633-1984.209027-1983.750027-1984.124827-1984.1710276.304817-1985.442854-3989.293913-3989.293913-3990.843560-3993.682148-inf4.539468-19.143696-19.495908-16.361074-5.8539175.193789
806388405-2858.281791-5712.567505-inf-10.451460-2864.737174-3.658011-11.684013-2865.969727-4.089405-2.2733835.998749-1378.5194734.31909912.638426-1984.311174-1983.852174-1984.226974-1984.2731746.319213-1985.451447-3990.724818-3990.724818-3987.215444-3994.211225-inf4.549833-18.886576-19.495133-16.300395-5.8285405.134877
\n
" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "display(nu[0])#, ns[0])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 64, - "outputs": [], - "source": [ - "# next: add error terms to alternatives and nodes - can be done with this right here by iterating over nests and\n", - "# adding at each level.\n", - "# will probably want to vectorise, but that's for later.\n", - "# add_random returns a random number per row - this seems like the right thing to use while iterating over nest nodes\n", - "# and leafs (which gives us single columns per alternative)\n", - "# for destination choice, we might need to rethink this pattern though, but we'll cross that bridge when we come to it\n", - "\n", - "def inverse_ev1_cdf(x, location=0.0, scale=1.0):\n", - " #quantile function of EV1\n", - " # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution where the scale is proportional to variance (not variance^{-1})\n", - " # this means nested scales are between 0 and 1\n", - " # x can be number or np array or pd df for vecops\n", - " return location - scale * np.log(-np.log(x))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 112, - "outputs": [], - "source": [ - "# fake random channel for prototyping as per Asim tests\n", - "from activitysim.core.random import Random\n", - "rng = Random()\n", - "\n", - "def add_ev1_random(df, nest_spec):\n", - " # TODO: generalise to logit for nest_spec==None by adding one rand (scale=1) to all columns\n", - " nest_utils_for_choice = df.copy() # we'll add random parts to this such that we can recursively choose from\n", - " for n in logit.each_nest(nest_spec):\n", - " if n.level == 1:\n", - " continue # skip the root level\n", - " # TODO: check parent nest level scale is what we want this is right\n", - " rands = inverse_ev1_cdf(rng.random_for_df(nest_utils_for_choice, n=1), scale=n.parent_nest_scale)\n", - " # this will be cleaner wtith xarrays\n", - " nest_utils_for_choice.loc[:,n.name] += rands[:,0]\n", - " return nest_utils_for_choice" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 115, - "outputs": [], - "source": [ - "utils_df = nu[0]\n", - "nest_spec = ns[0]\n", - "nest_utils_for_choice = add_ev1_random(utils_df, nest_spec)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 116, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.116115 -2855.401829 -0.085418 -0.101729 \n137248725 -0.865187 -2855.150902 0.350254 0.148855 \n211388201 -2855.281895 -5709.567609 -inf -17.128230 \n211388205 -2856.635577 -5710.921291 -inf -17.918686 \n806388401 -2855.821604 -5710.107318 -inf -9.031970 \n806388405 -2857.992149 -5712.277863 -inf -10.161818 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.387444 0.269616 -2855.394889 -5709.680604 -inf \n137248725 -2854.136859 0.705169 -2855.144442 -5709.430157 -inf \n211388201 -2871.413945 -5.588125 -20.987850 -2875.273564 -6.938992 \n211388205 -2872.204400 -5.973433 -21.553014 -2875.838729 -7.245448 \n806388401 -2863.317684 -3.070264 -10.680802 -2864.966516 -3.647355 \n806388405 -2864.447532 -3.062176 -11.394371 -2865.680085 -3.493570 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.822679 16.250835 -1367.832304 11.946618 -1970.527596 \n137248725 1.416687 16.732000 -1367.382139 12.573430 -1970.304027 \n211388201 -3.529757 -3.726114 -1380.907142 -2.354947 9.913300 \n211388205 -3.882701 -3.857039 -1381.038067 -2.536786 10.345617 \n806388401 -1.816422 6.302066 -1378.371156 4.610776 12.685721 \n806388405 -1.445835 6.594584 -1377.923639 5.146648 13.052200 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.527596 -3968.527596 -3968.527596 -3968.527596 -inf \n137248725 -3968.304027 -3968.304027 -3968.304027 -3968.304027 -inf \n211388201 -1987.688271 -1987.688271 -1987.688271 -1987.688271 5.276609 \n211388205 -1987.654383 -1987.654383 -1987.654383 -1987.654383 5.407303 \n806388401 -1984.132938 -1983.673938 -1984.048738 -1984.094938 6.414384 \n806388405 -1983.897400 -1983.438400 -1983.813200 -1983.859400 6.915048 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.647702 -3973.145591 -3973.145591 -3974.414085 -3975.433221 \n137248725 -1970.656867 -3975.391993 -3975.391993 -3971.028330 -3977.511255 \n211388201 -1988.046728 -3990.654273 -3990.654273 -1993.837992 -1996.844325 \n211388205 -1987.882804 -3992.966000 -3992.966000 -1990.428019 -1997.126285 \n806388401 -1985.366765 -3989.217824 -3989.217824 -3990.767471 -3993.606059 \n806388405 -1985.037673 -3990.311044 -3990.311044 -3986.801670 -3993.797451 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.319775 -24.959142 -24.065572 -7.937642 \n137248725 -inf -inf -23.947836 -24.710809 -23.747349 -7.313866 \n211388201 -inf 4.127014 -15.616441 -3.619423 -4.823285 -0.615994 \n211388205 -inf 4.133540 -15.596812 -3.665464 -4.825869 -0.787171 \n806388401 -inf 4.691645 -19.088912 -19.441124 -16.306290 -5.701741 \n806388405 -inf 5.377382 -18.588659 -19.197216 -16.002477 -5.000992 \n\n root \ntrip_id \n137248721 11.435801 \n137248725 11.480441 \n211388201 3.456910 \n211388205 3.643469 \n806388401 5.193789 \n806388405 5.134877 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILroot
trip_id
137248721-1.116115-2855.401829-0.085418-0.101729-2854.3874440.269616-2855.394889-5709.680604-inf0.82267916.250835-1367.83230411.946618-1970.527596-3968.527596-3968.527596-3968.527596-3968.527596-inf-1970.647702-3973.145591-3973.145591-3974.414085-3975.433221-inf-inf-24.319775-24.959142-24.065572-7.93764211.435801
137248725-0.865187-2855.1509020.3502540.148855-2854.1368590.705169-2855.144442-5709.430157-inf1.41668716.732000-1367.38213912.573430-1970.304027-3968.304027-3968.304027-3968.304027-3968.304027-inf-1970.656867-3975.391993-3975.391993-3971.028330-3977.511255-inf-inf-23.947836-24.710809-23.747349-7.31386611.480441
211388201-2855.281895-5709.567609-inf-17.128230-2871.413945-5.588125-20.987850-2875.273564-6.938992-3.529757-3.726114-1380.907142-2.3549479.913300-1987.688271-1987.688271-1987.688271-1987.6882715.276609-1988.046728-3990.654273-3990.654273-1993.837992-1996.844325-inf4.127014-15.616441-3.619423-4.823285-0.6159943.456910
211388205-2856.635577-5710.921291-inf-17.918686-2872.204400-5.973433-21.553014-2875.838729-7.245448-3.882701-3.857039-1381.038067-2.53678610.345617-1987.654383-1987.654383-1987.654383-1987.6543835.407303-1987.882804-3992.966000-3992.966000-1990.428019-1997.126285-inf4.133540-15.596812-3.665464-4.825869-0.7871713.643469
806388401-2855.821604-5710.107318-inf-9.031970-2863.317684-3.070264-10.680802-2864.966516-3.647355-1.8164226.302066-1378.3711564.61077612.685721-1984.132938-1983.673938-1984.048738-1984.0949386.414384-1985.366765-3989.217824-3989.217824-3990.767471-3993.606059-inf4.691645-19.088912-19.441124-16.306290-5.7017415.193789
806388405-2857.992149-5712.277863-inf-10.161818-2864.447532-3.062176-11.394371-2865.680085-3.493570-1.4458356.594584-1377.9236395.14664813.052200-1983.897400-1983.438400-1983.813200-1983.8594006.915048-1985.037673-3990.311044-3990.311044-3986.801670-3993.797451-inf5.377382-18.588659-19.197216-16.002477-5.0009925.134877
\n
" - }, - "execution_count": 116, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nest_utils_for_choice" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 106, - "outputs": [], - "source": [ - "all_alternatives = list(map(lambda x: x.name, filter(lambda x: x.is_leaf, logit.each_nest(nest_spec))))\n", - "def is_alternative(name):\n", - " return name in all_alternatives\n", - "\n", - "# NOW IN logit.group_nests_by_level(nest_spec)\n", - "# def group_nests_by_level(nest_spec):\n", - "# # group nests by level:\n", - "# depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", - "# #print(f\"Nesting depth is {depth}\")\n", - "# nest_levels = {x: [] for x in range(1, depth+1)}\n", - "# for n in logit.each_nest(nest_spec):\n", - "# nest_levels[n.level].append(n.name)\n", - "# assert len(nest_levels[1]) == 1\n", - "# # assert nest_levels[1][0] == 'root'\n", - "# return nest_levels\n", - "\n", - "nest_utils_for_choice[\"choice\"] = None\n", - "\n", - "for level, alts in logit.group_nests_by_level(nest_spec).items():\n", - " if level == 1:\n", - " continue\n", - " no_choices_made_yet = nest_utils_for_choice[\"choice\"].isnull()\n", - " choice_this_level = nest_utils_for_choice.loc[no_choices_made_yet][alts].idxmax(1)\n", - " nest_utils_for_choice.loc[no_choices_made_yet, \"choice\"] = \\\n", - " np.where(choice_this_level.apply(is_alternative), choice_this_level, None)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 107, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY DRIVEALONE SHARED2FREE \\\ntrip_id \n137248721 -1.116115 -2855.401829 -0.085418 -0.101729 \n137248725 -0.865187 -2855.150902 0.350254 0.148855 \n211388201 -2855.281895 -5709.567609 -inf -17.128230 \n211388205 -2856.635577 -5710.921291 -inf -17.918686 \n806388401 -2855.821604 -5710.107318 -inf -9.031970 \n806388405 -2857.992149 -5712.277863 -inf -10.161818 \n\n SHARED2PAY SHAREDRIDE2 SHARED3FREE SHARED3PAY SHAREDRIDE3 \\\ntrip_id \n137248721 -2854.387444 0.269616 -2855.394889 -5709.680604 -inf \n137248725 -2854.136859 0.705169 -2855.144442 -5709.430157 -inf \n211388201 -2871.413945 -5.588125 -20.987850 -2875.273564 -6.938992 \n211388205 -2872.204400 -5.973433 -21.553014 -2875.838729 -7.245448 \n806388401 -2863.317684 -3.070264 -10.680802 -2864.966516 -3.647355 \n806388405 -2864.447532 -3.062176 -11.394371 -2865.680085 -3.493570 \n\n AUTO WALK BIKE NONMOTORIZED WALK_LOC \\\ntrip_id \n137248721 0.822679 16.250835 -1367.832304 11.946618 -1970.527596 \n137248725 1.416687 16.732000 -1367.382139 12.573430 -1970.304027 \n211388201 -3.529757 -3.726114 -1380.907142 -2.354947 9.913300 \n211388205 -3.882701 -3.857039 -1381.038067 -2.536786 10.345617 \n806388401 -1.816422 6.302066 -1378.371156 4.610776 12.685721 \n806388405 -1.445835 6.594584 -1377.923639 5.146648 13.052200 \n\n WALK_LRF WALK_EXP WALK_HVY WALK_COM WALKACCESS \\\ntrip_id \n137248721 -3968.527596 -3968.527596 -3968.527596 -3968.527596 -inf \n137248725 -3968.304027 -3968.304027 -3968.304027 -3968.304027 -inf \n211388201 -1987.688271 -1987.688271 -1987.688271 -1987.688271 5.276609 \n211388205 -1987.654383 -1987.654383 -1987.654383 -1987.654383 5.407303 \n806388401 -1984.132938 -1983.673938 -1984.048738 -1984.094938 6.414384 \n806388405 -1983.897400 -1983.438400 -1983.813200 -1983.859400 6.915048 \n\n DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY DRIVE_COM \\\ntrip_id \n137248721 -1970.647702 -3973.145591 -3973.145591 -3974.414085 -3975.433221 \n137248725 -1970.656867 -3975.391993 -3975.391993 -3971.028330 -3977.511255 \n211388201 -1988.046728 -3990.654273 -3990.654273 -1993.837992 -1996.844325 \n211388205 -1987.882804 -3992.966000 -3992.966000 -1990.428019 -1997.126285 \n806388401 -1985.366765 -3989.217824 -3989.217824 -3990.767471 -3993.606059 \n806388405 -1985.037673 -3990.311044 -3990.311044 -3986.801670 -3993.797451 \n\n DRIVEACCESS TRANSIT TAXI TNC_SINGLE TNC_SHARED RIDEHAIL \\\ntrip_id \n137248721 -inf -inf -24.319775 -24.959142 -24.065572 -7.937642 \n137248725 -inf -inf -23.947836 -24.710809 -23.747349 -7.313866 \n211388201 -inf 4.127014 -15.616441 -3.619423 -4.823285 -0.615994 \n211388205 -inf 4.133540 -15.596812 -3.665464 -4.825869 -0.787171 \n806388401 -inf 4.691645 -19.088912 -19.441124 -16.306290 -5.701741 \n806388405 -inf 5.377382 -18.588659 -19.197216 -16.002477 -5.000992 \n\n root choice \ntrip_id \n137248721 11.435801 WALK \n137248725 11.480441 WALK \n211388201 3.456910 WALK_LOC \n211388205 3.643469 WALK_LOC \n806388401 5.193789 WALK_LOC \n806388405 5.134877 WALK_LOC ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYDRIVEALONESHARED2FREESHARED2PAYSHAREDRIDE2SHARED3FREESHARED3PAYSHAREDRIDE3AUTOWALKBIKENONMOTORIZEDWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMWALKACCESSDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMDRIVEACCESSTRANSITTAXITNC_SINGLETNC_SHAREDRIDEHAILrootchoice
trip_id
137248721-1.116115-2855.401829-0.085418-0.101729-2854.3874440.269616-2855.394889-5709.680604-inf0.82267916.250835-1367.83230411.946618-1970.527596-3968.527596-3968.527596-3968.527596-3968.527596-inf-1970.647702-3973.145591-3973.145591-3974.414085-3975.433221-inf-inf-24.319775-24.959142-24.065572-7.93764211.435801WALK
137248725-0.865187-2855.1509020.3502540.148855-2854.1368590.705169-2855.144442-5709.430157-inf1.41668716.732000-1367.38213912.573430-1970.304027-3968.304027-3968.304027-3968.304027-3968.304027-inf-1970.656867-3975.391993-3975.391993-3971.028330-3977.511255-inf-inf-23.947836-24.710809-23.747349-7.31386611.480441WALK
211388201-2855.281895-5709.567609-inf-17.128230-2871.413945-5.588125-20.987850-2875.273564-6.938992-3.529757-3.726114-1380.907142-2.3549479.913300-1987.688271-1987.688271-1987.688271-1987.6882715.276609-1988.046728-3990.654273-3990.654273-1993.837992-1996.844325-inf4.127014-15.616441-3.619423-4.823285-0.6159943.456910WALK_LOC
211388205-2856.635577-5710.921291-inf-17.918686-2872.204400-5.973433-21.553014-2875.838729-7.245448-3.882701-3.857039-1381.038067-2.53678610.345617-1987.654383-1987.654383-1987.654383-1987.6543835.407303-1987.882804-3992.966000-3992.966000-1990.428019-1997.126285-inf4.133540-15.596812-3.665464-4.825869-0.7871713.643469WALK_LOC
806388401-2855.821604-5710.107318-inf-9.031970-2863.317684-3.070264-10.680802-2864.966516-3.647355-1.8164226.302066-1378.3711564.61077612.685721-1984.132938-1983.673938-1984.048738-1984.0949386.414384-1985.366765-3989.217824-3989.217824-3990.767471-3993.606059-inf4.691645-19.088912-19.441124-16.306290-5.7017415.193789WALK_LOC
806388405-2857.992149-5712.277863-inf-10.161818-2864.447532-3.062176-11.394371-2865.680085-3.493570-1.4458356.594584-1377.9236395.14664813.052200-1983.897400-1983.438400-1983.813200-1983.8594006.915048-1985.037673-3990.311044-3990.311044-3986.801670-3993.797451-inf5.377382-18.588659-19.197216-16.002477-5.0009925.134877WALK_LOC
\n
" - }, - "execution_count": 107, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nest_utils_for_choice" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "nest_utils_for_choice = utils_df.copy()\n", - "\n", - "logit.make_choices_ru_frozen(nest_utils_for_choice, nest_spec, trace_label=\"blah\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "# OLD\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "id": "2f080150-c15a-4059-8a26-42a8c0072606", - "metadata": { - "execution": { - "iopub.execute_input": "2021-08-31T23:16:40.014024Z", - "iopub.status.busy": "2021-08-31T23:16:40.013770Z", - "iopub.status.idle": "2021-08-31T23:16:40.236326Z", - "shell.execute_reply": "2021-08-31T23:16:40.235553Z", - "shell.execute_reply.started": "2021-08-31T23:16:40.013973Z" - }, - "tags": [] - }, - "source": [ - "### make choice at each level" - ] - }, - { - "cell_type": "code", - "execution_count": 315, - "id": "2ed71b54-67e4-4087-957a-75f4fa184144", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:43.778081Z", - "iopub.status.busy": "2021-09-01T01:11:43.777757Z", - "iopub.status.idle": "2021-09-01T01:11:43.975332Z", - "shell.execute_reply": "2021-09-01T01:11:43.974588Z", - "shell.execute_reply.started": "2021-09-01T01:11:43.778055Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def get_alternatives(nests, name):\n", - " alts = list(filter(lambda x: x.name == name, nests))\n", - " assert len(alts) == 1, f\"{len(alts)} not one\"\n", - " alts = alts[0].alternatives\n", - " return alts\n", - "\n", - "def recursive_choice(row, columns, nest_levels, nests):\n", - " choices = row[columns].idxmax() #axis=1).values[0]\n", - " next_level_columns = get_alternatives(nests, choices)\n", - " #print(f\"{choices} leads to columns {next_level_columns}\")\n", - " if next_level_columns is None:\n", - " return choices \n", - " new_choice = recursive_choice(row, next_level_columns, nest_levels, nests)\n", - " return new_choice\n", - "\n", - "lower_bound = np.finfo(np.float64).eps # chance is very small but let's make it zero. could also check and replace if it ever happened\n", - "\n", - "def make_choice(utils_df, nests, nest_levels, seed=None):\n", - " rng = default_rng(seed=seed)\n", - " rands = rng.uniform(low=lower_bound, high=1.0, size=utils_df.shape[1])\n", - " probs_arr = utils_df - np.log(-np.log(rands))\n", - " choices = probs_arr.apply(lambda x: recursive_choice(x, nest_levels[1], nest_levels, nests), axis=1)\n", - " return choices" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "#rands = pipeline.get_rn_generator().random_for_df(utils_df, n=utils_df.shape[1])\n", - "seed = 9326543345\n", - "make_choice(utils_df, nests_, nest_levels, seed)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 316, - "id": "30424562-2627-446b-abd5-7c763c52060e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T01:11:49.444289Z", - "iopub.status.busy": "2021-09-01T01:11:49.444002Z", - "iopub.status.idle": "2021-09-01T01:11:49.664214Z", - "shell.execute_reply": "2021-09-01T01:11:49.663088Z", - "shell.execute_reply.started": "2021-09-01T01:11:49.444266Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4\n" - ] - } - ], - "source": [ - "# group nests by level:\n", - "depth = np.max([x.level for x in nests_])\n", - "print(depth)\n", - "nest_levels = {x: [] for x in range(1, depth+1)}\n", - "for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - "assert len(nest_levels[1]) == 1\n", - "assert nest_levels[1][0] == 'root'" - ] - }, - { - "cell_type": "code", - "execution_count": 453, - "id": "806ce752-0927-4d5b-a6cc-68d6c9b8a05e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:55:59.542669Z", - "iopub.status.busy": "2021-09-01T04:55:59.542360Z", - "iopub.status.idle": "2021-09-01T04:55:59.771722Z", - "shell.execute_reply": "2021-09-01T04:55:59.770801Z", - "shell.execute_reply.started": "2021-09-01T04:55:59.542615Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - " nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec)\n", - " nested_probabilities = \\\n", - " simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, trace_label=trace_label)\n", - " # global (flattened) leaf probabilities based on relative nest coefficients (in spec order)\n", - " base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) \n", - " return base_probabilities\n", - "#simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names)" - ] - }, - { - "cell_type": "code", - "execution_count": 451, - "id": "054fd9b1-72fc-49cc-a790-48ef75bcbaed", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:55:08.260140Z", - "iopub.status.busy": "2021-09-01T04:55:08.259914Z", - "iopub.status.idle": "2021-09-01T04:55:09.140095Z", - "shell.execute_reply": "2021-09-01T04:55:09.139119Z", - "shell.execute_reply.started": "2021-09-01T04:55:08.260116Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "trip_id\n", - "86627409 WALK_LRF\n", - "86627413 WALK_LRF\n", - "86673657 WALK_LOC\n", - "86673658 WALK\n", - "86673659 WALK\n", - " ... \n", - "2464446025 WALK\n", - "2464446029 WALK\n", - "2464449633 WALK\n", - "2464449634 WALK\n", - "2464449637 WALK\n", - "Length: 168, dtype: object" - ] - }, - "execution_count": 451, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def simple_simulate_rum(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " utils_df = compute_nested_utilities(raw_utilities, nest_spec)\n", - "\n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices\n", - "\n", - "simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" - ] - }, - { - "cell_type": "code", - "execution_count": 393, - "id": "f3c5c1ba-e9c3-4511-98bf-9ac98c987da7", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:39:52.539983Z", - "iopub.status.busy": "2021-09-01T02:39:52.539668Z", - "iopub.status.idle": "2021-09-01T02:39:52.796170Z", - "shell.execute_reply": "2021-09-01T02:39:52.795498Z", - "shell.execute_reply.started": "2021-09-01T02:39:52.539944Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " segment_trace_label, log_alt_losers=log_alt_losers, \n", - " trace_column_names=trace_column_names, seed=seed)\n", - "\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " choices_list.append(choices)\n", - "\n", - " choices_df = pd.concat(choices_list)\n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 454, - "id": "74eeedd3-4ade-4729-8170-79fa9dcf11f8", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:58:52.825042Z", - "iopub.status.busy": "2021-09-01T04:58:52.824768Z", - "iopub.status.idle": "2021-09-01T04:58:53.042884Z", - "shell.execute_reply": "2021-09-01T04:58:53.042061Z", - "shell.execute_reply.started": "2021-09-01T04:58:52.825004Z" - } - }, - "outputs": [], - "source": [ - "## caculate probabilities with Asim methodology, should be correct\n", - "def gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None):\n", - " full_probs = [] # analytical probs\n", - "\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " #choices = simple_simulate_rum(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " # segment_trace_label, log_alt_losers=log_alt_losers, \n", - " # trace_column_names=trace_column_names, seed=seed)\n", - " probs = simple_simulate_probabilities(trips_segment, spec, nest_spec, locals_dict, \n", - " estimator, segment_trace_label, log_alt_losers, \n", - " trace_column_names)\n", - " full_probs.append(probs)\n", - " probs_df = pd.concat(full_probs)\n", - " return probs_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": 455, - "id": "295a4fd1-8e85-47e2-af82-9f1beac31d63", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T04:59:46.126555Z", - "iopub.status.busy": "2021-09-01T04:59:46.126262Z", - "iopub.status.idle": "2021-09-01T04:59:52.979908Z", - "shell.execute_reply": "2021-09-01T04:59:52.979043Z", - "shell.execute_reply.started": "2021-09-01T04:59:46.126522Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2021-09-01 14:59:46.353007 Start\n", - "2021-09-01 14:59:52.976736 End\n" - ] - } - ], - "source": [ - "print(f\"{datetime.now()} Start\")\n", - "pipeline.open_pipeline(resume_after)\n", - "inject.get_injectable('preload_injectables', None)\n", - "model_name = \"trip_mode_choice\"\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "\n", - "step_name = model_name\n", - "args = {}\n", - "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "chunk_size = 0\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "trips_df = trips.to_frame()\n", - "#print(\"Running with %d trips\", trips_df.shape[0])\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - "orig_col = 'origin'\n", - "dest_col = 'destination'\n", - "\n", - "constants = {}\n", - "constants.update(config.get_model_constants(model_settings))\n", - "constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - "})\n", - "\n", - "skim_dict = network_los.get_default_skim_dict()\n", - "\n", - "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "\n", - "skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - "}\n", - "\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "\n", - "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - "\n", - "all_choices = []\n", - "for i in range(100):\n", - " if i % 10 == 0:\n", - " print(f\"{datetime.now()} iteration {i}\")\n", - " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=None)\n", - " all_choices.append(choices_df)\n", - "all_choices = pd.concat(all_choices, axis=1)\n", - "\n", - "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", - " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", - " log_alt_losers=None, trace_column_names=None)\n", - "\n", - "# update trips table with choices (and potionally logssums)\n", - "#trips_df = trips.to_frame()\n", - "#\n", - "#assign_in_place(trips_df, choices_df)\n", - "#assert not trips_df[mode_column_name].isnull().any()\n", - "\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - "print(f\"{datetime.now()} End\")" - ] - }, - { - "cell_type": "code", - "execution_count": 463, - "id": "dc87e904-16d1-44d4-81b7-cafbe4e223c3", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:01:40.901982Z", - "iopub.status.busy": "2021-09-01T05:01:40.901759Z", - "iopub.status.idle": "2021-09-01T05:01:41.337758Z", - "shell.execute_reply": "2021-09-01T05:01:41.336757Z", - "shell.execute_reply.started": "2021-09-01T05:01:40.901946Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#all_choices.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", - "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts = val_counts / all_choices.shape[1]\n", - "#val_counts = val_counts.merge(choices_df_asim[['trip_mode']].rename(columns={'trip_mode': 'asim'}), left_index=True, right_index=True)\n", - "#val_counts['prob_of_asim_choice'] = val_counts.apply(lambda x: x[x.asim], axis=1) # this is what our simulation says w.r.t. to asim choice\n", - "# for 100% and many samples should mostly agree" - ] - }, - { - "cell_type": "code", - "execution_count": 464, - "id": "09bf151c-a39a-4823-acfd-9c8ae747c338", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:01:46.119117Z", - "iopub.status.busy": "2021-09-01T05:01:46.118849Z", - "iopub.status.idle": "2021-09-01T05:01:46.354114Z", - "shell.execute_reply": "2021-09-01T05:01:46.353248Z", - "shell.execute_reply.started": "2021-09-01T05:01:46.119092Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "#val_counts['prob_of_asim_choice'].hist(bins=100);" - ] - }, - { - "cell_type": "code", - "execution_count": 490, - "id": "f3b6ade9-d875-4104-b438-b53ef42e342c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T05:10:08.653227Z", - "iopub.status.busy": "2021-09-01T05:10:08.652978Z", - "iopub.status.idle": "2021-09-01T05:10:08.915620Z", - "shell.execute_reply": "2021-09-01T05:10:08.914513Z", - "shell.execute_reply.started": "2021-09-01T05:10:08.653189Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", - "analytical 3.152 0.852 0.0 0.0 0.0 \n", - "experiment 3.243 0.880 0.0 0.0 0.0 \n", - "\n", - " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", - "analytical 0.0 0.0 0.0 0.685 0.0 \n", - "experiment 0.0 0.0 0.0 0.772 0.0 \n", - "\n", - " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK \\\n", - "analytical 0.129 0.0 0.182 0.260 1.334 63.708 \n", - "experiment 0.156 0.0 0.174 0.259 1.434 63.243 \n", - "\n", - " WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", - "analytical 0.0 0.0 0.455 18.355 10.887 \n", - "experiment 0.0 0.0 0.394 18.639 10.807 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "mode_share_analytical = (probs_nl.sum(axis=0) / probs_nl.shape[0]).to_frame('analytical')\n", - "assert np.allclose(mode_share_analytical.sum(), 1)\n", - "mode_share_rum = (val_counts.sum(axis=0) / val_counts.shape[0]).to_frame('experiment')\n", - "assert np.allclose(mode_share_rum.sum(), 1)\n", - "full_share = mode_share_analytical.join(mode_share_rum, how='outer').fillna(0)\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * full_share).T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e48a56e-d613-4dfd-bef6-62bee524f12f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50914e1d-750d-440c-9617-f96bd2a46c56", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2d70e0e-520e-49a6-8024-ae31345f6ead", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "f9e8ef02-1d57-4b21-8aef-25a6e1095c02", - "metadata": {}, - "source": [ - "### try zenith normalisation of simple_simulate_rum" - ] - }, - { - "cell_type": "code", - "execution_count": 629, - "id": "db235e90-d2e5-4e90-b41c-6d90a670e41d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:24.045881Z", - "iopub.status.busy": "2021-09-01T11:37:24.045637Z", - "iopub.status.idle": "2021-09-01T11:37:24.276488Z", - "shell.execute_reply": "2021-09-01T11:37:24.274898Z", - "shell.execute_reply.started": "2021-09-01T11:37:24.045844Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation belowi s equivalent.\n", - "def compute_nested_utilities_zenith(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) # / nest.product_of_coefficients #coefficient\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", - " \n", - " # now go over all leaves and correct for scale\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] /= nest.coefficient\n", - " \n", - " return nested_utilities\n", - "\n", - "\n", - "def simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " utils_df = compute_nested_utilities_zenith(raw_utilities, nest_spec)\n", - "\n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices\n", - "\n", - "#simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=1233974)" - ] - }, - { - "cell_type": "code", - "execution_count": 630, - "id": "f69bc3dc-84e4-4dd5-9a57-c053cadba4b5", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:24.828926Z", - "iopub.status.busy": "2021-09-01T11:37:24.828707Z", - "iopub.status.idle": "2021-09-01T11:37:25.046409Z", - "shell.execute_reply": "2021-09-01T11:37:25.045399Z", - "shell.execute_reply.started": "2021-09-01T11:37:24.828902Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=None, log_alt_losers=None, trace_column_names=None, seed=None):\n", - " choices_list = []\n", - " for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'):\n", - " #print(\"trip_mode_choice tour_type '%s' (%s trips)\" %\n", - " # (primary_purpose, len(trips_segment.index), ))\n", - " # name index so tracing knows how to slice\n", - " assert trips_segment.index.name == 'trip_id'\n", - "\n", - " coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose)\n", - "\n", - " locals_dict = {}\n", - " locals_dict.update(constants)\n", - " locals_dict.update(coefficients)\n", - "\n", - " segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose)\n", - "\n", - " expressions.annotate_preprocessors(\n", - " trips_segment, locals_dict, skims,\n", - " model_settings, segment_trace_label)\n", - "\n", - " locals_dict.update(skims)\n", - "\n", - " spec=simulate.eval_coefficients(model_spec, coefficients, estimator)\n", - " nest_spec = simulate.eval_nest_coefficients(nest_spec, coefficients, segment_trace_label)\n", - " choices = simple_simulate_rum_zenith(trips_segment, spec, nest_spec, locals_dict, estimator, \n", - " segment_trace_label, log_alt_losers=log_alt_losers, \n", - " trace_column_names=trace_column_names, seed=seed)\n", - "\n", - " # for consistency, always return dataframe, whether or not logsums were requested\n", - " if isinstance(choices, pd.Series):\n", - " choices = choices.to_frame('choice')\n", - " choices.rename(columns={'logsum': logsum_column_name,\n", - " 'choice': mode_column_name},\n", - " inplace=True)\n", - " choices_list.append(choices)\n", - "\n", - " choices_df = pd.concat(choices_list)\n", - " return choices_df" - ] - }, - { - "cell_type": "code", - "execution_count": 631, - "id": "8c4f4fb1-c4fb-4a05-9be4-8b9d7f25d1e6", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:37:32.250808Z", - "iopub.status.busy": "2021-09-01T11:37:32.250548Z", - "iopub.status.idle": "2021-09-01T20:20:35.257463Z", - "shell.execute_reply": "2021-09-01T20:20:35.255964Z", - "shell.execute_reply.started": "2021-09-01T11:37:32.250782Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "num_samples = 10\n", - "\n", - "rng_test = default_rng(23423)\n", - "\n", - "print(f\"{datetime.now()} Start\")\n", - "pipeline.open_pipeline(resume_after)\n", - "inject.get_injectable('preload_injectables', None)\n", - "model_name = \"trip_mode_choice\"\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "\n", - "step_name = model_name\n", - "args = {}\n", - "checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "chunk_size = 0\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "trips_df = trips.to_frame()\n", - "#print(\"Running with %d trips\", trips_df.shape[0])\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "\n", - "orig_col = 'origin'\n", - "dest_col = 'destination'\n", - "\n", - "constants = {}\n", - "constants.update(config.get_model_constants(model_settings))\n", - "constants.update({\n", - " 'ORIGIN': orig_col,\n", - " 'DESTINATION': dest_col\n", - "})\n", - "\n", - "skim_dict = network_los.get_default_skim_dict()\n", - "odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col,\n", - " dim3_key='trip_period')\n", - "dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col,\n", - " dim3_key='trip_period')\n", - "od_skim_wrapper = skim_dict.wrap('origin', 'destination')\n", - "skims = {\n", - " \"odt_skims\": odt_skim_stack_wrapper,\n", - " \"dot_skims\": dot_skim_stack_wrapper,\n", - " \"od_skims\": od_skim_wrapper,\n", - "}\n", - "model_spec = simulate.read_model_spec(file_name=model_settings['SPEC'])\n", - "nest_spec = config.get_logit_model_settings(model_settings)\n", - "estimator = estimation.manager.begin_estimation('trip_mode_choice')\n", - "\n", - "all_choices_zenith = []\n", - "all_choices = []\n", - "for i in range(num_samples):\n", - " \n", - " seed = rng_test.integers(0, 100000) #int(9.3 * (i+1)**3) # why not\n", - " \n", - " if i % 50 == 0:\n", - " print(f\"{datetime.now()} iteration {i}\")\n", - " choices_df_zenith = stuff_zenith(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", - " all_choices_zenith.append(choices_df_zenith)\n", - "\n", - " choices_df = stuff(trips_merged, model_settings, constants, skims, model_spec, nest_spec, estimator, logsum_column_name, mode_column_name, \n", - " trace_label=trace_label, log_alt_losers=None, trace_column_names=None, seed=seed)\n", - " all_choices.append(choices_df)\n", - " \n", - " t_ = choices_df_zenith.merge(choices_df, left_index=True, right_index=True, suffixes=['_zenith', '_asim'])\n", - " diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", - " \n", - " #print(f\"seed {seed} leads to {diffs.shape[0]} differences. tripids {diffs.index}\")\n", - "\n", - "all_choices_zenith = pd.concat(all_choices_zenith, axis=1)\n", - "all_choices = pd.concat(all_choices, axis=1)\n", - "\n", - "probs_nl = gimme_probabilities(trips_merged, model_settings, constants, skims, model_spec, nest_spec,\n", - " estimator, logsum_column_name, mode_column_name,trace_label=trace_label, \n", - " log_alt_losers=None, trace_column_names=None)\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()\n", - "\n", - "print(f\"{datetime.now()} End\")" - ] - }, - { - "cell_type": "code", - "execution_count": 643, - "id": "3abd7949-dc9c-469b-845e-26c17741f70e", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:33:21.533110Z", - "iopub.status.busy": "2021-09-01T20:33:21.531915Z", - "iopub.status.idle": "2021-09-01T20:33:21.740946Z", - "shell.execute_reply": "2021-09-01T20:33:21.739713Z", - "shell.execute_reply.started": "2021-09-01T20:33:21.533074Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Have 2038 samples\n" - ] - } - ], - "source": [ - "print(f\"Have {all_choices_zenith.shape[1]} samples\")" - ] - }, - { - "cell_type": "code", - "execution_count": 633, - "id": "8762bd6d-ac04-458c-869f-be86d8297351", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:21:27.778803Z", - "iopub.status.busy": "2021-09-01T20:21:27.778531Z", - "iopub.status.idle": "2021-09-01T20:21:28.675645Z", - "shell.execute_reply": "2021-09-01T20:21:28.674734Z", - "shell.execute_reply.started": "2021-09-01T20:21:27.778754Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "val_counts_zenith = all_choices_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts_zenith = val_counts_zenith / all_choices_zenith.shape[1]\n", - "\n", - "val_counts = all_choices.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "val_counts = val_counts / all_choices.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 634, - "id": "0f566105-9424-457b-86f5-d60c1d63aac5", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:21:30.954894Z", - "iopub.status.busy": "2021-09-01T20:21:30.954654Z", - "iopub.status.idle": "2021-09-01T20:21:31.179689Z", - "shell.execute_reply": "2021-09-01T20:21:31.178304Z", - "shell.execute_reply.started": "2021-09-01T20:21:30.954867Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
analytical3.1520.8520.00.00.00.00.00.00.6850.00.1290.00.1820.2601.33463.7080.00.00.45518.35510.887
experiment3.2430.8800.00.00.00.00.00.00.7720.00.1560.00.1740.2591.43463.2430.00.00.39418.63910.807
experiment_zenith3.1500.8740.00.00.00.00.00.00.6800.00.1360.00.1540.2361.28563.7670.00.00.51518.82410.379
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP \\\n", - "analytical 3.152 0.852 0.0 0.0 0.0 \n", - "experiment 3.243 0.880 0.0 0.0 0.0 \n", - "experiment_zenith 3.150 0.874 0.0 0.0 0.0 \n", - "\n", - " DRIVE_HVY DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY \\\n", - "analytical 0.0 0.0 0.0 0.685 0.0 \n", - "experiment 0.0 0.0 0.0 0.772 0.0 \n", - "experiment_zenith 0.0 0.0 0.0 0.680 0.0 \n", - "\n", - " SHARED3FREE SHARED3PAY TAXI TNC_SHARED TNC_SINGLE \\\n", - "analytical 0.129 0.0 0.182 0.260 1.334 \n", - "experiment 0.156 0.0 0.174 0.259 1.434 \n", - "experiment_zenith 0.136 0.0 0.154 0.236 1.285 \n", - "\n", - " WALK WALK_COM WALK_EXP WALK_HVY WALK_LOC WALK_LRF \n", - "analytical 63.708 0.0 0.0 0.455 18.355 10.887 \n", - "experiment 63.243 0.0 0.0 0.394 18.639 10.807 \n", - "experiment_zenith 63.767 0.0 0.0 0.515 18.824 10.379 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "mode_share_rum_zenith = (val_counts_zenith.sum(axis=0) / val_counts_zenith.shape[0]).to_frame('experiment_zenith')\n", - "assert np.allclose(mode_share_rum_zenith.sum(), 1)\n", - "full_share_incl_zenith = full_share.merge(mode_share_rum_zenith, left_index=True, right_index=True, how='outer').fillna(0)\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * full_share_incl_zenith).T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c0d3081-b45b-4573-9246-e4a9d39591dd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "612b2f04-45e2-4eb2-b52a-9f753427877f", - "metadata": {}, - "source": [ - "## investigate diverging seed and look at diff in formulation" - ] - }, - { - "cell_type": "code", - "execution_count": 619, - "id": "ac09dc75-a92f-43fb-a0cd-fe7de1f8b66d", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:12:02.988141Z", - "iopub.status.busy": "2021-09-01T11:12:02.987900Z", - "iopub.status.idle": "2021-09-01T11:12:03.212179Z", - "shell.execute_reply": "2021-09-01T11:12:03.210993Z", - "shell.execute_reply.started": "2021-09-01T11:12:02.988102Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# TODO: work out if our formulation and formulation based on asim probabilities is equivalent\n", - "def compute_nested_utilities_asim(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) / nest.product_of_coefficients\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1))\n", - " return nested_utilities\n", - "\n", - "def compute_nested_utilities_zenith_check(raw_utilities, nest_spec):\n", - " nested_utilities = pd.DataFrame(index=raw_utilities.index)\n", - " for nest in logit.each_nest(nest_spec, post_order=True):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] = \\\n", - " raw_utilities[name].astype(float) # scale correction is below\n", - " else:\n", - " with np.errstate(divide='ignore'):\n", - " nested_utilities[name] = \\\n", - " nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives] / nest.coefficient).sum(axis=1))\n", - " \n", - " # now go over all leaves and correct for scale\n", - " for nest in logit.each_nest(nest_spec):\n", - " name = nest.name\n", - " if nest.is_leaf:\n", - " nested_utilities[name] /= nest.coefficient\n", - " \n", - " return nested_utilities" - ] - }, - { - "cell_type": "code", - "execution_count": 688, - "id": "4b7c6368-21ca-4442-969e-f2f8ff868842", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T20:49:56.958862Z", - "iopub.status.busy": "2021-09-01T20:49:56.958623Z", - "iopub.status.idle": "2021-09-01T20:49:57.191884Z", - "shell.execute_reply": "2021-09-01T20:49:57.190975Z", - "shell.execute_reply.started": "2021-09-01T20:49:56.958827Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_d, estimator, tr_label, log_alt_losers, trace_column_names, custom_chooser=None, seed=None, use_zenith=True, raw_utilities=None):\n", - " trace_label = tracing.extend_trace_label(tr_label, 'eval_nl')\n", - " logit.validate_nest_spec(nest_spec, trace_label)\n", - " \n", - " if raw_utilities is None:\n", - " raw_utilities = simulate.eval_utilities(spec, trips_segment, locals_d,\n", - " log_alt_losers=log_alt_losers,\n", - " trace_label=trace_label, have_trace_targets=False,\n", - " estimator=estimator, trace_column_names=trace_column_names)\n", - "\n", - " if use_zenith:\n", - " utils_df = compute_nested_utilities_zenith_check(raw_utilities, nest_spec)\n", - " else:\n", - " utils_df = compute_nested_utilities_asim(raw_utilities, nest_spec)\n", - "\n", - " # test\n", - " #return compute_nested_utilities_zenith_check(raw_utilities, nest_spec), compute_nested_utilities_asim(raw_utilities, nest_spec)\n", - " \n", - " nests_ = list(logit.each_nest(nest_spec))\n", - " # group nests by level:\n", - " depth = np.max([x.level for x in nests_])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in nests_:\n", - " nest_levels[n.level].append(n.name)\n", - " assert len(nest_levels[1]) == 1\n", - " assert nest_levels[1][0] == 'root'\n", - " # make choices\n", - " choices = make_choice(utils_df, nests_, nest_levels, seed)\n", - "\n", - " return choices" - ] - }, - { - "cell_type": "code", - "execution_count": 626, - "id": "bd6cadc3-79f5-4824-b55e-639c39d5ab18", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:13:12.077886Z", - "iopub.status.busy": "2021-09-01T11:13:12.077652Z", - "iopub.status.idle": "2021-09-01T11:13:12.290222Z", - "shell.execute_reply": "2021-09-01T11:13:12.288861Z", - "shell.execute_reply.started": "2021-09-01T11:13:12.077863Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# t_z, t_a = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed)\n", - "# tr_id = 86673661\n", - "# display(t_z.loc[t_z.index==tr_id])\n", - "# display(t_a.loc[t_a.index==tr_id])" - ] - }, - { - "cell_type": "code", - "execution_count": 625, - "id": "42e6da62-58e8-4b16-9169-9faa1fc8a162", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T11:13:06.847583Z", - "iopub.status.busy": "2021-09-01T11:13:06.847204Z", - "iopub.status.idle": "2021-09-01T11:13:08.401700Z", - "shell.execute_reply": "2021-09-01T11:13:08.400711Z", - "shell.execute_reply.started": "2021-09-01T11:13:06.847554Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed 51006 leads to 34 differences out of 168. tripids Int64Index([ 86673657, 86673661, 106741681, 106741682, 106741685,\n", - " 106741686, 106741687, 106741688, 211327433, 211327437,\n", - " 444793574, 484173905, 484173909, 535170694, 535620053,\n", - " 708171014, 943749470, 943749471, 1060575853, 1091770617,\n", - " 1146472489, 1146472493, 1276281769, 1276281773, 1658748793,\n", - " 1658748797, 1767013726, 1767186577, 1767186578, 1768237161,\n", - " 1768237165, 1768237166, 2463663417, 2463663421],\n", - " dtype='int64', name='trip_id')\n" - ] - } - ], - "source": [ - "seed = 51006\n", - "x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True)\n", - "y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False)\n", - "t_ = x_.to_frame('trip_mode_zenith').merge(y_.to_frame('trip_mode_asim'), left_index=True, right_index=True)\n", - "diffs = t_.loc[t_.trip_mode_zenith != t_.trip_mode_asim]\n", - "print(f\"seed {seed} leads to {diffs.shape[0]} differences out of {t_.shape[0]}. tripids {diffs.index}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2f3c0506-a79d-4a31-9d56-71b9966d0a92", - "metadata": {}, - "outputs": [], - "source": [ - "seed 51006 leads to 3 differences. tripids Int64Index([86673661, 535170689, 1060575849], dtype='int64', name='trip_id')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1a5f332a-b1d7-4a80-bb21-d37a1611cf9f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24c5e4a1-6142-49d4-8da8-a1e266b3cb9b", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 753, - "id": "2098907f-28af-4f53-af74-6aaee8a6c53c", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T23:04:23.861048Z", - "iopub.status.busy": "2021-09-01T23:04:23.860748Z", - "iopub.status.idle": "2021-09-01T23:04:24.069307Z", - "shell.execute_reply": "2021-09-01T23:04:24.068434Z", - "shell.execute_reply.started": "2021-09-01T23:04:23.861024Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "rng_ = default_rng(seed=100)\n", - "new_utils = pd.DataFrame(0.1 * -np.log(-np.log(rng_.uniform(0,1,raw_utilities.shape))), columns=raw_utilities.columns)\n", - "new_utils.index = raw_utilities.index" - ] - }, - { - "cell_type": "code", - "execution_count": 767, - "id": "da48a178-82b2-44a5-b45a-e9de3bb5e5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T23:17:20.369675Z", - "iopub.status.busy": "2021-09-01T23:17:20.369424Z", - "iopub.status.idle": "2021-09-01T23:17:20.666506Z", - "shell.execute_reply": "2021-09-01T23:17:20.665626Z", - "shell.execute_reply.started": "2021-09-01T23:17:20.369637Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
1372487210.0566420.0373050.0427800.0295760.0861540.0265860.0883130.1003310.0280210.0185160.0643910.0200090.0263990.0226220.0230070.0217540.0871260.0211710.0364750.0276500.135171
1372487250.1019020.0228760.0312270.0699900.0569130.0466850.0884110.0877850.0254140.0381620.0239880.0288000.0393180.0201600.0483640.0230030.0339590.0345970.0377080.0995230.041213
2113882010.0375620.0700330.0559510.0403890.0200410.0959740.0946780.0943680.0360140.0250510.0348600.0349530.0354990.0280400.0436500.0247130.0313880.0259310.0669990.0526780.051228
2113882050.0354800.0562560.0439440.0480850.0420780.0622210.1068190.0872650.0351170.0303990.0351540.0264690.0506460.0357270.0242840.0329050.0394270.0410920.0547140.0484590.063458
8063884010.0549180.0404880.0331490.0542940.0434770.0679820.1524650.0741520.0334040.0287350.0592690.0243390.0369680.0277600.0235220.0609320.0234450.0239310.0446430.0337730.058353
8063884050.0327640.0573010.0184370.0770990.0469630.0635210.0949510.0890720.0308260.0378960.0357520.0198780.0495360.0796100.0269220.0248330.0248400.0266950.0799360.0401460.043022
\n", - "
" - ], - "text/plain": [ - " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n", - "trip_id \n", - "137248721 0.056642 0.037305 0.042780 0.029576 \n", - "137248725 0.101902 0.022876 0.031227 0.069990 \n", - "211388201 0.037562 0.070033 0.055951 0.040389 \n", - "211388205 0.035480 0.056256 0.043944 0.048085 \n", - "806388401 0.054918 0.040488 0.033149 0.054294 \n", - "806388405 0.032764 0.057301 0.018437 0.077099 \n", - "\n", - " SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\n", - "trip_id \n", - "137248721 0.086154 0.026586 0.088313 0.100331 0.028021 0.018516 \n", - "137248725 0.056913 0.046685 0.088411 0.087785 0.025414 0.038162 \n", - "211388201 0.020041 0.095974 0.094678 0.094368 0.036014 0.025051 \n", - "211388205 0.042078 0.062221 0.106819 0.087265 0.035117 0.030399 \n", - "806388401 0.043477 0.067982 0.152465 0.074152 0.033404 0.028735 \n", - "806388405 0.046963 0.063521 0.094951 0.089072 0.030826 0.037896 \n", - "\n", - " WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\n", - "trip_id \n", - "137248721 0.064391 0.020009 0.026399 0.022622 0.023007 0.021754 \n", - "137248725 0.023988 0.028800 0.039318 0.020160 0.048364 0.023003 \n", - "211388201 0.034860 0.034953 0.035499 0.028040 0.043650 0.024713 \n", - "211388205 0.035154 0.026469 0.050646 0.035727 0.024284 0.032905 \n", - "806388401 0.059269 0.024339 0.036968 0.027760 0.023522 0.060932 \n", - "806388405 0.035752 0.019878 0.049536 0.079610 0.026922 0.024833 \n", - "\n", - " DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n", - "trip_id \n", - "137248721 0.087126 0.021171 0.036475 0.027650 0.135171 \n", - "137248725 0.033959 0.034597 0.037708 0.099523 0.041213 \n", - "211388201 0.031388 0.025931 0.066999 0.052678 0.051228 \n", - "211388205 0.039427 0.041092 0.054714 0.048459 0.063458 \n", - "806388401 0.023445 0.023931 0.044643 0.033773 0.058353 \n", - "806388405 0.024840 0.026695 0.079936 0.040146 0.043022 " - ] - }, - "execution_count": 767, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nested_exp_utilities_cf = simulate.compute_nested_exp_utilities(new_utils, nest_spec)\n", - "nested_probabilities_cf = simulate.compute_nested_probabilities(nested_exp_utilities_cf, nest_spec, trace_label=None)\n", - "base_probabilities_cf = simulate.compute_base_probabilities(nested_probabilities_cf, nest_spec, spec)\n", - "base_probabilities_cf" - ] - }, - { - "cell_type": "code", - "execution_count": 799, - "id": "50dfd1ae-10c3-475c-94be-de783c2fa5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:15:29.056902Z", - "iopub.status.busy": "2021-09-02T00:15:29.056683Z", - "iopub.status.idle": "2021-09-02T00:27:21.965206Z", - "shell.execute_reply": "2021-09-02T00:27:21.964340Z", - "shell.execute_reply.started": "2021-09-02T00:15:29.056877Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 11min 49s, sys: 766 ms, total: 11min 50s\n", - "Wall time: 11min 52s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "num_samples = 10000 # 7.5s per 100\n", - "\n", - "data_zenith = []\n", - "data_asim = []\n", - "for i in range(num_samples):\n", - " seed = rng_.integers(0, 100000)\n", - " x_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=True, raw_utilities=new_utils)\n", - " y_ = simple_simulate_rum_zenith_check(trips_segment, spec, nest_spec, locals_dict, estimator, tr_label, log_alt_losers, trace_column_names, seed=seed, use_zenith=False, raw_utilities=new_utils)\n", - " data_zenith.append(x_)\n", - " data_asim.append(y_)\n", - " \n", - "data_asim = pd.concat(data_asim, axis=1)\n", - "data_zenith = pd.concat(data_zenith, axis=1)\n", - "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", - "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_asim = counts_asim / data_asim.shape[1]\n", - "\n", - "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", - "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", - "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", - "# assert np.allclose(mode_share_zenith.sum(), 1)\n", - "# assert np.allclose(mode_share_asim.sum(), 1)\n", - "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", - "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 796, - "id": "bb16f9d7-44f5-4316-b981-d65216a5e217", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:14:11.772592Z", - "iopub.status.busy": "2021-09-02T00:14:11.772347Z", - "iopub.status.idle": "2021-09-02T00:14:12.001077Z", - "shell.execute_reply": "2021-09-02T00:14:12.000166Z", - "shell.execute_reply.started": "2021-09-02T00:14:11.772556Z" - } - }, - "outputs": [], - "source": [ - "#temp_z = data_zenith.copy()\n", - "#temp_a = data_asim.copy()\n", - "#data_asim = data_asim.join(temp_a, lsuffix=\"_o\", rsuffix=\"_n\")\n", - "#data_zenith = data_zenith.join(temp_z, lsuffix=\"_o\", rsuffix=\"_n\")\n", - "# counts_zenith = data_zenith.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_zenith = counts_zenith / data_zenith.shape[1]\n", - "# counts_asim = data_asim.apply(lambda x: x.value_counts(), axis=1).fillna(0)\n", - "# counts_asim = counts_asim / data_asim.shape[1]\n", - "\n", - "# mode_share_zenith = (counts_zenith.sum(axis=0) / counts_zenith.shape[0]).to_frame('zenith')\n", - "# mode_share_asim = (counts_asim.sum(axis=0) / counts_asim.shape[0]).to_frame('asim')\n", - "# mode_share_base_prob = (base_probabilities_cf.sum(axis=0) / base_probabilities_cf.shape[0]).to_frame('probs')\n", - "# assert np.allclose(mode_share_zenith.sum(), 1)\n", - "# assert np.allclose(mode_share_asim.sum(), 1)\n", - "# assert np.allclose(mode_share_base_prob.sum(), 1)\n", - "# mode_share_comp = mode_share_zenith.join(mode_share_asim, how='outer').join(mode_share_base_prob, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 805, - "id": "031d8340-0af0-4608-8a83-ed2af4aaa3b2", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:41:01.948669Z", - "iopub.status.busy": "2021-09-02T00:41:01.948422Z", - "iopub.status.idle": "2021-09-02T00:41:02.175972Z", - "shell.execute_reply": "2021-09-02T00:41:02.174759Z", - "shell.execute_reply.started": "2021-09-02T00:41:01.948633Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
BIKEDRIVEALONEFREEDRIVEALONEPAYDRIVE_COMDRIVE_EXPDRIVE_HVYDRIVE_LOCDRIVE_LRFSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYTAXITNC_SHAREDTNC_SINGLEWALKWALK_COMWALK_EXPWALK_HVYWALK_LOCWALK_LRF
zenith8.0555.1674.6313.5023.6064.2663.9633.4983.8955.1764.7995.8514.9996.1214.6689.4864.2284.1252.9773.6713.317
asim6.0966.6984.9773.2023.5224.3344.1293.1843.2245.1496.2838.2964.7936.5844.5327.4844.1524.2732.6293.4333.023
probs8.8835.3214.7382.8903.1364.0033.5653.1623.7585.3244.9276.0495.3416.5415.03710.4273.9734.2242.5743.1472.979
\n", - "
" - ], - "text/plain": [ - " BIKE DRIVEALONEFREE DRIVEALONEPAY DRIVE_COM DRIVE_EXP DRIVE_HVY \\\n", - "zenith 8.055 5.167 4.631 3.502 3.606 4.266 \n", - "asim 6.096 6.698 4.977 3.202 3.522 4.334 \n", - "probs 8.883 5.321 4.738 2.890 3.136 4.003 \n", - "\n", - " DRIVE_LOC DRIVE_LRF SHARED2FREE SHARED2PAY SHARED3FREE \\\n", - "zenith 3.963 3.498 3.895 5.176 4.799 \n", - "asim 4.129 3.184 3.224 5.149 6.283 \n", - "probs 3.565 3.162 3.758 5.324 4.927 \n", - "\n", - " SHARED3PAY TAXI TNC_SHARED TNC_SINGLE WALK WALK_COM WALK_EXP \\\n", - "zenith 5.851 4.999 6.121 4.668 9.486 4.228 4.125 \n", - "asim 8.296 4.793 6.584 4.532 7.484 4.152 4.273 \n", - "probs 6.049 5.341 6.541 5.037 10.427 3.973 4.224 \n", - "\n", - " WALK_HVY WALK_LOC WALK_LRF \n", - "zenith 2.977 3.671 3.317 \n", - "asim 2.629 3.433 3.023 \n", - "probs 2.574 3.147 2.979 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# sample of 20000\n", - "with pd.option_context(\"precision\", 3):\n", - " display((100.0 * mode_share_comp).T)" - ] - }, - { - "cell_type": "code", - "execution_count": 806, - "id": "aece448f-4055-4d2a-b90d-8530caec9cb9", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-02T00:43:32.971972Z", - "iopub.status.busy": "2021-09-02T00:43:32.971684Z", - "iopub.status.idle": "2021-09-02T00:43:33.223345Z", - "shell.execute_reply": "2021-09-02T00:43:33.222488Z", - "shell.execute_reply.started": "2021-09-02T00:43:32.971932Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
trip_id137248721_z137248721_a137248721
BIKE0.092700.054350.100331
DRIVEALONEFREE0.053650.050050.056642
DRIVEALONEPAY0.038800.031550.037305
DRIVE_COM0.028700.024700.021171
DRIVE_EXP0.028800.026050.021754
DRIVE_HVY0.078150.099700.087126
DRIVE_LOC0.028750.027000.022622
DRIVE_LRF0.027800.024450.023007
SHARED2FREE0.043250.023000.042780
SHARED2PAY0.033600.015850.029576
SHARED3FREE0.073650.119200.086154
SHARED3PAY0.032600.039600.026586
TAXI0.035100.050300.036475
TNC_SHARED0.126200.181150.135171
TNC_SINGLE0.026250.038450.027650
WALK0.079400.048250.088313
WALK_COM0.031850.024600.026399
WALK_EXP0.057050.058950.064391
WALK_HVY0.025200.018400.020009
WALK_LOC0.034550.027400.028021
WALK_LRF0.023950.017000.018516
\n", - "
" - ], - "text/plain": [ - "trip_id 137248721_z 137248721_a 137248721\n", - "BIKE 0.09270 0.05435 0.100331\n", - "DRIVEALONEFREE 0.05365 0.05005 0.056642\n", - "DRIVEALONEPAY 0.03880 0.03155 0.037305\n", - "DRIVE_COM 0.02870 0.02470 0.021171\n", - "DRIVE_EXP 0.02880 0.02605 0.021754\n", - "DRIVE_HVY 0.07815 0.09970 0.087126\n", - "DRIVE_LOC 0.02875 0.02700 0.022622\n", - "DRIVE_LRF 0.02780 0.02445 0.023007\n", - "SHARED2FREE 0.04325 0.02300 0.042780\n", - "SHARED2PAY 0.03360 0.01585 0.029576\n", - "SHARED3FREE 0.07365 0.11920 0.086154\n", - "SHARED3PAY 0.03260 0.03960 0.026586\n", - "TAXI 0.03510 0.05030 0.036475\n", - "TNC_SHARED 0.12620 0.18115 0.135171\n", - "TNC_SINGLE 0.02625 0.03845 0.027650\n", - "WALK 0.07940 0.04825 0.088313\n", - "WALK_COM 0.03185 0.02460 0.026399\n", - "WALK_EXP 0.05705 0.05895 0.064391\n", - "WALK_HVY 0.02520 0.01840 0.020009\n", - "WALK_LOC 0.03455 0.02740 0.028021\n", - "WALK_LRF 0.02395 0.01700 0.018516" - ] - }, - "execution_count": 806, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# TODO: look at probs per trip, i.e. do not sum across trips\n", - "trip_id = 137248721\n", - "\n", - "counts_zenith.loc[counts_zenith.index == trip_id].T.merge(\n", - " counts_asim.loc[counts_asim.index == trip_id].T, suffixes=['_z', '_a'], left_index=True, right_index=True, how='outer').merge(\n", - " base_probabilities_cf.loc[base_probabilities_cf.index == trip_id].T, suffixes=['', '_probs'], left_index=True, right_index=True, how='outer').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ef84f10-d616-40e4-87e5-425499027ff3", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "431cee49-40a9-4dc7-93a5-75308fb302dd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1fcdc0ff-c510-4878-8672-8365e046442c", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/fru_utils.py b/notebooks/fru_utils.py deleted file mode 100644 index 77a4e920c2..0000000000 --- a/notebooks/fru_utils.py +++ /dev/null @@ -1,582 +0,0 @@ -import os -from datetime import datetime - -import numpy as np -import pandas as pd - -from activitysim.core import inject -from activitysim.core import tracing -from activitysim.core import config -from activitysim.core import pipeline -from activitysim.core import chunk -from activitysim.core import simulate -from activitysim.core import logit -from activitysim.abm.models.util import estimation -from activitysim.core import expressions -from activitysim.core.util import assign_in_place - - - - - -def mode_choice_for_trip(choose_individual_max_utility, trip_id_to_check, num_samples, trips_merged=None): - """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" - resume_after = "trip_scheduling" - model_name = "trip_mode_choice" - chunk_size = 0 # test_mtc means no chunking - - pipeline.open_pipeline(resume_after) - # preload any bulky injectables (e.g. skims) not in pipeline - inject.get_injectable('preload_injectables', None) - pipeline._PIPELINE.rng().begin_step(model_name) - #step_name = model_name - args = {} - #checkpoint = pipeline.intermediate_checkpoint(model_name) - inject.set_step_args(args) - - network_los = inject.get_injectable('network_los') - - trace_label = 'trip_mode_choice' - model_settings_file_name = 'trip_mode_choice.yaml' - model_settings = config.read_model_settings(model_settings_file_name) - - logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') - mode_column_name = 'trip_mode' - - if trips_merged is None: - trips = inject.get_table('trips') - tours_merged = inject.get_table('tours_merged') - trips_df = trips.to_frame() - tours_merged = tours_merged.to_frame() - tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] - trips_merged = pd.merge( - trips_df, - tours_merged, - left_on='tour_id', - right_index=True, - how="left") - assert trips_merged.index.equals(trips.index) - assert ('trip_period' not in trips_merged) - trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) - - orig_col = 'origin' - dest_col = 'destination' - - constants = {} - constants.update(config.get_model_constants(model_settings)) - constants.update({ - 'ORIGIN': orig_col, - 'DESTINATION': dest_col - }) - - skim_dict = network_los.get_default_skim_dict() - - odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col, - dim3_key='trip_period') - dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col, - dim3_key='trip_period') - od_skim_wrapper = skim_dict.wrap('origin', 'destination') - - skims = { - "odt_skims": odt_skim_stack_wrapper, - "dot_skims": dot_skim_stack_wrapper, - "od_skims": od_skim_wrapper, - } - - model_spec = simulate.read_model_spec(file_name=model_settings['SPEC']) - nest_specs = config.get_logit_model_settings(model_settings) - - estimator = estimation.manager.begin_estimation('trip_mode_choice') - - # grab one, duplicate num_samples times - trips_segment = trips_merged.loc[trips_merged.index == trip_id_to_check].copy() - primary_purpose = trips_segment['primary_purpose'].values[0] - trips_segment = trips_segment.loc[trips_segment.index.repeat(num_samples)] - - # need to add new row_states for rng here, need to ensure there are no collisions with existing keys - existing_indexes = pipeline._PIPELINE.rng().get_channel_for_df(trips_merged).row_states.index.values - num_new_indexes = trips_segment.shape[0] - new_indexes = np.arange(existing_indexes.max()+1, existing_indexes.max() + num_new_indexes + 1) - trips_segment.index = new_indexes - # name index so tracing knows how to slice - trips_segment.index.name = 'trip_id' - - pipeline._PIPELINE.rng().add_channel("trips", trips_segment) - - print("trip_mode_choice tour_type '%s' (%s trips)" % - (primary_purpose, len(trips_segment.index), )) - - coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose) - - locals_dict = {} - locals_dict.update(constants) - locals_dict.update(coefficients) - - segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose) - - expressions.annotate_preprocessors( - trips_segment, locals_dict, skims, - model_settings, segment_trace_label) - - locals_dict.update(skims) - - ################ Replace wrapper function - # choices = mode_choice_simulate(...) - spec=simulate.eval_coefficients(model_spec, coefficients, estimator) - nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label) - choices = simulate.simple_simulate( - choosers=trips_segment, - spec=spec, - nest_spec=nest_spec, - skims=skims, - locals_d=locals_dict, - chunk_size=chunk_size, - want_logsums=logsum_column_name is not None, - trace_label=segment_trace_label, - trace_choice_name='trip_mode_choice', - estimator=estimator, - trace_column_names=None, - choose_individual_max_utility=choose_individual_max_utility) - # for consistency, always return dataframe, whether or not logsums were requested - if isinstance(choices, pd.Series): - choices = choices.to_frame('choice') - choices.rename(columns={'logsum': logsum_column_name, - 'choice': mode_column_name}, - inplace=True) - - # THIS SHOULD NOW BE CONSISTENT - #if not choose_individual_max_utility: - alts = spec.columns - choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) - - finalise = True - if finalise: - inject.set_step_args(None) - # - pipeline._PIPELINE.rng().end_step(model_name) - pipeline.add_checkpoint(model_name) - if not pipeline.intermediate_checkpoint(): - pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME) - - pipeline.close_pipeline() - - print("Done") - return trips_merged, choices, nest_spec - - -# def comp_mode_shares(base_probs, choose_individual_max_utility, num_samples, trip_id_to_check): -# t_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility, -# trip_id_to_check=trip_id_to_check, num_samples=num_samples) -# sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0] -# return sim_mode_shares - -def comp_mode_shares(base_probs, sim_mode_shares, trip_id_to_check): - obs_probs = base_probs.loc[base_probs.index == trip_id_to_check].T - obs_probs.columns = ["mode_share_obs"] - ms_comp = obs_probs.merge(sim_mode_shares, left_index=True, right_index=True, how="outer").fillna(0) - ms_comp["diff"] = ms_comp["trip_mode"] - ms_comp["mode_share_obs"] - ms_comp["rel_diff"] = ms_comp["diff"] / ms_comp["mode_share_obs"] - #ms_comp.style.format('{:.2}') - #with pd.option_context("precision", 3): - # display(ms_comp) - return ms_comp - - -def run_trip_mode_choice(do_these_purposes=None, choose_individual_max_utility=True): - - """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" - resume_after = "trip_scheduling" - model_name = "trip_mode_choice" - chunk_size = 0 # test_mtc means no chunking - - pipeline.open_pipeline(resume_after) - # preload any bulky injectables (e.g. skims) not in pipeline - inject.get_injectable('preload_injectables', None) - pipeline._PIPELINE.rng().begin_step(model_name) - step_name = model_name - args = {} - checkpoint = pipeline.intermediate_checkpoint(model_name) - inject.set_step_args(args) - - trips = inject.get_table('trips') - tours_merged = inject.get_table('tours_merged') - network_los = inject.get_injectable('network_los') - - trace_label = 'trip_mode_choice' - model_settings_file_name = 'trip_mode_choice.yaml' - model_settings = config.read_model_settings(model_settings_file_name) - - logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') - mode_column_name = 'trip_mode' - - trips_df = trips.to_frame() - print("Running with %d trips", trips_df.shape[0]) - - tours_merged = tours_merged.to_frame() - tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] - - # - trips_merged - merge trips and tours_merged - trips_merged = pd.merge( - trips_df, - tours_merged, - left_on='tour_id', - right_index=True, - how="left") - assert trips_merged.index.equals(trips.index) - - # setup skim keys - assert ('trip_period' not in trips_merged) - trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) - - orig_col = 'origin' - dest_col = 'destination' - - constants = {} - constants.update(config.get_model_constants(model_settings)) - constants.update({ - 'ORIGIN': orig_col, - 'DESTINATION': dest_col - }) - - skim_dict = network_los.get_default_skim_dict() - - odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col, - dim3_key='trip_period') - dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col, - dim3_key='trip_period') - od_skim_wrapper = skim_dict.wrap('origin', 'destination') - - skims = { - "odt_skims": odt_skim_stack_wrapper, - "dot_skims": dot_skim_stack_wrapper, - "od_skims": od_skim_wrapper, - } - - model_spec = simulate.read_model_spec(file_name=model_settings['SPEC']) - nest_specs = config.get_logit_model_settings(model_settings) - - estimator = estimation.manager.begin_estimation('trip_mode_choice') - - choices_list = [] - for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'): - - if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes): - continue - - print("trip_mode_choice tour_type '%s' (%s trips)" % - (primary_purpose, len(trips_segment.index), )) - - # name index so tracing knows how to slice - assert trips_segment.index.name == 'trip_id' - - coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose) - - locals_dict = {} - locals_dict.update(constants) - locals_dict.update(coefficients) - - segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose) - - expressions.annotate_preprocessors( - trips_segment, locals_dict, skims, - model_settings, segment_trace_label) - - locals_dict.update(skims) - - ################ Replace wrapper function - # choices = mode_choice_simulate(...) - spec=simulate.eval_coefficients(model_spec, coefficients, estimator) - nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label) - choices = simulate.simple_simulate( - choosers=trips_segment, - spec=spec, - nest_spec=nest_spec, - skims=skims, - locals_d=locals_dict, - chunk_size=chunk_size, - want_logsums=logsum_column_name is not None, - trace_label=segment_trace_label, - trace_choice_name='trip_mode_choice', - estimator=estimator, - trace_column_names=None, - choose_individual_max_utility=choose_individual_max_utility) - # for consistency, always return dataframe, whether or not logsums were requested - if isinstance(choices, pd.Series): - choices = choices.to_frame('choice') - choices.rename(columns={'logsum': logsum_column_name, - 'choice': mode_column_name}, - inplace=True) - - # SHOULD NOW BE CONSISTENT - #if not choose_individual_max_utility: - alts = spec.columns - choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) - ################ - choices_list.append(choices) - choices_df_asim = pd.concat(choices_list) - - # update trips table with choices (and potionally logssums) - trips_df = trips.to_frame() - - if (do_these_purposes is not None): - trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)] - - assign_in_place(trips_df, choices_df_asim) - assert not trips_df[mode_column_name].isnull().any() - - finalise = True - if finalise: - inject.set_step_args(None) - # - pipeline._PIPELINE.rng().end_step(model_name) - pipeline.add_checkpoint(model_name) - if not pipeline.intermediate_checkpoint(): - pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME) - - pipeline.close_pipeline() - - print("Done") - - return trips_df - - - - -def eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, estimator, - log_alt_losers=False, - want_logsums=False, trace_label=None, - trace_choice_name=None, trace_column_names=None): - - trace_label = tracing.extend_trace_label(trace_label, 'eval_nl') - assert trace_label - have_trace_targets = tracing.has_trace_targets(choosers) - - logit.validate_nest_spec(nest_spec, trace_label) - raw_utilities = simulate.eval_utilities(spec, choosers, locals_d, - log_alt_losers=log_alt_losers, - trace_label=trace_label, have_trace_targets=have_trace_targets, - estimator=estimator, trace_column_names=trace_column_names) - # exponentiated utilities of leaves and nests - nested_exp_utilities = simulate.compute_nested_exp_utilities(raw_utilities, nest_spec) - nested_utils = simulate.compute_nested_utilities(raw_utilities, nest_spec) - # probabilities of alternatives relative to siblings sharing the same nest - nested_probabilities = simulate.compute_nested_probabilities(nested_exp_utilities, nest_spec, - trace_label=trace_label) - if want_logsums: - # logsum of nest root - logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) - # global (flattened) leaf probabilities based on relative nest coefficients (in spec order) - base_probabilities = simulate.compute_base_probabilities(nested_probabilities, nest_spec, spec) - # note base_probabilities could all be zero since we allowed all probs for nests to be zero - # check here to print a clear message but make_choices will raise error if probs don't sum to 1 - BAD_PROB_THRESHOLD = 0.001 - no_choices = (base_probabilities.sum(axis=1) - 1).abs() > BAD_PROB_THRESHOLD - if no_choices.any(): - print("BAD") - choices, rands = logit.make_choices(base_probabilities, trace_label=trace_label) - if want_logsums: - choices = choices.to_frame('choice') - choices['logsum'] = logsums - return choices, raw_utilities, nested_exp_utilities, nested_utils, nested_probabilities, base_probabilities - - -def simple_simulate_dev(choosers, spec, nest_spec, - skims=None, locals_d=None, - chunk_size=0, custom_chooser=None, - log_alt_losers=False, - want_logsums=False, - estimator=None, - trace_label=None, trace_choice_name=None, trace_column_names=None): - trace_label = tracing.extend_trace_label(trace_label, 'simple_simulate') - assert len(choosers) > 0 - result_list = [] - # segment by person type and pick the right spec for each person type - for i, chooser_chunk, chunk_trace_label in chunk.adaptive_chunked_choosers(choosers, chunk_size, trace_label): - # the following replaces choices = _simple_simulate(...) - if skims is not None: - simulate.set_skim_wrapper_targets(choosers, skims) - - # only do this for nested, logit is straight forward - assert nest_spec is not None - choices, raw_utilities, nested_exp_utilities, nested_utils, \ - nested_probs, base_probs = eval_nl_dev(choosers, spec, nest_spec, locals_d, custom_chooser, - log_alt_losers=log_alt_losers, - want_logsums=want_logsums, estimator=estimator, trace_label=trace_label, - trace_choice_name=trace_choice_name, trace_column_names=trace_column_names) - - result_list.append(choices) - chunk.log_df(trace_label, f'result_list', result_list) - - if len(result_list) > 1: - choices = pd.concat(result_list) - assert len(choices.index == len(choosers.index)) - return choices, raw_utilities, nested_exp_utilities, nested_utils, nested_probs, base_probs - - -def get_stuff(do_these_purposes=None): - #do_these_purposes=['escort'] - """open pipeline and load stuff for mode choice dev assuming model has been run and pipeline.h5 exists""" - resume_after = "trip_scheduling" - model_name = "trip_mode_choice" - chunk_size = 0 # test_mtc means no chunking - - pipeline.open_pipeline(resume_after) - # preload any bulky injectables (e.g. skims) not in pipeline - inject.get_injectable('preload_injectables', None) - pipeline._PIPELINE.rng().begin_step(model_name) - step_name = model_name - args = {} - checkpoint = pipeline.intermediate_checkpoint(model_name) - inject.set_step_args(args) - - trips = inject.get_table('trips') - tours_merged = inject.get_table('tours_merged') - network_los = inject.get_injectable('network_los') - - trace_label = 'trip_mode_choice' - model_settings_file_name = 'trip_mode_choice.yaml' - model_settings = config.read_model_settings(model_settings_file_name) - - logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME') - mode_column_name = 'trip_mode' - - trips_df = trips.to_frame() - print("Running with %d trips", trips_df.shape[0]) - - tours_merged = tours_merged.to_frame() - tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']] - - # - trips_merged - merge trips and tours_merged - trips_merged = pd.merge( - trips_df, - tours_merged, - left_on='tour_id', - right_index=True, - how="left") - assert trips_merged.index.equals(trips.index) - - # setup skim keys - assert ('trip_period' not in trips_merged) - trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart) - - orig_col = 'origin' - dest_col = 'destination' - - constants = {} - constants.update(config.get_model_constants(model_settings)) - constants.update({ - 'ORIGIN': orig_col, - 'DESTINATION': dest_col - }) - - skim_dict = network_los.get_default_skim_dict() - - odt_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=orig_col, dest_key=dest_col, - dim3_key='trip_period') - dot_skim_stack_wrapper = skim_dict.wrap_3d(orig_key=dest_col, dest_key=orig_col, - dim3_key='trip_period') - od_skim_wrapper = skim_dict.wrap('origin', 'destination') - - skims = { - "odt_skims": odt_skim_stack_wrapper, - "dot_skims": dot_skim_stack_wrapper, - "od_skims": od_skim_wrapper, - } - - model_spec = simulate.read_model_spec(file_name=model_settings['SPEC']) - nest_specs = config.get_logit_model_settings(model_settings) - - estimator = estimation.manager.begin_estimation('trip_mode_choice') - - choices_list = [] - raw_util_list = [] - nest_list = [] - nu_list = [] - nest_spec_list = [] - nested_probs_list = [] - base_probs_list = [] - - for primary_purpose, trips_segment in trips_merged.groupby('primary_purpose'): - - if (do_these_purposes is not None) and (primary_purpose not in do_these_purposes): - continue - - print("trip_mode_choice tour_type '%s' (%s trips)" % - (primary_purpose, len(trips_segment.index), )) - - # name index so tracing knows how to slice - assert trips_segment.index.name == 'trip_id' - - coefficients = simulate.get_segment_coefficients(model_settings, primary_purpose) - - locals_dict = {} - locals_dict.update(constants) - locals_dict.update(coefficients) - - segment_trace_label = tracing.extend_trace_label(trace_label, primary_purpose) - - expressions.annotate_preprocessors( - trips_segment, locals_dict, skims, - model_settings, segment_trace_label) - - locals_dict.update(skims) - - ################ Replace wrapper function - # choices = mode_choice_simulate(...) - spec=simulate.eval_coefficients(model_spec, coefficients, estimator) - nest_spec = simulate.eval_nest_coefficients(nest_specs, coefficients, segment_trace_label) - choices, raw_utilities, nested_exp_utilities, nested_utils, nested_probs, base_probs = simple_simulate_dev( - choosers=trips_segment, - spec=spec, - nest_spec=nest_spec, - skims=skims, - locals_d=locals_dict, - chunk_size=chunk_size, - want_logsums=logsum_column_name is not None, - trace_label=segment_trace_label, - trace_choice_name='trip_mode_choice', - estimator=estimator, - trace_column_names=None) - # for consistency, always return dataframe, whether or not logsums were requested - if isinstance(choices, pd.Series): - choices = choices.to_frame('choice') - choices.rename(columns={'logsum': logsum_column_name, - 'choice': mode_column_name}, - inplace=True) - alts = spec.columns - choices[mode_column_name] = choices[mode_column_name].map(dict(list(zip(list(range(len(alts))), alts)))) - ################ - choices_list.append(choices) - raw_util_list.append(raw_utilities) - nest_list.append(nested_exp_utilities) - nu_list.append(nested_utils) - nest_spec_list.append(nest_spec) - nested_probs_list.append(nested_probs) - base_probs_list.append(base_probs) - - choices_df_asim = pd.concat(choices_list) - - # update trips table with choices (and potionally logssums) - trips_df = trips.to_frame() - - if (do_these_purposes is not None): - trips_df = trips_df.loc[trips_df.primary_purpose.isin(do_these_purposes)] - - assign_in_place(trips_df, choices_df_asim) - assert not trips_df[mode_column_name].isnull().any() - - finalise = True - if finalise: - inject.set_step_args(None) - # - pipeline._PIPELINE.rng().end_step(model_name) - pipeline.add_checkpoint(model_name) - if not pipeline.intermediate_checkpoint(): - pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME) - - pipeline.close_pipeline() - - print("Done") - - return trips_df, raw_util_list, nest_list, nu_list, nest_spec_list, nested_probs_list, base_probs_list, \ - dict(list(zip(list(range(len(alts))), alts))) \ No newline at end of file diff --git a/notebooks/larch_probabilities_via_estimation.ipynb b/notebooks/larch_probabilities_via_estimation.ipynb deleted file mode 100644 index 1dc495ed64..0000000000 --- a/notebooks/larch_probabilities_via_estimation.ipynb +++ /dev/null @@ -1,592 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "collapsed": true, - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# Probabilities for three level nests differ\n", - "\n", - "Asim scales nested utilties differently, for two level and RU2 it's identical to what we do (which agrees with the\n", - "larch paper and Zenith), but for three levels there are small differences of probabilities and logsums. I\n", - "quadruple-checked the scaling and leaf and note utility calculation, let's use larch to see if we get the same as\n", - "Asim or if the results agree with my implementation. The idea is to use estimation mode to load a small data set, and\n", - " then use my Asim setup in validate_frozen_impl.ipynb to calculate Asim's and my probabilities by sampling, and check\n", - " what larch has to say by using m.probabilities() (no need to estimate, use parameters as given by asim model).\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "outputs": [], - "source": [ - "%load_ext autoreload" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 2, - "outputs": [], - "source": [ - "%autoreload 2" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 3, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "# I installed larch deps (which I collected in a requirements file, xmle and sharrow are not on pypi)\n", - "# python -m pip install -r ../larch/requirements.txt\n", - "\n", - "larch_path = \"/mnt/c/Users/jan.zill/code/larch\"\n", - "if larch_path not in sys.path:\n", - " sys.path.append(larch_path)\n", - "import larch" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 4, - "outputs": [], - "source": [ - "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_estimation_sf\")\n", - "os.chdir(example_dir)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 5, - "outputs": [], - "source": [ - "# create estimation test example\n", - "#!activitysim create -e example_estimation_sf -d test_estimation_sf" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 6, - "outputs": [], - "source": [ - "# run estimation mode\n", - "#!activitysim run -c configs_estimation/configs -c configs -o output_sf -d data_sf" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 7, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/mnt/c/Users/jan.zill/code/activitysim/activitysim/estimation/larch/simple_simulate.py:92: DtypeWarning: Columns (411) have mixed types.Specify dtype option on import or set low_memory=False.\n", - " chooser_data = _read_csv(chooser_data_file, index_col=values_index_col,)\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "req_data does not request avail_ca or avail_co but it is set and being provided\n", - "problem: chosen-but-not-available (2 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (2 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (1 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (2 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (1 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (2 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (1 issues)\n", - "problem: low-variance-data-co (1 issues)\n", - "problem: chosen-but-not-available (2 issues)\n", - "problem: low-variance-data-co (1 issues)\n" - ] - } - ], - "source": [ - "modelname = \"trip_mode_choice\"\n", - "\n", - "from activitysim.estimation.larch import component_model\n", - "model, data = component_model(modelname, return_data=True)\n", - "# data.coefficients\n", - "# data.spec\n", - "# data.chooser_data\n", - "model.load_data()\n", - "model.doctor(repair_ch_av='-')\n", - "model.loglike()\n", - "# DO NOT ESTIMATE just apply probs\n", - "# model.maximize_loglike(method='SLSQP', options={\"maxiter\": 1000})\n", - "probs = [m.probability() for m in model._k_models]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 8, - "outputs": [], - "source": [ - "probs_0 = pd.DataFrame(probs[0])\n", - "m_0 = model._k_models[0]\n", - "\n", - "probs_0.index = m_0.dataframes.data_ch.index\n", - "\n", - "probs_0 = probs_0.rename(columns={v-1: k for k,v in data.settings[\"CONSTANTS\"][\"I_MODE_MAP\"].items()})" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 9, - "outputs": [ - { - "data": { - "text/plain": "(2124,)" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": "(2124,)" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "display(m_0.dataframes.data_ch.index.unique().shape)\n", - "display(np.intersect1d(data.chooser_data.index.values, m_0.dataframes.data_ch.index.values).shape)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 11, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/mnt/c/Users/jan.zill/code/activitysim/test_estimation_sf\r\n" - ] - } - ], - "source": [ - "!pwd" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 14, - "outputs": [], - "source": [ - "#m_0.graph.to_png(filename=\"/mnt/c/Users/jan.zill/code/activitysim/test_estimation_sf/mtc_test.png\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 15, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'temp' is not defined", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/tmp/ipykernel_24538/1635136243.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mdisplay\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtemp\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", - "\u001B[0;31mNameError\u001B[0m: name 'temp' is not defined" - ] - } - ], - "source": [ - "display(temp)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 17, - "outputs": [], - "source": [ - "subg = m_0.graph.subgraph_from(22)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 23, - "outputs": [ - { - "data": { - "text/plain": "", - "text/html": "" - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subg.to_png(filename=\"/mnt/c/Users/jan.zill/code/activitysim/test_estimation_sf/mtc_mode_choice_subgraph.png\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 22, - "outputs": [ - { - "data": { - "text/plain": "", - "text/html": "
\n\nTree\n\n\ncluster_elemental\n\nElemental Alternatives\n\n\n\n1\n\nDRIVEALONEFREE \n(1)\n\n\n\n2\n\nDRIVEALONEPAY \n(2)\n\n\n\n24\n\nSHAREDRIDE2 \n(24)\n\n\n\n25\n\nSHAREDRIDE3 \n(25)\n\n\n\n22\n\nAUTO \n(22)\n\n\n\n22->24\n\n\n\n\n\n22->25\n\n\n\n\n\n23\n\nDRIVEALONE \n(23)\n\n\n\n22->23\n\n\n\n\n\n23->1\n\n\n\n\n\n23->2\n\n\n\n\n\n0\n\nRoot\n\n\n
" - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "m_0.graph.partial_figure(source=22)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 18, - "outputs": [], - "source": [ - "#data.chooser_data[['override_choice', 'override_choice_code']].drop_duplicates().reset_index(drop=True).set_index\\\n", - "# ('override_choice_code').to_dict()['override_choice']\n", - "#mode_map = pd.DataFrame.from_dict(data.settings[\"CONSTANTS\"][\"I_MODE_MAP\"], orient='index', columns=[\"mode_code\"])\\\n", - "# .reset_index().rename(columns={\"index\": \"mode_name\"})" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 19, - "outputs": [ - { - "data": { - "text/plain": "Int64Index([ 2961920, 2970120, 2998943, 3013252, 3015794, 3021985,\n 3022354, 3025019, 3055523, 3060361,\n ...\n 308028857, 308037836, 308057680, 308057681, 308070308, 308070309,\n 308073875, 308090603, 308105896, 308122624],\n dtype='int64', name='_caseid_', length=2124)" - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "m_0.dataframes.data_ch.index.unique()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 34, - "outputs": [ - { - "data": { - "text/plain": " trip_id model_choice override_choice \\\ntour_id \n2961920 23695361 WALK_LRF WALK_LRF \n2961920 23695365 WALK WALK_LRF \n\n util_DRIVEALONEFREE_Unavailable \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_zero_auto_households \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_persons_less_than_16 \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_for_joint_tours \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_work \\\ntour_id \n2961920 0.0 \n2961920 0.0 \n\n util_DRIVEALONEFREE_In_vehicle_time \\\ntour_id \n2961920 5.36 \n2961920 4.91 \n\n util_DRIVEALONEFREE_Terminal_time ... \\\ntour_id ... \n2961920 10.45084 ... \n2961920 10.45084 ... \n\n drive_heavyrail_available_outbound \\\ntour_id \n2961920 False \n2961920 False \n\n drive_heavyrail_available_inbound drive_commuter_available_outbound \\\ntour_id \n2961920 False False \n2961920 False False \n\n drive_commuter_available_inbound walk_ferry_available \\\ntour_id \n2961920 False False \n2961920 False False \n\n drive_ferry_available distance distance_walk_od distance_bike_od \\\ntour_id \n2961920 False 1.98 1.98 1.98 \n2961920 False 1.96 1.96 1.96 \n\n override_choice_code \ntour_id \n2961920 10 \n2961920 10 \n\n[2 rows x 478 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_idmodel_choiceoverride_choiceutil_DRIVEALONEFREE_Unavailableutil_DRIVEALONEFREE_Unavailable_for_zero_auto_householdsutil_DRIVEALONEFREE_Unavailable_for_persons_less_than_16util_DRIVEALONEFREE_Unavailable_for_joint_toursutil_DRIVEALONEFREE_Unavailable_if_didnt_drive_to_workutil_DRIVEALONEFREE_In_vehicle_timeutil_DRIVEALONEFREE_Terminal_time...drive_heavyrail_available_outbounddrive_heavyrail_available_inbounddrive_commuter_available_outbounddrive_commuter_available_inboundwalk_ferry_availabledrive_ferry_availabledistancedistance_walk_oddistance_bike_odoverride_choice_code
tour_id
296192023695361WALK_LRFWALK_LRF0.00.00.00.00.05.3610.45084...FalseFalseFalseFalseFalseFalse1.981.981.9810
296192023695365WALKWALK_LRF0.00.00.00.00.04.9110.45084...FalseFalseFalseFalseFalseFalse1.961.961.9610
\n

2 rows × 478 columns

\n
" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": " 1 2 3 4 5 6 7 8 9 10 ... 12 13 \\\n_caseid_ ... \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 ... 0.0 0.0 \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 ... 0.0 0.0 \n\n 14 15 16 17 18 19 20 21 \n_caseid_ \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n\n[2 rows x 21 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
12345678910...12131415161718192021
_caseid_
29619200.00.00.00.00.00.00.00.00.01.0...0.00.00.00.00.00.00.00.00.00.0
29619200.00.00.00.00.00.00.00.00.01.0...0.00.00.00.00.00.00.00.00.00.0
\n

2 rows × 21 columns

\n
" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\n_caseid_ \n2961920 0.0 0.0 2.027363e-06 0.0 \n2961920 0.0 0.0 2.943297e-09 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF ... \\\n_caseid_ ... \n2961920 1.061530e-06 0.0 0.111823 0.0 0.263130 0.625043 ... \n2961920 1.967559e-08 0.0 0.119603 0.0 0.272258 0.608139 ... \n\n WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP DRIVE_HVY \\\n_caseid_ \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 \n2961920 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \n_caseid_ \n2961920 0.0 2.489336e-08 4.965881e-08 4.072015e-07 \n2961920 0.0 2.435815e-08 4.996681e-08 4.311721e-07 \n\n[2 rows x 21 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRF...WALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
_caseid_
29619200.00.02.027363e-060.01.061530e-060.00.1118230.00.2631300.625043...0.00.00.00.00.00.00.02.489336e-084.965881e-084.072015e-07
29619200.00.02.943297e-090.01.967559e-080.00.1196030.00.2722580.608139...0.00.00.00.00.00.00.02.435815e-084.996681e-084.311721e-07
\n

2 rows × 21 columns

\n
" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "23695361\n" - ] - }, - { - "data": { - "text/plain": " SHARED2FREE SHARED3FREE WALK WALK_LOC WALK_LRF \\\n_caseid_ \n2961920 2.027363e-06 1.061530e-06 0.111823 0.263130 0.625043 \n2961920 2.943297e-09 1.967559e-08 0.119603 0.272258 0.608139 \n\n TAXI TNC_SINGLE TNC_SHARED \n_caseid_ \n2961920 2.489336e-08 4.965881e-08 4.072015e-07 \n2961920 2.435815e-08 4.996681e-08 4.311721e-07 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SHARED2FREESHARED3FREEWALKWALK_LOCWALK_LRFTAXITNC_SINGLETNC_SHARED
_caseid_
29619202.027363e-061.061530e-060.1118230.2631300.6250432.489336e-084.965881e-084.072015e-07
29619202.943297e-091.967559e-080.1196030.2722580.6081392.435815e-084.996681e-084.311721e-07
\n
" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "tour_id_to_check = 2961920\n", - "\n", - "display(data.chooser_data.loc[data.chooser_data.index == tour_id_to_check])\n", - "display(m_0.dataframes.data_ch.loc[m_0.dataframes.data_ch.index == tour_id_to_check])\n", - "display(probs_0.loc[probs_0.index == tour_id_to_check])\n", - "\n", - "# choose the first trip to look into probabilities below\n", - "trip_id_to_check = data.chooser_data.loc[data.chooser_data.index == tour_id_to_check].trip_id.values[0]\n", - "print(trip_id_to_check)\n", - "res_ = probs_0.loc[(probs_0.index == tour_id_to_check)]\n", - "display(res_.loc[:, (res_ != 0).any(axis=0)])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 46, - "outputs": [], - "source": [ - "data.chooser_data.loc[data.chooser_data.index.isin(m_0.dataframes.data_ch.index)].to_csv(os.path.join(example_dir, \"choosers_larch.csv\"))\n", - "probs_0.to_csv(os.path.join(example_dir, \"probabilities_larch.csv\"))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "# see if larch scales nest scale" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 46, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 46, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 46, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/notebooks/nested_logit_frozen_rand_individ_util.ipynb b/notebooks/nested_logit_frozen_rand_individ_util.ipynb deleted file mode 100644 index fbf4fce6be..0000000000 --- a/notebooks/nested_logit_frozen_rand_individ_util.ipynb +++ /dev/null @@ -1,440 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Frozen randomness\n", - "\n", - "Let's try to draw from nested logit model by drawing error terms. For logit models, this is simple because error terms are independent and therefore we can uniquely invert the CDF and simply draw from that. For nested models, this is not the case. However, we know we can write the probabilities as nested logits and therefore we think we can draw repeatedly like for logit models, taking the nesting structure into account.\n", - "\n", - "Let's start with two levels and a model where we know the probabilities, i.e. we fix the utility functions and the nesting scales, like for red bus/blue bus. We then draw like ActivitySim does, and like we want to do.\n", - "\n", - "Next, we extend to three levels, where the additional nest error term has not been derived yet." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T05:31:19.564033Z", - "iopub.status.busy": "2022-05-01T05:31:19.563663Z", - "iopub.status.idle": "2022-05-01T05:31:19.824207Z", - "shell.execute_reply": "2022-05-01T05:31:19.823432Z", - "shell.execute_reply.started": "2022-05-01T05:31:19.563960Z" - } - }, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "import numpy as np\n", - "from numpy.random import default_rng" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T05:31:20.023960Z", - "iopub.status.busy": "2022-05-01T05:31:20.023246Z", - "iopub.status.idle": "2022-05-01T05:31:20.029538Z", - "shell.execute_reply": "2022-05-01T05:31:20.028264Z", - "shell.execute_reply.started": "2022-05-01T05:31:20.023927Z" - } - }, - "outputs": [], - "source": [ - "def logsum(utilities, nest_scale=1.0):\n", - " scaled_utils = utilities / nest_scale\n", - " max_util = np.max(scaled_utils)\n", - " return max_util + np.log(np.sum(np.exp(scaled_utils - max_util)))\n", - "\n", - "\n", - "# total probability of alternative being chosen is product of two terms:\n", - "# conditional probability of alternative given nest is chosen: exp(util / nest_scale)\n", - "# marginal probability that any alternative in nest is chosen: exp(nest_scale * logsum)\n", - " \n", - "# If you think about a single case, the probabilities are indicator variables and we take the max of each. This is what Zenith does I think.\n", - "# Given that these expressions are those of two logits, it seems natural to draw correspondingly.\n", - "# This must be related to the strict derivation of max() distributions of Hunt." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Two-level" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T09:55:14.957470Z", - "iopub.status.busy": "2022-05-01T09:55:14.957183Z", - "iopub.status.idle": "2022-05-01T09:55:14.964046Z", - "shell.execute_reply": "2022-05-01T09:55:14.963056Z", - "shell.execute_reply.started": "2022-05-01T09:55:14.957441Z" - } - }, - "outputs": [], - "source": [ - "alternatives = {1: \"car\", 2: \"blue bus\", 3: \"red bus\"}\n", - "\n", - "utility_spec = {\n", - " 1: {\"cost\": -1.0, \"asc\": 0.0},\n", - " 2: {\"cost\": -1.0, \"asc\": 0.2},\n", - " 3: {\"cost\": -1.5, \"asc\": 0.1},\n", - "}\n", - "\n", - "# blue and red bus are nested together with scale 0.5\n", - "\n", - "def utility(x, utility_spec, alternative):\n", - " assert alternative in utility_spec.keys()\n", - " return utility_spec[alternative][\"cost\"] * x + utility_spec[alternative][\"asc\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T09:55:15.467608Z", - "iopub.status.busy": "2022-05-01T09:55:15.467366Z", - "iopub.status.idle": "2022-05-01T09:55:15.478279Z", - "shell.execute_reply": "2022-05-01T09:55:15.476833Z", - "shell.execute_reply.started": "2022-05-01T09:55:15.467571Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.4402110749048506 0.5234199037442896 0.036369021350859815\n", - "1.0\n" - ] - } - ], - "source": [ - "cost = 3.0\n", - "nest_scale = 0.6\n", - "\n", - "util_3 = utility(cost, utility_spec, 3)\n", - "util_2 = utility(cost, utility_spec, 2)\n", - "logsum_bus = logsum(np.array([util_2, util_3]), nest_scale=nest_scale)\n", - "nest_util = nest_scale * logsum_bus\n", - "\n", - "util_1 = utility(cost, utility_spec, 1)\n", - "prob_1 = np.exp(util_1) / (np.exp(util_1) + np.exp(nest_util))\n", - "\n", - "nest_cond_prob = np.exp(nest_util) / (np.exp(util_1) + np.exp(nest_util))\n", - "nest_marg_prob_2 = np.exp(util_2 / nest_scale) / (np.exp(util_2 / nest_scale) + np.exp(util_3 / nest_scale))\n", - "nest_marg_prob_3 = np.exp(util_3 / nest_scale) / (np.exp(util_2 / nest_scale) + np.exp(util_3 / nest_scale))\n", - "\n", - "prob_2 = nest_cond_prob * nest_marg_prob_2\n", - "prob_3 = nest_cond_prob * nest_marg_prob_3\n", - "\n", - "print(prob_1, prob_2, prob_3)\n", - "print(sum([prob_1, prob_2, prob_3]))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T09:55:20.523480Z", - "iopub.status.busy": "2022-05-01T09:55:20.523247Z", - "iopub.status.idle": "2022-05-01T09:55:21.341578Z", - "shell.execute_reply": "2022-05-01T09:55:21.340682Z", - "shell.execute_reply.started": "2022-05-01T09:55:20.523453Z" - } - }, - "outputs": [], - "source": [ - "# Asim does this:\n", - "# probs = [prob_1, prob_2, prob_3]\n", - "# cum_probs = [0] + list(np.cumsum(probs))\n", - "#\n", - "# num_draws = 10000000\n", - "#\n", - "# # now draw from U and put into arrays, then value count?\n", - "# rng = default_rng(999)\n", - "# rands = rng.uniform(size=num_draws)\n", - "#\n", - "# hits, bins = np.histogram(rands, bins=cum_probs)\n", - "# print(f\"closed form: {probs},\\nsimulated: {hits / num_draws}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T09:55:24.027745Z", - "iopub.status.busy": "2022-05-01T09:55:24.027421Z", - "iopub.status.idle": "2022-05-01T09:55:24.034035Z", - "shell.execute_reply": "2022-05-01T09:55:24.031703Z", - "shell.execute_reply.started": "2022-05-01T09:55:24.027708Z" - } - }, - "outputs": [], - "source": [ - "def inverse_ev1_cdf(x, location=0.0, scale=1.0):\n", - " \"quantile function of EV1\"\n", - " #return location - (1.0 / scale) * np.log(-np.log(x))\n", - " # let's follow https://en.wikipedia.org/wiki/Gumbel_distribution where the scale is proportional to variance (not variance^{-1})\n", - " return location - scale * np.log(-np.log(x))\n", - "\n", - "# for utilities with full set of ascs location=0. Do we always assume location=0 in estimation anyways?\n", - "# the scale of the error term is unidentified and therefore set to 1 in most applications, meaning the standard deviation is pi/sqrt(6)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-17T03:38:03.583638Z", - "iopub.status.busy": "2022-04-17T03:38:03.583273Z", - "iopub.status.idle": "2022-04-17T03:38:03.588839Z", - "shell.execute_reply": "2022-04-17T03:38:03.586978Z", - "shell.execute_reply.started": "2022-04-17T03:38:03.583594Z" - } - }, - "source": [ - "## The Zenith way\n", - "\n", - "Basically, probabilities are now indicators - choose at the lowest level, then walk up. Choice is product of these. We draw error terms for each alternative, where nest roots are now alternatives too.\n", - "\n", - "OR: do we choose a" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:03:23.999857Z", - "iopub.status.busy": "2022-05-01T10:03:23.999571Z", - "iopub.status.idle": "2022-05-01T10:03:30.033848Z", - "shell.execute_reply": "2022-05-01T10:03:30.032689Z", - "shell.execute_reply.started": "2022-05-01T10:03:23.999816Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "closed form probs: 0.440211, 0.523420, 0.036369\n", - " simulated probs: 0.4403354, 0.5233336, 0.036331\n", - "CPU times: user 3.88 s, sys: 2.52 s, total: 6.39 s\n", - "Wall time: 6.73 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "# conditional error term are given by logit with scale given by nest scale\n", - "num_draws_dec = 10000000\n", - "#mu = 1.0 / nest_scale\n", - "\n", - "util_3 = utility(cost, utility_spec, 3)\n", - "util_2 = utility(cost, utility_spec, 2)\n", - "logsum_bus = logsum(np.array([util_2, util_3]), nest_scale=nest_scale)\n", - "util_1 = utility(cost, utility_spec, 1)\n", - "\n", - "\n", - "rng_dec = default_rng(9)\n", - "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", - "ev1_lower = inverse_ev1_cdf(rands_dec)\n", - "\n", - "lower_utils_2 = util_2 / nest_scale + ev1_lower[num_draws_dec:]\n", - "lower_utils_3 = util_3 / nest_scale + ev1_lower[:num_draws_dec]\n", - "\n", - "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec))\n", - "nest_util = nest_scale * logsum_bus + ev1_upper\n", - "\n", - "ev1_upper = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec))\n", - "full_util_1 = util_1 + ev1_upper\n", - "\n", - "choices = np.array([full_util_1, nest_util]).argmax(axis=0)\n", - "nest_indexes = np.nonzero(choices == 1)[0]\n", - "nest_choices = np.array([lower_utils_2[nest_indexes], lower_utils_3[nest_indexes]]).argmax(axis=0)\n", - "nest_choices += 1\n", - "choices = np.append(choices[choices == 0], nest_choices)\n", - "\n", - "vals, counts = np.unique(choices, return_counts=True)\n", - "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", - "\n", - "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", - "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## The error term decomposition way -> not working. just use the zenith way and write it as indicators for individuals.\n", - "\n", - "We can decompose the error term into one for the nest and one within nests according to Hildebrandt. However, I\n", - "cannot seem to reproduce the exact probabilities. Why?\n", - "\n", - "Looks like one of the location parameters is wrong; 0.125 added to nest makes it right (tested for one set of params)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T09:55:25.164028Z", - "iopub.status.busy": "2022-05-01T09:55:25.163822Z", - "iopub.status.idle": "2022-05-01T09:55:25.473704Z", - "shell.execute_reply": "2022-05-01T09:55:25.472525Z", - "shell.execute_reply.started": "2022-05-01T09:55:25.164005Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "closed form probs: 0.445227, 0.533045, 0.021728\n", - " simulated probs: 0.4755317, 0.5039483, 0.02052\n", - "0.9363, 1.0577, 1.0589\n", - "CPU times: user 4.12 s, sys: 2.3 s, total: 6.42 s\n", - "Wall time: 6.99 s\n" - ] - } - ], - "source": [ - "%%time\n", - "# conditional error term are given by logit with scale given by nest scale\n", - "num_draws_dec = 10000000\n", - "\n", - "mu = 1.0 / nest_scale\n", - "\n", - "rng_dec = default_rng(9)\n", - "\n", - "rands_dec = rng_dec.uniform(size = 2 * num_draws_dec) # we need one for each alternative if num_draws_dec signifies the total number of choices we want to simulate\n", - "ev1_dec = inverse_ev1_cdf(rands_dec, scale=1.0/mu)\n", - "lower_level_utils_2 = np.repeat(util_2, num_draws_dec) + ev1_dec[num_draws_dec:]\n", - "lower_level_utils_3 = np.repeat(util_3, num_draws_dec) + ev1_dec[:num_draws_dec]\n", - "\n", - "#location_nest = - 1.0 / mu * np.log(2.0 * np.exp(mu * 0.5772))\n", - "location_nest = - np.log(2.0) / mu\n", - "#location_nest = (- np.log(2.0) / mu) - ((1.0 - 1.0 / (mu + 1.0)) * 0.57721 * mu / (mu**2 - 1.0))\n", - "#print(location_nest, - np.log(2.0) / mu)\n", - "\n", - "scale_nest = mu / np.sqrt(mu**2 - 1.0)\n", - "nest_error_terms = inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec), location=location_nest, scale=1.0/scale_nest)\n", - "\n", - "full_utils_2 = lower_level_utils_2 + nest_error_terms\n", - "full_utils_3 = lower_level_utils_3 + nest_error_terms\n", - "\n", - "# what's the distribution of error term for car? it's a degenerate nest, so bottom level is 1\n", - "# this here agrees with Bhat and Koppelman's mode choice script.\n", - "full_utils_1 = util_1 + inverse_ev1_cdf(rng_dec.uniform(size=num_draws_dec))\n", - "\n", - "choices = np.array([full_utils_1, full_utils_2, full_utils_3]).argmax(axis=0)\n", - "vals, counts = np.unique(choices, return_counts=True)\n", - "probs_dec = {i+1: counts[i] / num_draws_dec for i in vals}\n", - "\n", - "print(f\"closed form probs: {prob_1:.6f}, {prob_2:.6f}, {prob_3:.6f}\")\n", - "print(f\" simulated probs: {probs_dec[1]}, {probs_dec[2]}, {probs_dec[3]}\")\n", - "print(f\"{prob_1 / probs_dec[1]:.4f}, {prob_2 / probs_dec[2]:.4f}, {prob_3 / probs_dec[3]:.4f}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Three-level\n", - "\n", - "and recursive - maybe use Asim structure directly?\n", - "\n", - "\n", - "Could also use larch to apply models, would be great to add there too?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file diff --git a/notebooks/validate_frozen_impl.ipynb b/notebooks/validate_frozen_impl.ipynb deleted file mode 100644 index cf89790669..0000000000 --- a/notebooks/validate_frozen_impl.ipynb +++ /dev/null @@ -1,1651 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7ece3567-e4b1-4c3f-a264-20625abb6ad7", - "metadata": {}, - "source": [ - "# validate results\n", - "\n", - "## TODO\n", - "what happened to tracing when I fixed probability calcs - it seems like all trip ids are attached when I add one by\n", - "hand below - why?" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "bd74ba44-0dfb-439a-a6ab-7ceedfc5f523", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.385153Z", - "iopub.status.busy": "2022-05-01T10:17:57.384881Z", - "iopub.status.idle": "2022-05-01T10:17:57.534433Z", - "shell.execute_reply": "2022-05-01T10:17:57.533096Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.385047Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%load_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "c3dba451-1e10-403e-8614-35d57e6577f4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:17:57.536623Z", - "iopub.status.busy": "2022-05-01T10:17:57.536012Z", - "iopub.status.idle": "2022-05-01T10:17:57.542755Z", - "shell.execute_reply": "2022-05-01T10:17:57.541685Z", - "shell.execute_reply.started": "2022-05-01T10:17:57.536567Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "73deaac4-e7ac-4aff-b086-4980dc6dd903", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:00.350944Z", - "iopub.status.busy": "2022-05-01T10:18:00.350730Z", - "iopub.status.idle": "2022-05-01T10:18:12.760977Z", - "shell.execute_reply": "2022-05-01T10:18:12.760013Z", - "shell.execute_reply.started": "2022-05-01T10:18:00.350919Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "import argparse\n", - "from datetime import datetime\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from numpy.random import default_rng\n", - "from statsmodels.stats.gof import powerdiscrepancy\n", - "from statsmodels.stats.proportion import multinomial_proportions_confint\n", - "import numba\n", - "import matplotlib.pyplot as plt\n", - "\n", - "from activitysim.cli import run\n", - "from activitysim.core import inject\n", - "\n", - "from fru_utils import run_trip_mode_choice, comp_mode_shares, mode_choice_for_trip" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "7e553abd-fe0d-4cdc-aeb1-9dc80cb2757f", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:16.944537Z", - "iopub.status.busy": "2022-05-01T10:18:16.944291Z", - "iopub.status.idle": "2022-05-01T10:18:17.124764Z", - "shell.execute_reply": "2022-05-01T10:18:17.123725Z", - "shell.execute_reply.started": "2022-05-01T10:18:16.944501Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "pd.set_option(\"max_columns\", 500)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "41fec4e8-a174-4e66-87d2-1e8c7979de90", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:17.839947Z", - "iopub.status.busy": "2022-05-01T10:18:17.839070Z", - "iopub.status.idle": "2022-05-01T10:18:18.019676Z", - "shell.execute_reply": "2022-05-01T10:18:18.018689Z", - "shell.execute_reply.started": "2022-05-01T10:18:17.839911Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "root_dir = \"/mnt/c/Users/jan.zill/code/activitysim\"\n", - "example_dir = os.path.join(root_dir, \"test_example_mtc\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2de710a2-d292-42f9-9d4a-4dcef1365506", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:18.894533Z", - "iopub.status.busy": "2022-05-01T10:18:18.894303Z", - "iopub.status.idle": "2022-05-01T10:18:19.078807Z", - "shell.execute_reply": "2022-05-01T10:18:19.077951Z", - "shell.execute_reply.started": "2022-05-01T10:18:18.894508Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "os.chdir(example_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "outputs": [], - "source": [ - "from activitysim.core import pipeline\n", - "pipeline.close_pipeline()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "730be239-8704-4483-bbb8-ffae0f17c5d4", - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T10:18:22.111723Z", - "iopub.status.busy": "2022-05-01T10:18:22.111490Z", - "iopub.status.idle": "2022-05-01T10:18:22.297437Z", - "shell.execute_reply": "2022-05-01T10:18:22.296501Z", - "shell.execute_reply.started": "2022-05-01T10:18:22.111697Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Configured logging using basicConfig\n", - "INFO - Configured logging using basicConfig\n", - "INFO - Read logging configuration from: configs/logging.yaml\n", - "INFO - SETTING configs_dir: ['configs']\n", - "INFO - SETTING settings_file_name: settings.yaml\n", - "INFO - SETTING data_dir: ['data']\n", - "INFO - SETTING output_dir: output\n", - "INFO - SETTING households_sample_size: 100\n", - "INFO - SETTING chunk_size: 0\n", - "INFO - SETTING chunk_method: hybrid_uss\n", - "INFO - SETTING chunk_training_mode: disabled\n", - "INFO - SETTING multiprocess: None\n", - "INFO - SETTING num_processes: None\n", - "INFO - SETTING resume_after: None\n", - "INFO - SETTING trace_hh_id: [982875]\n", - "INFO - ENV MKL_NUM_THREADS: None\n", - "INFO - ENV OMP_NUM_THREADS: None\n", - "INFO - ENV OPENBLAS_NUM_THREADS: None\n", - "INFO - NUMPY blas_opt_info libraries: ['openblas', 'openblas']\n", - "INFO - NUMPY lapack_opt_info libraries: ['openblas', 'openblas']\n", - "INFO - run single process simulation\n", - "INFO - Time to execute open_pipeline : 0.163 seconds (0.0 minutes)\n", - "INFO - Time to execute preload_injectables : 0.08 seconds (0.0 minutes)\n", - "INFO - #run_model running step initialize_landuse\n", - "Running step 'initialize_landuse'\n", - "INFO - Time to execute all models until this error : 0.393 seconds (0.0 minutes)\n", - "ERROR - activitysim run encountered an unrecoverable error\n", - "Traceback (most recent call last):\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/cli/run.py\", line 261, in run\n", - " pipeline.run(models=config.setting('models'), resume_after=resume_after)\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\", line 617, in run\n", - " run_model(model)\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\", line 476, in run_model\n", - " orca.run([step_name])\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\", line 2168, in run\n", - " step()\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\", line 973, in __call__\n", - " return self._func(**kwargs)\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/initialize.py\", line 100, in initialize_landuse\n", - " with chunk.chunk_log(trace_label, base=True):\n", - " File \"/home/jan/.pyenv/versions/3.8.2/lib/python3.8/contextlib.py\", line 113, in __enter__\n", - " return next(self.gen)\n", - " File \"/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/chunk.py\", line 929, in chunk_log\n", - " assert base == (len(CHUNK_SIZERS) == 0)\n", - "AssertionError\n" - ] - }, - { - "ename": "AssertionError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mAssertionError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/tmp/ipykernel_2932/4273387002.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 2\u001B[0m \u001B[0mrun\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0madd_run_args\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mparser\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[0margs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mparser\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mparse_args\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m'-c'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'configs'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'-o'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'output'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'-d'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m'data'\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m \u001B[0mrun\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;31m# 2mins full example run\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 5\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0minject\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mis_injectable\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'preload_injectables'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 6\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mactivitysim\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mabm\u001B[0m \u001B[0;31m# register abm steps and other abm-specific injectables\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/cli/run.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(args)\u001B[0m\n\u001B[1;32m 259\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'run single process simulation'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 260\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 261\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodels\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetting\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'models'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mresume_after\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 262\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 263\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetting\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'cleanup_pipeline_after_run'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(models, resume_after)\u001B[0m\n\u001B[1;32m 615\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mmodel\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mmodels\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 616\u001B[0m \u001B[0mt1\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mprint_elapsed_time\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 617\u001B[0;31m \u001B[0mrun_model\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodel\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 618\u001B[0m \u001B[0mmem\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrace_memory_info\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"pipeline.run after {model}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 619\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mrun_model\u001B[0;34m(model_name)\u001B[0m\n\u001B[1;32m 474\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"#run_model running step {step_name}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 475\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 476\u001B[0;31m \u001B[0morca\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mstep_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 477\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 478\u001B[0m \u001B[0mt0\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mprint_elapsed_time\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"#run_model completed step '%s'\"\u001B[0m \u001B[0;34m%\u001B[0m \u001B[0mmodel_name\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mt0\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdebug\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(steps, iter_vars, data_out, out_interval, out_base_tables, out_run_tables, compress, out_base_local, out_run_local)\u001B[0m\n\u001B[1;32m 2166\u001B[0m \u001B[0mstep\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mget_step\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mstep_name\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 2167\u001B[0m \u001B[0mt2\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 2168\u001B[0;31m \u001B[0mstep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 2169\u001B[0m print(\"Time to execute step '{}': {:.2f} s\".format(\n\u001B[1;32m 2170\u001B[0m step_name, time.time() - t2))\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/orca/orca.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 971\u001B[0m kwargs = _collect_variables(names=self._argspec.args,\n\u001B[1;32m 972\u001B[0m expressions=self._argspec.defaults)\n\u001B[0;32m--> 973\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_func\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 974\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 975\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0m_tables_used\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/abm/models/initialize.py\u001B[0m in \u001B[0;36minitialize_landuse\u001B[0;34m()\u001B[0m\n\u001B[1;32m 98\u001B[0m \u001B[0mtrace_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m'initialize_landuse'\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 99\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 100\u001B[0;31m \u001B[0;32mwith\u001B[0m \u001B[0mchunk\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchunk_log\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrace_label\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbase\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 101\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 102\u001B[0m \u001B[0mmodel_settings\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mconfig\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mread_model_settings\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'initialize_landuse.yaml'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmandatory\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m~/.pyenv/versions/3.8.2/lib/python3.8/contextlib.py\u001B[0m in \u001B[0;36m__enter__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 111\u001B[0m \u001B[0;32mdel\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkwds\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfunc\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 112\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 113\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mnext\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgen\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 114\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mStopIteration\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 115\u001B[0m \u001B[0;32mraise\u001B[0m \u001B[0mRuntimeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"generator didn't yield\"\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/chunk.py\u001B[0m in \u001B[0;36mchunk_log\u001B[0;34m(trace_label, chunk_tag, base)\u001B[0m\n\u001B[1;32m 927\u001B[0m \u001B[0;31m# avoids breaking the assertion below.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 928\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 929\u001B[0;31m \u001B[0;32massert\u001B[0m \u001B[0mbase\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mCHUNK_SIZERS\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 930\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 931\u001B[0m \u001B[0mtrace_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34mf\"{trace_label}.chunk_log\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mAssertionError\u001B[0m: " - ] - } - ], - "source": [ - "parser = argparse.ArgumentParser()\n", - "run.add_run_args(parser)\n", - "args = parser.parse_args(['-c', 'configs', '-o', 'output', '-d', 'data'])\n", - "run.run(args) # 2mins full example run\n", - "if not inject.is_injectable('preload_injectables'):\n", - " from activitysim import abm # register abm steps and other abm-specific injectables\n", - "run.handle_standard_args(args) # possibly update injectables" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "outputs": [], - "source": [ - "## need to do this before loading checkpoint tables\n", - "#trace_hh_ids = 1024353\n", - "##1024353 has trip id 642446345 (at least) where logsum and probs are different\n", - "##2821179 # has trip_ids 2464104881 and 2464104885\n", - "#inject.add_injectable(\"trace_hh_id\", trace_hh_ids)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 9, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Couldn't find checkpoint 'trip_scheduling' in checkpoints\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 init\n", - "1 initialize_landuse\n", - "2 initialize_households\n", - "3 compute_accessibility\n", - "4 school_location\n", - "Name: checkpoint_name, dtype: object\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Couldn't find checkpoint 'trip_scheduling' in checkpoints", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mIndexError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mload_checkpoint\u001B[0;34m(checkpoint_name)\u001B[0m\n\u001B[1;32m 353\u001B[0m \u001B[0;31m# truncate rows after target checkpoint\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 354\u001B[0;31m \u001B[0mi\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcheckpoints\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mcheckpoints\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mCHECKPOINT_NAME\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0mcheckpoint_name\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 355\u001B[0m \u001B[0mcheckpoints\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcheckpoints\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mloc\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0mi\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/.venv/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001B[0m in \u001B[0;36m__getitem__\u001B[0;34m(self, key)\u001B[0m\n\u001B[1;32m 4603\u001B[0m \u001B[0mkey\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mcom\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcast_scalar_indexer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mwarn_float\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 4604\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mgetitem\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 4605\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mIndexError\u001B[0m: index 0 is out of bounds for axis 0 with size 0", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001B[0;31mRuntimeError\u001B[0m Traceback (most recent call last)", - "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/notebooks/fru_utils.py\u001B[0m in \u001B[0;36mrun_trip_mode_choice\u001B[0;34m(do_these_purposes, choose_individual_max_utility)\u001B[0m\n\u001B[1;32m 190\u001B[0m \u001B[0mchunk_size\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m0\u001B[0m \u001B[0;31m# test_mtc means no chunking\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 191\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 192\u001B[0;31m \u001B[0mpipeline\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mopen_pipeline\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 193\u001B[0m \u001B[0;31m# preload any bulky injectables (e.g. skims) not in pipeline\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 194\u001B[0m \u001B[0minject\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mget_injectable\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'preload_injectables'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mopen_pipeline\u001B[0;34m(resume_after)\u001B[0m\n\u001B[1;32m 513\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdebug\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"open_pipeline - open existing pipeline\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 514\u001B[0m \u001B[0mopen_pipeline_store\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moverwrite\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 515\u001B[0;31m \u001B[0mload_checkpoint\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mresume_after\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 516\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[0;31m# open new, empty pipeline\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/mnt/c/Users/jan.zill/code/activitysim/activitysim/core/pipeline.py\u001B[0m in \u001B[0;36mload_checkpoint\u001B[0;34m(checkpoint_name)\u001B[0m\n\u001B[1;32m 362\u001B[0m \u001B[0mprint\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcheckpoints\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mCHECKPOINT_NAME\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 363\u001B[0m \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0merror\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 364\u001B[0;31m \u001B[0;32mraise\u001B[0m \u001B[0mRuntimeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 365\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 366\u001B[0m \u001B[0;31m# convert pandas dataframe back to array of checkpoint dicts\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mRuntimeError\u001B[0m: Couldn't find checkpoint 'trip_scheduling' in checkpoints" - ] - } - ], - "source": [ - "%%time\n", - "trips_df_ru = run_trip_mode_choice()\n", - "trips_df = run_trip_mode_choice(choose_individual_max_utility=False)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "3ed520e9-75ef-4eec-ab86-6aaf1fe453fc", - "metadata": { - "execution": { - "iopub.execute_input": "2021-09-01T02:28:53.643404Z", - "iopub.status.busy": "2021-09-01T02:28:53.643055Z", - "iopub.status.idle": "2021-09-01T02:29:01.223951Z", - "shell.execute_reply": "2021-09-01T02:29:01.222865Z", - "shell.execute_reply.started": "2021-09-01T02:28:53.643367Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "setting trace_hh_id is wrong type, should be an int, but was \n", - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'atwork' (27 trips)\n", - "trip_mode_choice tour_type 'eatout' (33 trips)\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", - "trip_mode_choice tour_type 'othmaint' (46 trips)\n", - "trip_mode_choice tour_type 'school' (37 trips)\n", - "trip_mode_choice tour_type 'shopping' (77 trips)\n", - "trip_mode_choice tour_type 'social' (19 trips)\n", - "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n", - "Running with %d trips 482\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'atwork' (27 trips)\n", - "trip_mode_choice tour_type 'eatout' (33 trips)\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", - "trip_mode_choice tour_type 'othmaint' (46 trips)\n", - "trip_mode_choice tour_type 'school' (37 trips)\n", - "trip_mode_choice tour_type 'shopping' (77 trips)\n", - "trip_mode_choice tour_type 'social' (19 trips)\n", - "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n", - "CPU times: user 18.3 s, sys: 2.3 s, total: 20.6 s\n", - "Wall time: 17.8 s\n" - ] - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 10, - "outputs": [ - { - "data": { - "text/plain": " trip_mode_fru trip_mode_asim\nWALK 309 304\nWALK_LOC 85 85\nWALK_LRF 55 53\nBIKE 15 17\nTNC_SINGLE 7 4\nDRIVEALONEFREE 5 4\nSHARED2FREE 3 4\nSHARED3FREE 2 1\nWALK_HVY 1 4", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_mode_frutrip_mode_asim
WALK309304
WALK_LOC8585
WALK_LRF5553
BIKE1517
TNC_SINGLE74
DRIVEALONEFREE54
SHARED2FREE34
SHARED3FREE21
WALK_HVY14
\n
" - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c_ = [\"trip_mode\", \"mode_choice_logsum\"]\n", - "trips = trips_df_ru[c_].merge(trips_df[c_], left_index=True, right_index=True, how=\"outer\", suffixes=[\"_fru\", \"_asim\"])\n", - "pd.merge(trips_df_ru.trip_mode.value_counts(), trips_df.trip_mode.value_counts(), left_index=True, right_index=True,\n", - " suffixes=[\"_fru\", \"_asim\"])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 11, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logsums agree\n" - ] - } - ], - "source": [ - "if np.allclose(trips.mode_choice_logsum_fru, trips.mode_choice_logsum_asim):\n", - " print(\"Logsums agree\")\n", - "else:\n", - " ((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru).hist(bins=50)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 12, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.000000e+00\n" - ] - } - ], - "source": [ - "x_ = np.abs((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim)).values\n", - "print(f\"{x_.max():e}\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 13, - "outputs": [], - "source": [ - "#ls_ = np.abs(((trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) / trips.mode_choice_logsum_fru))\n", - "#ls_.sort_values(ascending=False)\n", - "#trips.loc[trips.index == 642446345]\n", - "#trips.loc[np.abs(trips.mode_choice_logsum_fru - trips.mode_choice_logsum_asim) > 0]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "# Run MC validation\n", - "\n", - "see logit._each_nest parent_nest_scale for leaf node, and simulate.compute_nested_utilities for discussion of scales\n", - "of alternatives\n", - "\n", - "easiest way is to run a simulation" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 14, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running with %d trips 482\n", - "trip_mode_choice tour_type 'atwork' (27 trips)\n", - "trip_mode_choice tour_type 'eatout' (33 trips)\n", - "trip_mode_choice tour_type 'escort' (6 trips)\n", - "trip_mode_choice tour_type 'othdiscr' (43 trips)\n", - "trip_mode_choice tour_type 'othmaint' (46 trips)\n", - "trip_mode_choice tour_type 'school' (37 trips)\n", - "trip_mode_choice tour_type 'shopping' (77 trips)\n", - "trip_mode_choice tour_type 'social' (19 trips)\n", - "trip_mode_choice tour_type 'univ' (26 trips)\n", - "trip_mode_choice tour_type 'work' (168 trips)\n", - "Done\n" - ] - } - ], - "source": [ - "from fru_utils import get_stuff\n", - "t, ru, neu, nu, ns, nested_probs, base_probs = get_stuff() #do_these_purposes=[\"social\"]) #social\n", - "bp = pd.concat(base_probs)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 107, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'social' (10000 trips)\n", - "Done\n", - "CPU times: user 8.39 s, sys: 453 ms, total: 8.84 s\n", - "Wall time: 9.28 s\n" - ] - } - ], - "source": [ - "%%time\n", - "#prun\n", - "\n", - "num_samples = 10000\n", - "trip_id_to_check = 642446345 # 1e5 takes about 25s EDIT: BEFORE APPLY, now it's more like 65s and 1e4 takes 8s\n", - "# work 615236801 also in trace log\n", - "# social 2464104885 642446345 1767182945\n", - "\n", - "\n", - "#bp = base_probs[7] # 7 for social, 9 for work. Check t[t.index == trip_id_to_check] to see which purpose, and\n", - "# output above to see which index\n", - "# choose_individual_max_utility = False\n", - "#ms_comp = comp_mode_shares(bp, choose_individual_max_utility, num_samples, trip_id_to_check)\n", - "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)\n", - "\n", - "choose_individual_max_utility = True\n", - "t_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=choose_individual_max_utility,\n", - " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", - "sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", - "ms_comp = comp_mode_shares(bp, sim_mode_shares, trip_id_to_check)\n", - "#display(ms_comp.loc[((ms_comp.mode_share_obs != 0) | (ms_comp.trip_mode != 0)), [\"mode_share_obs\", \"trip_mode\"]].T)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 64, - "outputs": [ - { - "data": { - "text/plain": "(0.3439655414093617, nan)" - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Note power tests will be inf when expected count is zero (any non-zero count will be impossible)\n", - "eps = 0 #1e-14\n", - "non_zero_shares = ms_comp.loc[(ms_comp.mode_share_obs > eps) & (ms_comp.trip_mode > eps)][[\"mode_share_obs\", \"trip_mode\"]]\n", - "ddof = non_zero_shares.shape[0] - 1 # what to use here? num alternatives - 1? What about zero expected count?\n", - "\n", - "non_zero_shares.trip_mode *= num_samples\n", - "\n", - "sum_ms = non_zero_shares.mode_share_obs.sum()\n", - "if sum_ms != 1.0:\n", - " if np.abs(1.0 - sum_ms) > 1e-1:\n", - " print(f\"NOT GOOD {sum_ms} should be one\")\n", - " #print(f\"sum ms is not one but {sum_ms}, re-normalising\")\n", - " non_zero_shares.mode_share_obs /= sum_ms\n", - "\n", - "powerdiscrepancy(non_zero_shares.trip_mode.values, non_zero_shares.mode_share_obs.values, lambd=0, ddof=ddof)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 99, - "outputs": [ - { - "data": { - "text/plain": "array([[4.17345080e-03, 5.98926219e-03, 5.00000000e-03],\n [1.45406892e-04, 6.18851051e-04, 3.00000000e-04],\n [4.13473933e-02, 4.66025328e-02, 4.39000000e-02],\n [5.64412252e-02, 6.25036058e-02, 5.94000000e-02],\n [8.87346329e-01, 8.95325012e-01, 8.91400000e-01]])" - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from statsmodels.stats.proportion import multinomial_proportions_confint\n", - "ci = multinomial_proportions_confint(non_zero_shares.trip_mode.values, alpha=0.999, method='goodman')\n", - "vals = non_zero_shares.trip_mode.values / num_samples\n", - "np.hstack((ci, vals.reshape((len(vals),1))))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 106, - "outputs": [ - { - "data": { - "text/plain": "array([ True, True, True, True, True])" - }, - "execution_count": 106, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numba\n", - "\n", - "@numba.jit\n", - "def within(lower, upper, x):\n", - " return (x >= lower) & (x <= upper)\n", - "\n", - "@numba.jit\n", - "def numpy_within(x):\n", - " lower = x[0]\n", - " upper = x[1]\n", - " val = x[2]\n", - " return within(lower, upper, val)\n", - "\n", - "test = np.hstack((ci, vals.reshape((len(vals),1))))\n", - "np.apply_along_axis(numpy_within, 1, test)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "# All trips for paper\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "%%time\n", - "\n", - "num_samples = 10000 #about 8.5s per 10k per trip, so about 70mins for 10k sample. 1e5 is about 10 times as long, so\n", - "# about 11.5h\n", - "\n", - "trips_to_check = bp.index.values #[100:103]\n", - "\n", - "res = []\n", - "\n", - "for trip_id_to_check in trips_to_check:\n", - " _, c_, _ = mode_choice_for_trip(choose_individual_max_utility=True,\n", - " trip_id_to_check=trip_id_to_check, num_samples=num_samples)\n", - " sim_mode_shares = c_.trip_mode.value_counts() / c_.shape[0]\n", - " res.append(sim_mode_shares.rename(trip_id_to_check).to_frame().T)\n", - "\n", - "result_sim = pd.concat(res).fillna(0)\n", - "result_sim.to_csv(\"temp_res_sim_10ksamples.csv\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": true - } - } - }, - { - "cell_type": "code", - "execution_count": 158, - "outputs": [], - "source": [ - "#result_sim.merge(bp, left_index=True, right_index=True, how=\"outer\", suffixes=[\"_sim\", \"_asim\"]).fillna(0)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 155, - "outputs": [], - "source": [ - "merged_results = result_sim.reset_index().rename(columns={\"index\": \"trip_id\"}).melt(id_vars=[\"trip_id\"]).merge(\n", - " bp.reset_index().rename(columns={\"index\": \"trip_id\"}).melt(id_vars=[\"trip_id\"]),\n", - " on=[\"trip_id\", \"variable\"],\n", - " how=\"outer\",\n", - " suffixes=[\"_sim\", \"_obs\"]\n", - ").fillna(0)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 302, - "outputs": [], - "source": [ - "# should use only those without zero probability - they are really not available so not a valid choice\n", - "merged_results_wo_zeros = merged_results.loc[(merged_results.value_sim != 0) & (merged_results.value_obs != 0)]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 303, - "outputs": [ - { - "data": { - "text/plain": "" - }, - "execution_count": 303, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA19UlEQVR4nO3de3iU9Znw8e89ySST8zkcQiAEgkA4G89Kq1TX+iKotda6tbWvlmu3a6W173ZdW7Vit8etVla7lVbb6u5WtK4WWbVdgRZtxRIUOYkkBBIScj5MDjOTOTy/94/JjAkJMEgmp7k/1+XlPM88k/k9QOae3+m+xRiDUkqp2GUb7QYopZQaXRoIlFIqxmkgUEqpGKeBQCmlYpwGAqWUinHxo92AjyI3N9cUFRWNdjOUUmpc2bVrV4sxJu/E8+MyEBQVFVFeXj7azVBKqXFFRKqHOq9DQ0opFeM0ECilVIzTQKCUUjFOA4FSSsU4DQRKKRXjorpqSESeAlYCTcaYBUM8L8CjwDWAC7jNGPNONNuklFLjTYfbw6GGHho7e5mUnkhGUhyTMhxkJjmG5edHe/nor4DHgKdP8vwngZK+/y4A/r3v/0oppQgGge0ftFLZ3I1loLKpi1l5qXS4vJwzhWEJBlENBMaY7SJSdIpLVgNPm2Au7B0ikikiU4wx9dFsl1JKjRdHmlzUdbjZsL0Kj8/CYbexdkUJhVlJHGro4fyZZx8IRnuOoAA41u+4tu/cICKyRkTKRaS8ubl5RBqnlFKjxbIMVc3ddPX6eXRLBR6fBYDHZ/Holgq6ev00dvYOy3uNm53FxpgNwAaAsrIyraajlJpwLMtQ09ZDU1cv9U4PVc3dzJuSHg4CIR6fhcsbYFJ64rC872gHgjqgsN/xtL5zSikVUzweP/sbOqlq6eG+3+0LDwP91x0X4LDbBgQDh91GXmoisyYlD8t7j/bQ0Cbg8xJ0IeDU+QGlVCzx+y0OHO/gL0dbqXd6wkEAgt/8uzxuvnPdAhz24Me1w27jO9ctYNak5PGxakhEfgN8HMgVkVrgAcAOYIz5GfAKwaWjlQSXj34xmu1RSqmxxO+3eHV/PZ0ePw9tPsAdlxUPGgb6wi938+raC3n6i+fT2BVcPjpncsqwBQGI/qqhz57meQP8QzTboJRSY5HH42dPvRMQCrOSuGhmNsCQw0Cv7mth7uR0/s/CKdhsMuxtGe05AqWUihmWZTja2kN9h5tev8WDm/dT3erGYbfx4KpS3j7cwl1XlLB+a0V4juCh1Qs4d3omRbmpUQkCoIFAKaWiLrQa6J2aDu59cW/4Q/6uK0p4Zkc19U4PD2zazxO3nss9L+xlzfJi5k5OY2qGg/mTM0hIiItq+0Z7slgppSY0yzK8tr+B/363LhwEIDgRvH5rBTcsmxY+7nD5aHd5mZmTwhUl+SyZnh31IAAaCJRSKmr8fot3atq5+7ndWIYh9wNI32iPw25jcrqDzXdeyuolBTgcIzdgo0NDSik1zCzLcKS5mzqnB6fbxx2XFZOaGDfkRLAxwf+vW7WAxVMzRjQAhEhw4c74UlZWZrRmsVJqrPH7LfYfd9LQ1Uun2zdgY9jXPjEHR7yN7712cMBEcEaynawkOwunRD8IiMguY0zZiee1R6CUUsMgtCegoqmbkvy0QRvDHnn9EHdePps1y4uZnpVMc3cvDruNK+bkEx8/uqP0GgiUUuosWZZh33Ente3BLKFDbQzz+CwKs5PJSrbjtyxK8lMpnZox6kEANBAopdRH5vdb7Dvu5Fi7i7y0xAFZQoeaD8hOTqDD7eVv5k0ekdVAkRr9UKSUUuOQ32/x4u46bv75Dr7ym9385XBr+IP/hV213HVFyYD8QA+tXsCkDDufLJ0ypoIAaI9AKaXOSGh3cEt374B5AMt82Auod3p4Zkc1a5YXM39KOpnJdpZMzRyVFUGR0B6BUkpFyLIMb1Q2ceB4J42dvQOGfk7sBYQ2hs2dnMb5RTljNgiA9giUUioifr/F3uMd9HgCIJCZZB8wD1Dv9LCxvIYNt55LU1cvM3NTWFqYFbX8QMNJA4FSSp2G2+3jlfcb+Wa/PEHrVs3nwVWlPLBpf/jclz8+m5rWHjKSE1hckDkuggDohjKllDqp0AaxdrePv/uPXYNWAX3rmrkU5qTQ4fKFl4XmpCRSOmVsLAs9kW4oU0qpCIQmg/2Wn/aeAI2dHhLibUPuC2jo8tLU7eWCmdlMyUiiKCdl3PQC+tNAoJRSfUKZQv94sJ6yojzu3xRcFbR2xewh9wXYBJYWZnJBUc6Y7AFEavy2XCmlhollGapbuyk/2kaXx8enzp0RDgIAz5XXsnbFwH0Ba1eUMHdSGpcU547rIADaI1BKxTjLMuyubcXtNdS2u0lOiKfD7R3w7b/e6eHpt6p54tZz6XT7SXfEMy0ziZl50asaNpI0ECilYlKoali900NTVy9Vzd08V15Lu8vLd69fyIycJKpb3eHr211eHPFxJKQKSwuyxvS+gDM1ce5EKaUi5PUGeLeunZpW94BU0aHSkfe+uJef3rKML//XO/2Wiy5gWnYiU9LH54TwqWggUErFFL/f4o+VzSTFxw1KFb1+awW3X1rM49sq6en18/T/PZ/Gzl4mpyeOSL2A0TIx70oppYZgWYY9dR24vQE6erwnLR3psNvITk1gWWHWuJ8IjoQGAqVUTAgtDb37ud2nXRK6bvUClhVkxkQQAF0+qpSKEUdbe8JBAIZeEvov1y/kinPyuW7RVJKS7KPZ3BGlPQKl1IQT2h3c2OlhUrqDopwUGjs9Qy4J/dGNi6lo6uKi4hzOm5EdM72A/jQQKKUmDMsyHGnp4f36TmrbXRjA7Qtw3oxspmUlDRoKand5qWjqYmZOSswGAdChIaXUBOH1Bthe0czv3qvjYGMXv9lZA8Dz5bV86Zly9tY5eeyWpQOGgu5bOZ+LinO4dtHUmA0CoNlHlVITgNcbYPO+eu7tlyb6ritK2Fhew8pFBTy+rRKH3cb/fOUyAGraekhOiGdSeiLTsyfevoCT0eyjSqkJye+3+MuR1nAQgIF7AqTvM97js2ju9nBhcS6z8lNHscVjjwYCpdS4FEoRUdvhHjQRDMEP/jgbBPpOO+w28tMco9DSsS/qg2IicrWIfCAilSJyzxDPTxeRbSLyrojsEZFrot0mpdT45vdbvH2kldp2N81dvaQkxofH/kMcdhvzJqfz3+/U4rDbePimJRTlpIxSi8e2qPYIRCQOeBy4EqgFdorIJmPMgX6XfQt4zhjz7yIyH3gFKIpmu5RS45NlGY6193CszU11m4uHNh/A47OYkZPEAytLeXDzh2Ujv3v9QuZNSeORzywmP80xbovGjIRoDw2dD1QaY6oARORZYDXQPxAYIL3vcQZwPMptUkqNI36/xf56J+2uXnp6LTrcPho7PWzYXhUeDqpudfOz7ZX88MbFHGrs4rwZWVw0M4eEhDiKcnU+4HSiPTRUABzrd1zbd66/bwOfE5Fagr2Brwz1g0RkjYiUi0h5c3NzNNqqlBpjvN4AL+6u4zMbdrDzqJMPGrt4aPMBLMOgOYHqVjeHGrv4xRtVFGQmkZAQN0qtHn/GwsLZzwK/MsZMA64BnhGRQe0yxmwwxpQZY8ry8vJGvJFKqZFlWYZdx9rDGUJFGBAAhpoTsAk8fNMSivO0F3Amoj00VAcU9jue1neuv9uBqwGMMW+JiAPIBZqi3Dal1BhW09ZDp8cfXgJ6zqQ06jtcOOw2XthVy11XlLB+a8WAOYFl0zNjal/AcIl2INgJlIjITIIB4GbglhOuqQFWAL8SkXmAA9CxH6VimNcboLGrl063jyffrAp/2H/72lLu/eRcvvvqQZ7ZUc2a5cWU5Kcyb3I6xROkbORoiGogMMb4ReRO4PdAHPCUMWa/iKwDyo0xm4CvAz8Xka8RnDi+zYzH7c5KqY+sf5K4giwHlU09NHf1crzDTVZyAvXO4D6Bb7+8n198oYyHP70YBEry05ilAeCsRX1DmTHmFYKTwP3P3d/v8QHgkmi3Qyk1NlmWYesHjVQ1dTMjN5W6I+1866XB5SNDwaDT7aMgK4nSKRkxnR9oOOmfolJqVNW09XC83U3AwP7jznAQgA9TRdywbBoQnBCeOzmdxTFSOWyk6J+kUmrEWJahqrmbtw63cLipm6Mt3Rxs6CI7NZFnd9YMuSy0f/lI3R0cHZprSCk1Ik4sFemw21i7ooSn36qm3eXlritKMJghy0eWzchi9eJLdT4gSrRHoJQaESeWivT4LB7dEhz2CQ0B2US464qB5SO/d/1CLp6ZQ8mkNA0CUaI9AqVUVIVWBB1q7OKOy4p5YVct9U4P8OGwT+ix2xfg+fJa1iwvZs6kNOZNTmdmru4LiDYNBEqpYdV/KejUTAe7jzn5pxf2DLkKyGG3EVos7rDbuHRWLksLg5vCNACMHB0aUkoNm9A8wBd/9Vd2VbdT3eoKBwEYuAooNEcQShP9g08toqwom8vnTmJWvs4FjCTtESilhs3R1h6eevMwd60oobWrl9Ye75CrgGZkJ/HITUs41tbD16+aw+QMBxcW5eiH/yjRQKCUOiv9h4IQuPvKORxpddPtDdDT0jPkKqDqNjelU+NZXJgVc3WDxyIdGlJKfWShoaBr1r/BZ3/+Nve8sAeP34RTRT9XXst9K+cPWAV01xUlbN5Tx9zJ6VxQnENRrg4DjTbtESilPrITl4SuXFTA7mMd4eN2l5eNf63h4ZuWcLChk4AFG8tr+Ker5+nGsDFExmN+t7KyMlNeXj7azVAqplmW4d2ado47Pbh6/bT09BJnE7p7A/zijSqykhO49cIZrN9aQVZyAp8um6aZQkeZiOwyxpSdeF57BEqpiHi9AfYcd9LQ6WFKuoMAAXYddfLolg9rAjywcj6vH2gI1wronyp6/pQMXRI6RmkgUEqdlsfjZ9Peeu7f9GFW0HWrSnm3pnXA0tAHNx/g8VuWsW7zfm6/tJg4G5TNyObi4hxNEjeGaSBQSp2SZRneO+4MBwEIfujfv2k/T9x6Ln840BK+1uOzsCzD+puX4vIGmJTuoChHewFjnQYCpdSQQstCm7t6w7UA+vP4LDpcvgHnHHYbs/JTtWbwOKN9NaXUAJZlONrSzUu767hm/Ru8UdlCdop9yGLx+WmJA5aGapro8Ul7BEqpsNC+gIMNnWzYXhXuBfzHjiM8uKqUBzbt7zdHsIBp2Q5+/cXzcXn9mh9oHNNAoJQKC+0LuOOy4nAQeGFXLbdeOIMt79fzxK3n0uHykZ+WSEZSPIVZqRRm6TDQeKdDQ0opLMtQ2dhFRWMXP71lGZfMyqZsRgYA9U4Pz+yoZu6UTBo6PBxq7Oae/96Dw67fIycK/ZtUKsZZluHVffV8/fn3BlQOu/WiIuAo5dVO2l1ekuxx/GRLBe0ur84FTDAaCJSKUaH6wQ2dnnAQgA8rh61ZXsxdK+bQ1NlLZrKdlMR4lhRm6pLQCUiHhpSKQR6Pn03vHWflY2+y40jbkEtDLQNOt59/21ZBXloiF8zM4aJZuZoeYgLSHoFSMcSyDMfauqluc3PPf39YMGaoVNE2gYykeP7p6nksLMjUD/8JTHsESsUIv99ix5EW9h3vwun2D1gV9P+uOmfAfoC1K0qYnZdKXloCV5dO1iAwwWmPQKkJLrRB7J1jHXzrpWCaiMduWRruBdQ7PTz55hG+8TfnMDUzCYc9jqQEGyKGcyZlaBCIARoIlJqgLMtQ3drDrpoOatp6BmwQq213sXZFSThzaLvLS6/fIiUxjhR7PDlpCRRm6YRwrNBAoNQEEwoANe0uOt1+Wro8xNsGzgH8+i/V/N3yYtYsL8YyYBOYkZPMBTNySEiIG8XWq9GggUCpCcTvt9hxtJUGp4djbS6eK6+l3eXlJ59ZMmBCuN7p4am/HOGh1Qvo7vUzOy+V2flp2gOIURoIlBrnQllCW3t6qW1z888v7g1vDLvrihKe2VHN9159n/tWzuehzQfCz915eQlJCXFcWKS9gFgX9UAgIlcDjwJxwC+MMd8f4pqbgG8DBnjPGHNLtNul1EQQShJ393O7uf3SYp58s2rAxrD1Wyu4/dJiHt9WSZfHx5rlxUzPTiYvLZEZ2clMz9Z5ABVhIBCRFMBtjLFEZA4wF3jVGOM7zevigMeBK4FaYKeIbDLGHOh3TQnwz8Alxph2Ecn/iPeiVMwI9QKOtvTwQUMnWckJiDDkxjCR4JLQuZPTiY8T4m3CsmlZ2gtQYZH2CLYDl4lIFvAHYCfwGeBvT/O684FKY0wVgIg8C6wGDvS75kvA48aYdgBjTFPkzVcq9vTvBfQfArLZTr4x7IFrS+n1WxTnpTItU3sBaqBIN5SJMcYF3AD81BjzaaA0gtcVAMf6Hdf2netvDjBHRP4sIjv6hpIGN0BkjYiUi0h5c3NzhM1WauII7QfYebQtHATgwyEgf8Bw1xUlAzaGfee6BVw4M5vZ+SlcXpLH9GxND6EGi7RHICJyEcEewO1954arXxkPlAAfB6YB20VkoTGmo/9FxpgNwAaAsrIyM0zvrdS44PdbvH6wkSMtPbh9gSGHgDx+i5ffq+Phm5ZQ1dxNUU4KUzIcZKUkaJI4dUqR9gi+SnAc/0VjzH4RKQa2RfC6OqCw3/G0vnP91QKbjDE+Y8wR4BDBwKCUIhgEdla34XT78PgCJCfEDVk2cu6kNFYuKuAHr71PnE1wJNhYOj1Lk8Sp0xJjIv9yLSLpgDHGdEV4fTzBD/YVBAPATuAWY8z+ftdcDXzWGPMFEckF3gWWGGNaT/Zzy8rKTHl5ecTtVmo8sixDTVsP79R0cG+/JaFf+8QcHPE2vvfawfC5h1YvYHpOEp1uP+kOO5PSE3VFkBpERHYZY8pOPB/pqqEy4JdAWvBQOoD/a4zZdarXGWP8InIn8HuCQ0lP9fUo1gHlxphNfc9dJSIHgADwj6cKAkpNdKEAcKTFRU+vPxwEIDgE9Mjrh7jz8tncfmkxcTaYk59GQryNvFQHF8zUspHqzEU6R/AU8GVjzBsAInIpwcCw6HQvNMa8Arxywrn7+z02wN19/ykVs0KpIeo63LT2eKlq7ibuhNQQ8OF8wJNvVnH3lXPwWRZp9jitGKY+skgDQSAUBACMMW+KiD9KbVIq5liWYesHjVQ0docTwTnsNh65acmQS0LPL8ri4uIyHPFxmiBOnbVTBgIRWdb38E8i8gTwG4K7fz8D/DG6TVMqdhxp6WFPrXNAhlCPz+L7rw1ODXHfyvkYDBcW5+mHvxoWp+sR/PiE4wf6PdYlnEoNA7/f4khLN5YZvDO4utUdTg0xd3IaNoRp2UnMn6J1AtTwOWUgMMZcPlINUSqWeL0B9tY7ae7qJTkhjoxkO+mJcUMOA7m8AaZnJ/PSu8e4bul0DQJq2EW6aiiDYG9ged+pPwHrjDHOaDVMqYnK5fbydk0H79a0Yxl4+b06bj5vOlMyHNz7ybl899WBy0Jn5aXgCwS455OlujFMRcWZrBraB9zUd3wrwVVDN0SjUUpNVG63j1cONPGtlwamin52Zw2rlxQwb3I6P7pxMa5eP20uLzmpdvzG4ryiXA0AKmoiDQSzjDGf6nf8oIjsjkJ7lJqQLMtwrL2HeqcnHARgYKpoy8D7DZ2s31IZzhMkwLJp2RoEVFRFGgjcInKpMeZNABG5BHBHr1lKTQyWZTjS0sP79Z1kJttp7fEOuS8gri9jxOz8NO5aMZul07OYmZ3E9BxND6GiL9JA8HfA031zBQDtwBei0ySlxj/LMlQ1d3OgoZPDTd3hkpG/vO28ISeElxRm0tMbIM0Rx6pFUzU/kBpREQUCY8x7wOK+XEMYYzr7Py8iXzDG/DoK7VNqXAmlh9hV08E3hygZ+cs/H2bdqlLu37Q//Nx3rltIT6+XpAQ7y0vyNQCoEXdGSedO+kNE3jHGLDv9lcNDk86psShUMOZgQ+eAjWEQ/NYfKhn5rzcuIC89iQ6Xj0npiWQkxZMYH68rglTUnSzpXKRpqE/784fp5yg17oSGgUIFY4baGNa/ZGR+ehINTg9xIjjdPkry0nUoSI2q4QoEustYxaRQL+Ca9W/wRmVLOAAMVS/AJvDta0vx+CxSEuMpzEpixTmTiI8frl9DpT4a7REodRaOtvYMKBvpsNt4YVftoJKR962cz/kzs/ntrhrcvgBXz5/M4ulZGgTUmBDpqqHT+fMw/Rylxg2vN8DxDjd3XFYMwPYPmrjrihLWb63gmR3VrFlezMzcFJLi40hJjCM72c4Pb1yicwFqzIk0xcQk4LvAVGPMJ0VkPnCRMeZJAGPMnVFso1JjRmhfQE1bD8bAg5v3U93qDq8Mem1fPWuWF7O0MJPUxHg63D6ykhNYNDWDhIThKvOt1PCKtEfwK4IpJb7Zd3wI2Ag8GYU2KTXm+P0W++udHG11UdX84b6A0LLQeqeH9Vsr+rKEpvOxOboMVI0fkQ5Q5hpjngMsCJagJFhWUqkJz+Px8/bRVg42dGET4c+Vzdx64QyykhNYv7WCG5ZNC17ns5g3JZ2rSydrEFDjSqQ9gh4RyaFvdZCIXAho5lE14Xm9AV7eV899v9sX3gD2wMpSXninhhuWTePxbZVI32e+w24jLzVRg4AadyINBHcDm4BZIvJnIA+4MWqtUmoU+f0WBxs6aXN5cXsD1La7yEpOoN7pweOzeHDzfn5442IONXbhsNswJhgE1q1awKKpGad/A6XGmEhTTLwjIh8DziG4VPQDY4wvqi1TaoRZluFoSzf1nR7q2t0D0kD0nwvw+Cw8Xj82gYdWLyAn1c7ykvNZrBPCapyKdNXQ5084tUxEMMY8HYU2KTXi/H6L1w82cqSlB7cvMKh2cChV9OPbgimiC7KSmJSeiMdnUTIplYIMXRKqxq9Ih4bO6/fYAawA3gE0EKhxz++32FndxoH6YI6gOy4rPmWKiHWrF5DuiCdgwUUz07UXoMa9SIeGvtL/WEQygWej0SClRpLXG+B/DzZiDBRkJHHHZcWknqR28HlFWcydtJi0pDhKp2ZqD0BNGB91Z3EPMHM4G6LUSAolimvs8lDT5uLRLRXh+YCvfWIO/3z1XL732oe1g9etKiU5IY6CjCRNEKcmnEjnCF7mw8RyNmA+8Fy0GqVUtFiW4XBzNxVN3VjGkJFkDwcBCA4BPfL6Ie68fDZrlhczOy+VNEc83oBh4eQMHI7hysqi1NgR6b/qf+332A9UG2Nqo9AepaLCsgy1HT2UH+3g3n4FY75z3YIh5wOKclNId8TjtyzSk+wsnKIrgtTEFekcwZ+i3RClosXvt3jrSDM2sXG0tYc7LivmhV211Ds9HGtzDTkfMDndQW5qAtOzdTWQmvhOGQhEpIuhaw0IYIwx6VFplVLDxOsNsLuugwand8Du4NC+gOfKa7lv5Xwe2nwg/NyPP72EZdOzNAComHHKQGCMSRuphig1nCzLcLipm4ONXWSn2MNBAAbuC3jyzSp6PD6e/MJ5uL1+inJSdDJYxZwzmvkSkXyC+wgAMMbUDHuLlDpLlmV4dV8DX38+WDDmrhWzh5wHiLPBfSvnk+6IJ90RxyWzc0epxUqNroiyj4rIKhGpAI4AfwKOAq9G+NqrReQDEakUkXtOcd2nRMSIyKDCykpFyrIMe2s7+KCxkzsuK2ZKhgPLDF068oKZ2RRmOfj1X46QkmgfpRYrNfoi7RE8BFwIvG6MWSoilwOfO92LRCQOeBy4EqgFdorIJmPMgROuSwPWAm+fSeOV6s/vt3hlXz3feGHPgLmA1/bVhyuH9V8t9IvtVbx1pI2HbwpWDVMqVkUaCHzGmFYRsYmIzRizTUR+EsHrzgcqjTFVACLyLLAaOHDCdQ8BPwD+McL2KAUEewDVrT0cd7rp8vg53Nw9IFNoaC5gY3kNP7xxMZVNXVw6Oze4OSxzFvddW6qlI1XMizQQdIhIKvAG8J8i0kRwd/HpFADH+h3XAhf0v0BElgGFxpj/EZGTBgIRWQOsAZg+fXqEzVYTmdcboLymjSOtrgGrfk7MFBpng8+UTefHfzjIN/5mLmUzsvWDX6l+Iq1Qtg3IIDh88xpwGLj2bN9cRGzAw8DXT3etMWaDMabMGFOWl5d3tm+txjG/3+K9Y+28eqCBNpePDdsPD1oRFKoa5rDbuKg4h/lT03jy8+fxyQVTNAgodYJIewTxwB+ANoK1ijcaY1ojeF0dUNjveFrfuZA0YAHwRwmWeZoMbBKRVcaY8gjbpmKE1xtgX72TY+3uk9YNhoGZQteuKAFgeYnWEFbqZCLqERhjHjTGlAL/AEwB/iQir0fw0p1AiYjMFJEE4GaClc5CP9dpjMk1xhQZY4qAHYAGATWA1xtgd007v9tznFt+8TZrn93NE9urhqwbDMEAcM6kNNYsL2Zmbgrn6VCQUqcU6dBQSBPQALQC+ae7uK/I/Z3A74H3geeMMftFZJ2IrDrTxqrY4/UGeGnPcbZ+0DTkprAblk0LzwNAMAj8y/ULyUqOZ/XiAq6aP5n4+DP9Z65UbIk0++iXgZsI1ip+HvjSiUtAT8YY8wrwygnn7j/JtR+P5Geqic+yDEdbe2jq8nD/7/adtlhM2Yws1t+8hKKcZOZPydAPf6XOQKRzBIXAV40xu6PYFqUA8Hj8vHKggXtf3MuD15aGA8BQyeFsAmtXlJCWGM/FxbkaAJT6CCKdI/hnDQJqJHg8fv5ytDWcKjo5MR6H3cYLu2q564qS8A5hh93GfSvnk5oQR0FWEgsLMjUIKPURaZUNNSaECsYcbu6m0+0Lf/P/+fbDPLCylAc37+eZHdWsWV7MrLxUclMT6O71s2BqOqU6FKTUWdFAoEaV1xtg73En9U4P6UnxNHV5cPUGwsNAe+o64a/V/OuNiwkYQ2aynWOtPWQlJ7DinEkaAJQaBhoI1Khxu328vL+B+/vVCXjg2lKmZsRz95VzePh/D+HxWRxq6sYXsFg6PZOGTg8Xz87TtBBKDSMxZqi6M2NbWVmZKS/XrQbjUWg1UH2Hm/h4G+/VtNPZGwhXDHPYbfzrjYs51u5iWlYyFU1dLCnM5OKiHK0XrNRZEpFdxphBGZ71N0uNGMsybDnYyN46J5aBOIGinGT+cKCaWy+cEd4d3OP14/YFSEmI45JZuSyeqkXjlYomHWBVI8KyDAeOO6ls6mbD9ioe21rJE9uraOjs5e8/PpuN5TXcsGwaDruNlIR45k1Op9vrZ+m0TA0CSkWZBgIVdX6/xeY9x2nq6uXRLRUDdgc/uqUCt9di5aIC4mzwwLWlJNptzMxN4prSKSQkxI1y65Wa+DQQqKjyegPsqmnnGy/soccbGHJ3cE+vnzgbXDwrl/mT0/jY7DzmTtF9AUqNFO1zq2EXKhbT1OWhobOXOJuQlZyAPU6G3B3c5vKyaFomIobSqbonQKmRpoFADSvLMmz9oJHj7W5aerzhSeG//1gxG/9aw7pVpdy/aX94uejdV84hLy0RG4ZzC7M1CCg1CjQQqGFhWYaath4auzw44uNIS7LT2uPlhV3BmgFrV5Rw0awc/mNHNT+5aQkGSHXEk2S3kZuayPRs3Reg1GjRQKDOmtcb4L3jHdR3uKnr8IQnhPuXjXx0SwU/vWUZH58bIDPFjt0mtLt9XDIrVwOAUqNMA4H6SCzLUNXcTVOXm9qOXmrbXQBs2F41qGbA7ZcW8/i2Slq6eynITMLt9TM1P5Wl07VgjFJjgQYCdcb8fovXDzZypKWHxYUZ1La7KMhIIjkxnqzkhHDJSBhYM6AoN4Ukexzn5KfpslClxhANBCpilmU40tJDRVMX8TYhJ8VOXbsn3AsI1Qh++q0P6weHagb84FOLKNOSkUqNSRoIVEQsy/Dqvga+/vzu8If+Q6sX8Ni2wRvE1iwvZv2WShx2G/evnK91g5Ua4zQQqIhUNXeHgwAEP/Tv+92+8Ph/iMdnMXdyOj+/9Vwyk+3kpemKIKXGOg0E6qT8fot9x500d/diGTPkruC4E5b9O+w25uSnMntS2gi2VCl1NnT3jhqS32/x4u461m58l4rGbg4c7wyXiQxx2G3MnZw+oHzkDz61iOK81NFoslLqI9IegRrSvuNOHttWwdevmss3fvseWckJ3HVFCeu3Dtwj8LM/VnL7pcXE2eCiWTmcp0tClRp3NBCosFDRmNaeXtp6vHymbDqVTV14fBb1Tg/P7Kjm9kuLSYy3saAgnQdf3k91q5tDTd388FOLOG+6pohQajzSQKCA4O7gvxxp5Z2adkry08hIimf91gruuKw4nCiu3unh8W3B1UDP3H4+D396Cd6AxaR0h5aOVGoc00AQw/x+i/31TtpdvbT1+Hl0yyFWLirA5fXj9Vt4fBYv7KodNCT00OoFLJ2Wpd/+lZogNBDEIL/f4mBDJ+83dPHYtgq+vaqU3NQE/t9Vc7HHCT/dVsnlc/Nx2G0DhoTibHDprFyWTNNaAUpNJPrbHGO83gAv7q7jD+83snFnNQ+sLMXp8rOrup3vvvI+X924m89eMIM9xzpYu6IkHAyefLOK2XmpLJuepekhlJpgtEcQQyzLsL/BSUaSncKsHKZnJ/Pl/3pnUKbQB1/ezw9vXMz3Xnmfn3++DH/AYnp2CjNzdR5AqYlIA8EEF6oWdtzpxjIWLq+FMdDS7eVbL+07aaZQt9dPu8tLQWaS7gtQaoLToaEJLFQt7LX9Ddz74l7ibDacLh9f3bibg41dQ+4UDmUKTUmI5+GbllCUkzJKrVdKjZSoBwIRuVpEPhCRShG5Z4jn7xaRAyKyR0S2iMiMaLcpFvj9Fu/UtOP2WsQJ3HL+DHZUtYXLRAJD7hS2CaxbvYCF09K5unSyDgUpFQOiGghEJA54HPgkMB/4rIjMP+Gyd4EyY8wi4LfAD6PZponOsgx1Hd28cbiF7RXNfNDYRcDAI68fwjKEg0BoWWj/9BAPrV7AVfMmccOSAmbkpGoQUCpGRHuO4Hyg0hhTBSAizwKrgQOhC4wx2/pdvwP4XJTbNGH5/RYHG50cbOjh37YG9wTE2aAwK3lAL6D/TuE1y4uZMymNzGQ7Fxbl6LJQpWJQtH/rC4Bj/Y5r+86dzO3Aq0M9ISJrRKRcRMqbm5uHsYkTg2UZXtlXj9tn0dzl4c7LS4i3wfPltVQ0deGw2wb1AtpdXopyUlhYkM7FxbkaBJSKUWNm1ZCIfA4oAz421PPGmA3ABoCysjIzgk0b00L5gdp7vDQ43fT6rUHF41/bV8/aFSU8uqViQC9g3uR0XRKqlIp6IKgDCvsdT+s7N4CIfAL4JvAxY0xvlNs0YXi9Ad473oHHG6Chs5dub4CHXx96SejTb1Xz6y+ej8GQn6a5gZRSH4p2INgJlIjITIIB4Gbglv4XiMhS4AngamNMU5TbM2F4vQH+cLCRY20u3L4AG7ZXccdlxSddEtru8pKXlqh7ApRSg0R1UNgY4wfuBH4PvA88Z4zZLyLrRGRV32U/AlKB50Vkt4hsimabxjvLMhxu6qb8WBsYeHRLxYDVQCdbEqp7ApRSJxP1OQJjzCvAKyecu7/f409Euw0ThWUZXtvfwA9ee581y2fR2OkZEACGyhT63esXsmx6ptYNVkqd1JiZLFYn139C+GBDJ1/+2GyOO90kJ8QNCgChyeBZeamUTkmnOE/3AyilTk0DwRhlWYYjLT3UtPUgIjy29RDXLytkw/aq8Lf9r31iDv989Vy+99rBcACYnZfKpHQHS6dlapZQpVRExJjxtxKzrKzMlJeXj3YzosayDG9UNtHlDtDT6yfFEU+yPS6cKTTEYbdx5+WzcfusYM3g4hymZjoozNJhIKXUYCKyyxhTduJ57RGMMX6/xZHWTpq7fHzrpb3hb//fv2HRkCuCPH6LJ9+sCtYMnqE1g5VSZ04DwRji8fh5q7oFjC0cBCD4gX+kpTucHiLEYbdx2excblhaoPsClFIfmX59HAMsy1DV1MXOY+30+gzNXZ5B3/6fK6/lodULBiSJe/imJZxXlK0Twkqps6I9glHm9QbYXtlMY1cvD20+gMdnsXbF7EHf/ttdXopyk9n4pQtx+QJMStfdwUqp4aGBYJR4vQHeb+gkYAwpifE01jm547JiXthVy3PlteHcQKE5gu9ct5DFU3UlkFJq+GkgGAUej5+dx9po6fbS4PQMShL3zI5qnn6rmh/duJiKpi6WFmZxwQwNAkqp6NA5ghEUrB/cza7adlq7feSkJPDszppBSeJuWDaNdpeX9KR4Lpudy0VF2SQnJYxy65VSE5X2CEaAZRlq2nqoaOqmrcfLA33lIvv3AOqdHiAYDOJs8L3rF3JRUY72ApRSUac9giiyLMOR5m7eqWmjodNDamJ8OAjAwB5AiMNu45JZuVy7aKoGAaXUiNAeQZR4vQEONnXS67fo6Q3Q3esn3iZkJSeEv/3Dhz0AoG9SeAFLp2XqxjCl1IjRQBAFXm+APx9pRrDR0t3LsTYXz5XX0u7ycveVc/jln4+Gg0GoBzBvcjpTMxyUTs3QIKCUGlEaCIaRZRmqmrtpdblp7/Fz74t7B80FPPy/h1izvJj1Wypx2G2sW7WA7JQ4zp0+SQOAUmpUaCAYJm63j/fqnXS6/djjbeEgAANLRj6+rZKCjCTuWjGbshlZZCbZKZmUObqNV0rFNP0KOgxcbi+b9zdw2y938l6tk3dr2k9aMtJht1HndJOaGE+Hy8ecvLRRarVSSgVpj+AsuN0+9jZ04vYFuO93+8If9pZhyARxNoF/uW4hBVkOEuNszJuUjsOhfwVKqdGln0IfUZfbw6v7mrl/075BReNffq9uUMnI71y3gKWFmczM1QRxSqmxRQPBGXK5vexr6KLXb3H/pn0DagZ7fBYv7Krl1gtnsLG8htsvLSbOBvMmpzN3Uioz83UYSCk19ugcwRnocHt4t87JkZYefH6LrORg2odQzWCH3Ua908PG8hruuXoes/NSSLLHER8nzMhNHeXWK6XU0LRHEAG/3+JwSyfv1XZxf99cgMNuY+2KEp5+K5geIlQzeN7kdLJTEvD4/MTZhKvmT2ZmrqaLVkqNXdojOAXLMhxr7+KvR9s41OgiOSGOOfnBb/Yen8WjWyr4dFkwPUS7y0tBZhKOBMEQYGpmMp+YN4lZ+TonoJQa27RHMATLMtR39lDf4aWuw01Vc3d4Z/ADK0vhr9XsqevE47MoyU/j3/92GVkpCSTZhU63j7Lpubo5TCk1buin1Qksy3C4xcmOqg4+9+TbrH12N09sr+K2i4vISk7gwc37uWP5LCA4QZyZZGdSeiL+gEVaop1LZudrEFBKjSv6iXWC4509tHT5+eYJO4Mfef0Qf3vBdDw+C7fXH04P4TcBctMSuHhWLsX5aToMpJQad3RoiGCSuOr2Lro9FodbeghYZsidwXmpiTjsNgqzknnqtvPITIpndm6apotWSo1rMR8IXG4v1R0u2nr8NHf14rDHEW+TIXcGpyTGs27VAhLtwrz8NK0appSaEGI2EHg8fo45u3G6AtQ5PQMmhDfceu6g4vFrV5QwJcNBUoKNOfnpOg+glJowYi4QuNxePmjqJmAM3b0Bmjo9NHf38tLuOm69cAbP7Kjmvt/t4+tXncOa5cVYBmwCM3NSWDg1Q4eBlFITTkx9re1wezjU3E1zdy9ur0WS3UZuWgLPlR/jlvODaSFuWDaN6lY3xhgA5uSnctnsHK44J1+DgFJqQop6j0BErgYeBeKAXxhjvn/C84nA08C5QCvwGWPM0eFuR4fbw58r2vigsQvLQJzA5PREfAHDVz9Rwk9er2DlooJwqui8VAfZKYkU5TgoyNRNYUqpiSuqgUBE4oDHgSuBWmCniGwyxhzod9ntQLsxZraI3Az8APjMcLajw+1h77FOatpcbNheNWDcPyspniMtLlYuKgjXDl63egE5qfEUZiTrhLBSasKL9tDQ+UClMabKGOMFngVWn3DNauDXfY9/C6wQkWH9+n2ooQen2x+e/IUPU0RMyUoO9hBssKQwk4+X5HFt6WTOmZypQUApFROiHQgKgGP9jmv7zg15jTHGDziBnBN/kIisEZFyESlvbm4+o0Y0dvbS0+sfcm9AT68fm8DS6VnMzk9k0bRMkpLsZ/TzlVJqPBs3k8XGmA3GmDJjTFleXt4ZvXZSeiLtLi8O+8Dbddht5KYmUjolnaXT05menanLQpVSMSfan3p1QGG/42l954a8RkTigQyCk8bDZs7kFKZmJnH3lXPCwcBht/G96xeSnxrHx2bnkZnkGM63VEqpcSPaq4Z2AiUiMpPgB/7NwC0nXLMJ+ALwFnAjsNWE1m4Ok8wkB8vPyaGqycUTt55Lr9ciPz2BorxkDQBKqZgX1UBgjPGLyJ3A7wkuH33KGLNfRNYB5caYTcCTwDMiUgm0EQwWwy4zycGyGfqhr5RSJ4r6PgJjzCvAKyecu7/fYw/w6Wi3Qyml1NB0ZlQppWKcBgKllIpxGgiUUirGaSBQSqkYJ8O8UnNEiEgzUP0RX54LtAxjc8YDvefYoPccG87mnmcYYwbtyB2XgeBsiEi5MaZstNsxkvSeY4Pec2yIxj3r0JBSSsU4DQRKKRXjYjEQbBjtBowCvefYoPccG4b9nmNujkAppdRAsdgjUEop1Y8GAqWUinETNhCIyNUi8oGIVIrIPUM8nygiG/uef1tEikahmcMqgnu+W0QOiMgeEdkiIjNGo53D6XT33O+6T4mIEZFxvdQwkvsVkZv6/p73i8h/jXQbh1sE/66ni8g2EXm379/2NaPRzuEkIk+JSJOI7DvJ8yIi6/v+TPaIyLKzekNjzIT7j2DK68NAMZAAvAfMP+GaLwM/63t8M7BxtNs9Avd8OZDc9/jvY+Ge+65LA7YDO4Cy0W53lP+OS4B3gay+4/zRbvcI3PMG4O/7Hs8Hjo52u4fhvpcDy4B9J3n+GuBVQIALgbfP5v0mao/gfKDSGFNljPECzwKrT7hmNfDrvse/BVaIiIxgG4fbae/ZGLPNGOPqO9xBsGLceBbJ3zPAQ8APAM9INi4KIrnfLwGPG2PaAYwxTSPcxuEWyT0bIL3vcQZwfATbFxXGmO0E67OczGrgaRO0A8gUkSkf9f0maiAoAI71O67tOzfkNcYYP+AEckakddERyT33dzvBbxTj2Wnvua/LXGiM+Z+RbFiURPJ3PAeYIyJ/FpEdInL1iLUuOiK5528DnxORWoK1T74yMk0bVWf6+35KUS9Mo8YeEfkcUAZ8bLTbEk0iYgMeBm4b5aaMpHiCw0MfJ9jj2y4iC40xHaPZqCj7LPArY8yPReQighUPFxhjrNFu2HgxUXsEdUBhv+NpfeeGvEZE4gl2KVtHpHXREck9IyKfAL4JrDLG9I5Q26LldPecBiwA/igiRwmOpW4axxPGkfwd1wKbjDE+Y8wR4BDBwDBeRXLPtwPPARhj3gIcBBOzTWQR/b5HaqIGgp1AiYjMFJEEgpPBm064ZhPwhb7HNwJbTd8szDh12nsWkaXAEwSDwHgfO4bT3LMxxmmMyTXGFBljigjOi6wyxpSPTnPPWiT/rl8i2BtARHIJDhVVjWAbh1sk91wDrAAQkXkEA0HziLZy5G0CPt+3euhCwGmMqf+oP2xCDg0ZY/wicifwe4KrDp4yxuwXkXVAuTFmE/AkwS5kJcFJmZtHr8VnL8J7/hGQCjzfNy9eY4xZNWqNPksR3vOEEeH9/h64SkQOAAHgH40x47anG+E9fx34uYh8jeDE8W3j/EsdIvIbggE9t2/u4wHADmCM+RnBuZBrgErABXzxrN5vnP95KaWUOksTdWhIKaVUhDQQKKVUjNNAoJRSMU4DgVJKxTgNBEopFeM0ECilVIzTQKDUKYhI9wi8x6pTpdBWKtp0H4FSpyAi3caY1NFuh1LRpD0CFVNE5Psi8g/9jr8tIt/qK9TzjojsFZFBqaxF5OMisrnf8WMiclvf43NF5E8isktEfn+qdMAicle/4kDP9p27TUQe63v8KxH5977MoVV97/uUiLwvIr8avj8JpT6kgUDFmo3ATf2ObyJYl+J6Y8wygsV7fhxpbQoRsQP/BtxojDkXeAr4l1O85B5gqTFmEfB3J7kmC7gI+BrBnDKPAKXAQhFZEkm7lDoTEzLXkFInY4x5V0TyRWQqkAe0Aw3AIyKyHLAI5nWf1Hf+dM4hmOH0f/tiRxxwquRfe4D/FJGXCCaIG8rLxhgjInuBRmPMXgAR2Q8UAbsjaJdSEdNAoGLR8wQzzk4m2EP4W4JB4VxjjK8vZbXjhNf4GdiDDj0vwH5jzEURvvf/IViG8FrgmyKycIhrQunBrX6PQ8f6O6uGnQ4NqVi0kWC22RsJBoUMoKkvCFwOzBjiNdXAfBFJFJFM+tIeAx8AeX0FURARu4iUDvWmfYVyCo0x24B/6ntfnYhWo06/XaiY05fGOA2oM8bUi8h/Ai/3DcWUAweHeM0xEXkO2AccIVggHmOMV0RuBNaLSAbB36mfAPuHeOs44D/6rhNgvTGmY3yXylYTgS4fVUqpGKdDQ0opFeN0aEipKBCRx4FLTjj9qDHml6PRHqVORYeGlFIqxunQkFJKxTgNBEopFeM0ECilVIzTQKCUUjHu/wMEkxgTX3YboAAAAABJRU5ErkJggg==\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import seaborn as sns\n", - "\n", - "sns.scatterplot(data=merged_results_wo_zeros, x=\"value_sim\", y=\"value_obs\")\n", - "#sns.regplot(data=merged_results, x=\"value_sim\", y=\"value_obs\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 304, - "outputs": [ - { - "data": { - "text/plain": "LinregressResult(slope=0.9998410722635966, intercept=4.4762082968718975e-05, rvalue=0.9999904091368037, pvalue=0.0, stderr=0.00011336879523644994, intercept_stderr=5.826949896067866e-05)" - }, - "execution_count": 304, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#from scipy.special import rel_entr\n", - "from scipy.stats import linregress\n", - "linregress(merged_results_wo_zeros.value_sim.values, merged_results_wo_zeros.value_obs.values)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 305, - "outputs": [ - { - "data": { - "text/plain": "" - }, - "execution_count": 305, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEKCAYAAADenhiQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAB0GklEQVR4nO29eYBcV3ng+/vuraX3RepuydYSS1hG2I4NtobgGT+PAg6xw3t2Mk8h9gwDmWennUBiIANjM2GLCfPsAWJwQoj9zE5iA3ohiBcvYBwhmFhg2cbBxrIlWsJaLPWi3qprv/d7f9x7q6urq7qrqqu6qlvnB21V3bq36py7nO+cbxVVxWAwGAyGWmI1ugEGg8FgWH0Y4WIwGAyGmmOEi8FgMBhqjhEuBoPBYKg5RrgYDAaDoeYY4WIwGAyGmtNw4SIi14jIiyJyWERuL/J5VES+7n/+YxE5L++zD/jbXxSR38zb/l4ReV5EnhORB0SkZZm6YzAYDAYaLFxExAY+C1wLXAjcKCIXFux2EzCuqucDdwN3+cdeCNwAXARcA/yNiNgisgG4FdihqhcDtr+fwWAwGJaJUIN///XAYVUdAhCRB4HrgZ/n7XM98FH/9W7gr0VE/O0PqmoKOCIih/3vexmvX60ikgHagJOLNaSvr0/PO++8WvTJYDAYzhqeeuqpUVXtL9zeaOGyATiW9/448Gul9lHVrIhMAmv97fsLjt2gqk+IyCfxhEwC+K6qfrfYj4vIIDAIsHnzZg4cOLD0HhkMBsNZhIj8stj2httcao2I9OKtarYA5wLtIvK2Yvuq6n2qukNVd/T3zxO8BoPBYKiSRguXE8CmvPcb/W1F9xGRENANjC1w7NXAEVUdUdUM8A/Av61L6w0Gg8FQlEYLlyeBbSKyRUQieIb3PQX77AHe4b/eBTyuXrbNPcANvjfZFmAb8BM8ddgbRKTNt828CXhhGfpiMBgMBp+G2lx8G8ofA4/ieXV9QVWfF5E7gAOqugf4PPBV32B/Bt/zy9/vG3jG/yzwLlV1gB+LyG7gaX/7M8B9y903g8FgOJsRk3LfY8eOHWoM+gaDwVAZIvKUqu4o3N5obzHDKmHvwWHu3TfEsfE4m3rbuOWqrezcPtDoZhkMhgbRaJuLYRWw9+AwH97zPMPTSXpawwxPJ/nwnufZe3C40U0zGAwNwggXw5K5d98QYVtoi4QQ8f4N28K9+4Ya3TSDwdAgjHAxLJlj43Faw/acba1hm+Pj8Qa1yGAwNBpjczFURb6NZSqRwXFd+jpm84MmMg4be9sa2EKDwdBIjHAxVExgYwnbQk9rmKzjMjydBmBte5RExiHjKLdctbXBLTUYDI3CCBdDxeTbWAD6O70Vy0zKIWRl2Gi8xQw1xngjrjyMcDFUzLHxOD2t4Tnb+jqiTCYy/PC2NzaoVYbVSuFKOfBGvAOMgGlijEHfUDGbettIZJw524yNxVAvjDfiysQIF0PF3HLVVjKOEk9nUfX+NTYWQ70w3ogrEyNcDBWzc/sAd1x3EQOdLUwmMgx0tnDHdRcZFYWhLpiV8srE2FwMVbFz+4ARJoZl4ZartvLhPc8TT2dpDdvGG3GFYISLoSYYbx5Dvdi5fYA78Gwvx8fjxhtxhWCEi2HJGG8eQ70xK+WVhxEuhgUpZ0VSGPfSFgkRT2e5d9+QGRAMhrMUY9A3lKTcbMfGm8dgMBRihIuhJOXGFxhvHoPBUIhRixlKEkTiTyUyjMZSpB2XiG0xPJXkxvv251RlV2xdw+6nTxhvHoPBkKPhKxcRuUZEXhSRwyJye5HPoyLydf/zH4vIeXmffcDf/qKI/Gbe9h4R2S0iB0XkBRG5Ypm6s6rY1NvGaCzFyckEWUexRUhmHaZTDkdGYzlV2Vf3/xLXcTg+nuDQcIyIbZm4F4PhLKehwkVEbOCzwLXAhcCNInJhwW43AeOqej5wN3CXf+yFwA3ARcA1wN/43wfwGeARVd0OXAq8UO++rEZuuWor4/EMAGJB1nVxXO+z4ekU08ksjquMxzOMxjNsG+igty3M0OgM79v9LDfet99UozQYzlIavXJ5PXBYVYdUNQ08CFxfsM/1wJf917uBN4mI+NsfVNWUqh4BDgOvF5Fu4Crg8wCqmlbVifp3ZfWxc/sAnS0hLCCZccm6s5+5CicnE7wykcAScFxlOpllbCaNq0o8lTXljg2Gs5hGC5cNwLG898f9bUX3UdUsMAmsXeDYLcAI8EUReUZE7heR9mI/LiKDInJARA6MjIzUoj+rjv6OKI6CyNztAlgIaUcBiNgWo7EUFoJtCRlXTYLBJmfvwWFuvG8/V971uFllGmpOo4VLPQgBlwGfU9XXATPAPFsOgKrep6o7VHVHf3//crZxxaCq/r9FPhRF8VYxfR1R0o6LiLdvxPZuLeOS3JyU62ZuMFRLo73FTgCb8t5v9LcV2+e4iISAbmBsgWOPA8dV9cf+9t2UEC6GxYmlHXrbQozEMrlttngCJesqbRGbtohNyBYituUJGIT+zigw3yW5HmliTOqZyjGBr4Z60+iVy5PANhHZIiIRPAP9noJ99gDv8F/vAh5Xbzq9B7jB9ybbAmwDfqKqp4BjIvJq/5g3AT+vd0dWK53REGN5ggXAUbAs2NDTxt/8x8v45K5LGehsoTVsYYmwtiNMRzQ0LxV/PWbLZgZeHSbw1VBvGrpyUdWsiPwx8ChgA19Q1edF5A7ggKruwTPMf1VEDgNn8AQQ/n7fwBMcWeBdqhpE8v0J8He+wBoC/suydmwVMZ1I4xbZ7riw67INuVlu8G+wijh0eoq0o0RCVs7mUo/ZspmBV8em3jaGp5O58wYm8NVQWxqtFkNVHwIeKtj24bzXSeB3Sxz7ceDjRbb/FNhR04aepYzMZEp+9sTQGW4t2BYM6B/e8zzdttAatnOriXg6y/quljn7L3W2XKzkspmBL45JY2+oNw0XLoaVgeT+4xnsBUoO4KVWE+msSyLj1HS2bGbgi1PKJnUHcOfDL3BoOAbA1r6iTpUGQ1U02uZiqDNLdTedN+D4XmNhW0oO4KX0+RFbal4e2ZRcXpjFbFLxjMvG3la2DXSQdlxjrzLUDCNcVjG1MHbfds12ultDKLPuyLZAV2u45ABeKpHltnVdNS+PbEouL8xCyUfLTUxqMFSDUYutYmph7N65fYCb/t0W/vYHQ8QzDgic093CX/z2rxb9jr0Hh5mIpzk6FidsC+s6o4Rsi4yjXLF1DXc9cpCh0RkA2iP2vOOrwRSSKs1CNikFY68y1A2zclnFFKqnppMZXplI8JOjZ8pWke09OMzup09wTk8LF5/bxda+diyr+G0TrJTSjsvGnhZQOD6RIGwJuy7bwFf3/5JDwzFUFVXl8IiXg8yoYerHQuUQTKkEQz0xwmUVkz94TCcznJxIknGVlpBVtoqsEtVJ/r5drRG2revkvLXt9LZHeWLoDNPJLLYl2Jbl/YkQS2VrqoYxKU3mspBNytirDPXECJdVTP7gMTyVRH1rfF9HtGz9eiXBdsX2zTouT788zk+OniGZdXPpZMDLV+a4WjM1jAmonM9CNiljrzLUE2NzWcUE7qb37hvi6FiclpBFX0eULl/PXo5+Pd/VdzqZYWQ6RTLr0B4Jsffg8JyBqNAteDqZ4cREkpAttIQsZtKOF1iJYluCKthWaa+zSjEBlcVZyCZl7FWGemFWLqucndsHeGDwDbz+vDWs727JCRYoT78erH5GY0lOjCdIOy6WCG0Re96qoFDNcmoyCcC6zhb6OqLYlhcok3ZcHNfFUaUjGqqZGsakNKkdRr1oWCpGuJwlVKtfD1QnMykHxct2fG53K/2dLfPUaoVqFgU29HgCras1zMaeVqIh75YTEc7vb+eTuy6dkzpmKQOaMVDXBqNeNNQCoxY7S8hXkR0fj7OxguzBO7cP0NUaZvOaNiSvsEuxVUG+muXG+/YzPJ3MfdbVGiZkCwOdLTww+IY5xwUDWtiWOQPaHVC22sakNKkNRr1oqAVGuJxFLEW/Xk2alUoG+1rF5FQrQFc7lZQlWM58be998Gn2/OspHNezw113yXruvuGymv+OYfkxwsVQFtWsCioZ7IsNaIGn2ZV3Pb7ogJjLxjw8zXQiQ8pRjo0n+PGRMdoiFr+6ofesFDR7Dw57+cNGYoQti3Vd0UVXhcuVr+29Dz7Nt376Su6946r//mkjYFYBokVLDJ597NixQw8cONDoZtSMehblqseqIFChBQPaVCLDiYkEIUs4f6AjJ8yKucoGKrWM43BqMkWxO7qn1Yu9OZtcbfNtJ66riHgeeuf2tGBbxdWT+ceF/azWC537pbD1A/+Em3exBC91nW0Jv/gfv1Wz3zHUFxF5SlXnZaE3K5dVSC3sF8Wop9tqsDI6MR5nPJ7JCYiO6GzwZik1WaBSG4tliwoWgKlklrUd0bPKbhCcF8dVbBFEBBdlZDrFlr72kmqu5VAv7j04PEewQC4nKk7hB4YViREuK4xyViQrySB7z2Mvcf+PjjCTdhC80sn5TCQy6NgMm9e2Lxi82dMaJu0UK2vm4erZ55YcnJeIbZF1FBEvcDXtuIuqueod/7JQ8G7gsl4upsx1c2JckVcQ5bqIrpR4j3see4nPPH6YRMYhZM0XLAGTySxTiUzJATFwQY7YpW9nS84+t+TgvPR1RHFRXNf7s0Ua7kV3bDxOV7T49brukvVlf49xm25ejHBZQZSb56szGuLwcIyDp6YYGokxnSw9MDeS+390BEsgZFmoLjxbPT2dLDkgBjE8Xa0hSn1LV0uo4QNqtVQb/xOcl5AtnNvdgljgKGzpa2+47WlTbxv9Xa10RedOgta2hbj7hsvK7rMpG9C8NFy4iMg1IvKiiBwWkduLfB4Vka/7n/9YRM7L++wD/vYXReQ3C46zReQZEfn/lqEby0I5K5K9B4c5Ph4nmXXJOMpM2uHYmThTiUzTDawzaQfLzy+WWUClBV4tmVIDYhC8ed7aDtZ2RIjasyLGEuiMWrzmnO6GD6jVsJSZeX5Qq6vwuk29fP7tO3j4PVc1/DwEgq+/y8u2/ar+djavaeNTb31dRX1eKav0s5GG2lxExAY+C/wGcBx4UkT2qOrP83a7CRhX1fNF5AbgLuD3RORC4AbgIuBc4DERuUBVgxDtdwMvAF3L1J26U46L6J0Pv0Ai4xIScPFsDa5C1JaGDyiFtEc8TyTHXViwrGkL8+r1XQu2fzlyZDVCt79U+1mz5Q7LP4cdERsRYTKRmeM0cON9+8vusylz3bw0euXyeuCwqg6pahp4ELi+YJ/rgS/7r3cDbxIvTPx64EFVTanqEeCw/32IyEbgLcD9y9CHZaOcFC5HxuJYAuGQTTRk0xq2CdvCyEymgS0vzs1XbskJv1L+QWvawnS0lK56WS5LTS1T6QqiVrm5VtPMvPAcZlxvZf2x6y/mgcE35ARHJX02ZQOal0YLlw3Asbz3x/1tRfdR1SwwCaxd5NhPA/8Nb/JeEhEZFJEDInJgZGSkyi4sH6stRfqtV1/Au994fs47yBLo74jwK2vaaAlbREMWr17fVbKP5Q7gtTD6VqLbr6WReTXlSyv3HFbS59X2TKwmVp0rsoj878Cwqj4lIjsX2ldV7wPuAy+Isv6tWzqLqTm29rVzaDiGqOd6qv7KYFt/+zK2snxuvfoCLtnYMy9ob7FBopJYnmBQyzrKkckZ0o6LbQl3PvxC2YNQJSlRaukKvprypZV7Divtc7Op/gwejV65nAA25b3f6G8ruo+IhIBuYGyBY/8dcJ2IHMVTs71RRL5Wj8Y3I7dds53etrAXM+K4CNDbFua2a7aXdXwlq4FapWSvZvZZyUri2HicqUSal8/EmUk7ZB0l67gcGomV3e5KZtO1VGWtppl5uedwNfX5bKbRK5cngW0isgVPMNwA/MeCffYA7wCeAHYBj6uqisge4O9F5C/xDPrbgJ+o6hPABwD8lcv7VPVty9CXpmDn9gE+sevSqqKry10N1CMDQKWzz0pWEgKMxGZtTgpkXYj4wqic361kNl1rI/NqmJnvPTjM+EyKo2MzuRxnIdsqeQ5XQ5/Pdhq6cvFtKH8MPIrn2fUNVX1eRO4Qkev83T4PrBWRw8CfArf7xz4PfAP4OfAI8K48T7GzkmA18cFvPwcwz1C6GOWuBpohtqDcWfDeg8O8Mpko+h0CZa8mKplNGyPzXHK531xlY08rCByfSBKxLbMiWcU0euWCqj4EPFSw7cN5r5PA75Y49uPAxxf47r3A3lq0s9mpxWqi3NXAcqZkL0W5K4l79w2hzCZFzCfInHzJRx/l5iu3cOvVFyz4m+XOpnduH2DX8YlcWpv2iM3NV245awfRQhtUV2uEeDpLT1vkrD0nZwMNFy6GuVQbS7GYEbmc7y1XndMMsQXlJlc8Nh4nalukcckW8R0MW17bP/P4YYBFBUw57D04zO6nT9DfGWWzL/h2P32CSzb2nJWDaTNMRgox+cjqjxEuTUTh6uPIaIxbvvYUnS0htg10Vl3gqdxVTbmrgWbxYCpcSQRqwfwBY1NvG47rMhbLELLcXFwNQMgSQn4+sqzr8rf7hnhi6MySB5x8QT+VyDAaS5HKutz64DPcc8PrzrpBrBkmI/nUK2u4YS6mnotPM9Rzya9pMpXIcNK3FYQt4Zye1gVrahTWQwGIp7MMdLYAlPysWLnhcpwBytlvOWeHpWqQ7LpsA7ufPkHGcZiMZ0g5Lo4LtkAkZJN1XBzVnMBZ3xWlryO6pBomV971OD2tYaaTWU5OJrAQECXrKht62nLfWc35WYkz7uWqD1Psd4udq4WelWL1bZqVZrkXTD2XFUD+6mM0lsJCEAsyri4aK7HQauKD336ubLVEJXaFhfZb7tlhKbXgE0NnuOO6izxBaHmC8NnjE2Qcl6zjkinIxDw2kyYasulqDVcdmxLM1INraFmCq9ASsuY4PlR6flbqjLtW9WEqGUwXOlfNqKYrh/z+d0ZDjMRSdLeGm/ZeMMKlichXH6QdF1uErOOpcg6emiJiW0zG00WPXegB3rRv+dUSy11TZqEBo1AQBqn+XV+wBMb+kCVYCKOxFF2t4aoHnEDQp7IutuWnt1Ho74zmvrOa87OS6vQUslTX4koF60LnqtnUdOVQ2P/DwzGyrtIeCSGRhYvpNQojXJqI/NVHxLZIZBxc9QY9W4S045J1lb0Hh0tmBy62/Yqta/js3l/guEo0ZNHZEiISsnM2kvz68+msi6pXEjdsCxes66pqlrncs8NKBozAaH/3Y4cAr4BWyBIsEQRyRceqHXACQX/rg88wk87SErLo74zS2eKthjb2tlV8fvYeHObpl8dxVYnYFn0d0SUJwJVGpYJ1ofP7sesvrshm2Azqp8L+O6pY4mk4wPs37bgcH0+UHB+Wm0ZH6BvyyI+laA1bfj1xCNniu9MKa9rDFcWTBJ5La9rDRGwhmXUYj2fYddmGnN7/w3ue5+hYjMl4hpl0lumUQyyVZSqZ5chorKq8WMudE6vS2JJbr76AN2xdy6v627no3G429LSi6j20EdtaUmxKMBiFbSFkWXS1huiIhuZ852LnJz8DwjV3/4D3734WAT/zgnJyMrFgAbXVRqVZDxY6v5XELDVLMbLC/geF8ZJZl5OTCa/SKN790SzF0oxwaTJ2bh/ggcE3cOBDb2Zte4SobeG4SsgSetvCTMYz7B8a45KPPsqOv/jeoqlX7nrkIMNTSUZjaUSETb1tbOxt5YmhM8DsjGgqkfVsA767rqveIDYez1QVILncgYTVpAzJb2NHNMTajjCWCK1hq+qUI/mD0TndrfS2hTkzk+HlM3FGplPMpDLcu2+IK7auKXl+Cge0o2fijMczdLeGAQn+v2ABtdVGpZOVxe6/4Dn74W1vXDDQuBkChmF+//s7o7661RMqwX/Wd7c0TbE0oxZrYrYNdOZUPdPJDCcnkmTdWZVNIu1wdCxWUve89+AwLw3HsMWrS551lZMTSc7pjuZmfPn15wsDDRVIZT3Dd6Wql1oZcSv9zUq/vy1scWQsjusq4ZBFR9RmW5WqwL0Hh7n1wWeIpx2iIU911e97643HM2zsbaU1bDM8nWT30yfYddkGnhg6M+/8FNYzcVxPBRJLZTm3p4WRac+1WVSW5Hm2kqgmmWUt7r9mMf4X9t/2J5vj8TSuKlF7VvWqqk2hKjXCpYnJv6GGp5IoiqonKEKWhavKVCLL+u5QUd1zMOtS11OpiYCLcnoqxes29wKztgpPFTQ/e443Q07xuk29Fbe/mfND5RtI13dFOTGRJJ116e+IVOR5k2+vmk5mSWddIiHJqa4AppNZsq5b1JOtmOtr4YAWsS0yjkvacelsCedsNwOdLXNUmyvNi6wSqhEWtbj/msX4X6z/H3rLhdy7b6gp2lcMI1yalL0Hh7nrkYOcnIjj+N5G0ZCFZXnGZ/AM0WnHLTmTOjYeZ11nlJOTSXC9/R3Xi1R/6fQUN963nyu2rmH30yfoag0xU0S4WEJTGDtr/Tv56o6hkRi2eLqm0Viarf0dZXne5A/q8VQW148Zc1wlbFvgkgugjNpzNdCL2QvyB4y+jignJhKELEFV583ayzF2r4aVTSMmK80SMJxPvnahGdsXYIRLE7L34DDv3/0s4/EMlkhugM86LmHbQpVcrZbAq2xjb9u8AaQjYjORyGDhCaHgpgxbcE536zz1zOh0Gkc19/0AYdtiS1/7gsbOes+Yc4kPHYex6RTHxxM8MTTGpt5WPnb9xVX9Vv7qIHD7RmY9xcpRfQT2LEeVrKOEbcG2BMdVbEtBlFRWsS2hJWwzNBIj7bhEbM/If97ajqLfWzhghGyhpy1Mf0d0Xkngwr4E5Lc//zrZAs8cG+emrxzggoEObrtm+6oSQLWmEerdYpR81q67aDaOq4HtK4YRLk3IvfuGmE5msX33WADXdcgqWK63BHH92sBd7WEyjnLF1jXzbr7RWIqpRJaQLURCQirriZfetkjOOJmvnikVSV2qFsxyxV3cu2+IjOMwPJXCyZu2HRtP8P7dz/KJXZdWHejYFgkRsS2y/hcHXjiLqRaK2bPSjhK2vKDJkCUksy7tkRBv2t7Pnn89haqiQMZxmEk7vGHLmqLfXUoFUqqPi6lu8oulnZxMYuEJmSOjM7nJAFQe1Hm20Azq3YWetUoyny8nRrg0IcfG42RdN5f3CiBkW7j+6kMUsq4SsoXuljC3X/uaojdfKpMgbAth28oZ7G0LJhMZZtJObhYdBGZWOktbLmPnsfE4k/FMrma1+J4HimfPWGqFx76OCCcmkqBe+pdyPNsK7VlhyzvHGVdpj9is727JpTi5d98QnVGbyUQ2l6HZEnjoudNseewlHvrZKxwZ887Z1r723Gpioejz/BVGoNospRoJrtORyZlcxgAlUN/NehatlADNs3GF1SyOBZVghEsTsqm3jdHp1Bz1lKqXD8uyrJzXUSLjEM94Q26xmy/ruogIW/s99cvQSIxkxiHlKqBFAzMrmaVVa+zMHxw6IjYiwnQqu2C25lcmExSmwbPE62PVFR6ZFaTn97cjIsRSnqF8sQGr0J5lCYQsrwiZJTAyncoN3IeGp8k4XvyM5dvLFCWTdfmrfz6MqncMwKHh2IKrsWLqkYU8z4LzNzydnFX/MatSDQYohRUxeC2380KzCLJmcSyoBCNcGsBiN+wtV23N2VxUvBHVVRDxgijLTWkRsizf/92jvzPKL/0ZsljeAJMfmLmU2X+5xsRC/f/hkRkANvS0zBkogDl5lIIAMYWcRdMST/20mPqq1LleirojON/ndrcyGkuRzLqekADSjtLbFqKvI8rwdNLzFvPtZQHqZ2d2HM+NNBA6orrgamyhHGqlki4G18m2PHWqMJuOJn+AWgmD13KmwGkmL7xmNtyXwgRRLjOFAXJBWv38gMid271SxdsGOhARRITz+9tpDdtMxjMcPDXF0EiM6WQmN7ssFjTW2TIbGa7qGZZFIGpLLjDz3J4W1rZHq5/9X3cRYUs4NBzj+HiC9oi94DH5g8NoLI0tnhF8NJbOBajd+fALc85R2nFp9wVMQDDT72wJccXWNblo9uAc7j04zLWf3sdNXznAM8fGsYWaRlcH5zuVdbzMyq6XWdmywFVlbCbNdDJLWyREb1sYFy/6X1HcPKcJYXZ1iv96odVYpZHqMHudzlvThuMH3Z3THcW2JDdArZTqmdX0v1qaJYASqgsSbjRm5bLMFNb6GJvx7B3xVHbezCj/xtl7cJg/+OoBMr7hOes4HB9P0N8Z4by1HSWNwBRsi/j2l8J040uZocYz7hxV3UKzu3K8tA4Nx9jY2zpndtrXAed2t6Kqc+wT1168nt1Pn8jNLo+OxfiDrx7IGegtAXWFk5PeKiMYHJb6UO7c7lWb/OzeX5DK8zLIumCLErKsXALMvo4oM6kMaQcyjueW3N3uRe67rjtP/RmyrJLXo1r1SHA/5ZdKKFT/NYNX1GLUQj209+Awdz78QlE7Vz7NZudoBseCSmi4cBGRa4DPADZwv6reWfB5FPgKcDkwBvyeqh71P/sAcBPgALeq6qMissnffx2eAuU+Vf3MMnVnQQqTD2ZdLTut/l2PHMTJG8QUz6j/ymSKVyZTbPvvD3F+gWtpQKGQquXyulI1RTleWkDR2elkIsMPb3vjnO350ezTyQxjsQwZP8+S4mckhly24y197UseHIIB+umXx3Hd+eUtHQVLXdKOJzFGYylELDqiQsbx+jmTcrxcbypkHJdgXeYq9LaFS16PpapHFhqgVsLgtdT+7z04zPt2P8tEPLOonWsl2jmaiYaqxUTEBj4LXAtcCNwoIhcW7HYTMK6q5wN3A3f5x14I3ABcBFwD/I3/fVngv6rqhcAbgHcV+c5l557HXuKWrz1FKuv66hSXVNbNqUiCwbXYzGjvwWEOnpqe9ZYq8v0ZV3nx1DR/8NUDXP6x75bMOVbr5XWlaop89UtfRwRHFcdV+joiOVXM1r72svNI5f/+yHRqdgXArOrMc2zwVkdLHRzy1ZquatHSyQAZ17umI9NJRmJp2qM253S30h61mUk72JbnreW6nnty1vEyUW8b6FjQtXolqkdqyVL7f+++IWKprK+Otfw/ydm58mmkqjA/cWmpZ7mcfRpJo1curwcOq+oQgIg8CFwP/Dxvn+uBj/qvdwN/LSLib39QVVPAERE5DLxeVZ8AXgFQ1WkReQHYUPCdy8reg8N8du8v/BWLkHa8qoSCJxQiIvR3RoH5g2gwmAVrlsL8XzAbUOkCrqOcmcnwzMvjJb2OajlDrXR2V46XFlD27HReDRxLcsI3ZHlpU7zSxjrHxlAt+Ss1Ly3L/KwGAZYo8bTDQGeEvg4vx9hUIgsoE/EskZBFOGT5ucOEe992eVnXZSWsMOrJUvp/bDzuBbnmGbpEKJo/r1EBlIWOBM8eG+f3v/Qk4MVUXXfJeq5/7camcTYoRaOFywbgWN7748CvldpHVbMiMgms9bfvLzh2Q/6BInIe8Drgx8V+XEQGgUGAzZs3V9uHRfGCAN15rrTB28CAG7K8WiuBrSQ4NmwLYcubDZdTlFr97xyPZ7jrkYN1vdmqUVOUMziU+1AHvz8yncRxNWeTAnJxPY7rGdO3rmnn9mtfs6TzEejhp5MZsk6JZQsQsYWLN/TO09unHRfXv46q5IQf6JKuVbO4zDY7m3rbGI2lvPikMuxcjRDk+ROYX47GcuEG4K12v/XTV9j30gg97dGmjktqtHCpGyLSAfy/wHtUdarYPqp6H3AfwI4dO8oZt6vipdNTqJYWDG7esqRwn2PjcWwBLbpm8SmyOTAsD43OVNfoMql0dlfuIFjsoS51bGBYD+aitu+37KqLImxfX9wWVQ2bets4OhZjLJZBRLBQ8kVMyPfIO6e7heN+O/NXdsFqR/AES4AALw3Hqir01Ewus8tFtcL0lqu25mwu6gbOMYptC4eGp7nxvv0NF8z5E5Kp1NyVcaClGItnOadnrjBstrikRrsinwA25b3f6G8ruo+IhIBuPMN+yWNFJIwnWP5OVf+hLi2vgIyjc9xNS7FtoJPu1rnFwDb1tnF6KoVtiReEV+R7FhVadWbn9vJqYyyl8NJCxz4xdIaNva1ceG43561to8W3wbRGQnz+7Tt4+D1X1WywuOWqrZyZyaC+uM+PXRG85KLndrcSsq2coM3X23e1ekJG8/7AE4jVurk2k8vscrCU+2jn9gE+uetSzu9vR9WLSXLxrl17xK5bMbBK7CNB7ZapRGbB71zOYnzV0Gjh8iSwTUS2iEgEz0C/p2CfPcA7/Ne7gMdVVf3tN4hIVES2ANuAn/j2mM8DL6jqXy5LL1j45omELGwRQgucbVdhKpGZN/u45aqtZFyv9LAlELYsb3ZcRpsU2LK2eW62pQyCCx2bb9TvbAnT3xklalvMpB3u3TdU04Fi5/YBOqI2EdvCUS8FT39HhLAtWBZs6WsnZM/adgoN0Oet7eB3XntO7vssgZCAWMK6zurijY6Nx8k6LkMjMX7+yhTPn5zkyMgMT7883nRG3lqwVGG6c/sAt1/7Gs7paaUlbBGxvViyVyZT81LiVEqxcaBSYXjLVVuZTGQ4MZGY/2EugJimj0tqqFrMt6H8MfAonivyF1T1eRG5AzigqnvwBMVXfYP9GTwBhL/fN/AM9VngXarqiMiVwH8GfiYiP/V/6r+r6kP16kd+1t7JeIaTE17WXku8ma1tCR0tNvGUQ3YBq8nJyQRrsxG29M1my925fYBt/R0cPeMZIiO2BY4ituQ8zOJpp+i3hm3h9mtfU+vuVs1S4gYWOjZf9RQUVVOUlpDF8HSS9+9+lrXtEWJppyb2iAvWdc1zYrAtIZ52imYtLqbie+GVH8y5pn0dUUK2MOAXF6uEjoidy3Tg5C1XXdddleqxWsSfBAIqMO6LCC7KyHT17uql1JNtYasid/1/PT7BmZl0Uc1DsOn6S8/h+tdubOq4pLKEi4i0AwlVdUXkAmA78LCqLrxuKwN/0H+oYNuH814ngd8tcezHgY8XbPsRxb1160aQtXcslsFVzd0Urvr1UxyIp31bQAkEb2A4PZ0i62qu1soTQ2cYnUmjCmvbI/R1RHnh1BSWCO0Rm5FYeo5gsWRWHfYnv35+U6VTX0rcwELHBkb90ViSU5OpXHLInlYbx/VKNU+nspzf31ETe0QxJ4ZIyObO/3BJ2d95+7WvKZqBupqZp/g6V6dwNBKpWdBoM1GL+JNAQAWxViIs2V29VMzXkbE42wbmllcoJQzveewlPvP44QVV2r/z2nO4+4bLgPLu4UY9++WqxfYBLSKyAfgu3srgS/Vq1Erj2HicM7E0GddLAplP1p1N8+FqaamXC/hTmIinOXB0jE9//xBHx2Ks72phTXuY8XiGU5MJ2iMhOqI2E4nsvC+0RGgJWbxmfSe3Xn0BsDQddS1ZStzAYseqegGl+TaMiUSGVyYSWOINvMVUKNXECtQi1qSW8SrTqSxr2sLzVq9BDaBmMvJWSrHrU4v4k8Cu0dcRxcWLN3L9VUy1Qv6l01Ocmkzm0jMFam4o3z5y/4+OYIk3ScxPDWQJvKq/nSu2rs0JlnIIakM98/I4pyYTuRCF5Xj2y1WLiarGReQm4G9U9X/mqZzOejqjIY77UeHFCALtQpY3ywyyEQeuySHfzThAgKwvaMZnMvR1tNDX0UJbJJSLBbnla0+hKCGBjP89tnjeSgNdLXNqsCxnsr+FWErcQKljAd8VOUXEt3Uo+J5c3nkO56kQwU8xc3qKaz+9j5eGY4Rtz95RyaqmFi6qtXJz3dTbxjMvj8/efxIkJa2+RHUzUM8CWcHqM2wL53a3eBoDxxvAq/Es3HtwmFjK8eKpZLbM9dpshK197cyknbLc9WfSjjdO+DFa5GlBqhF6dz1ykPF4xiuN7hcaXI4QBahAuIjIFcB/wouYB89GYsCbNcPiMSghPxo463i5p4J8WpmCcIn896m8EPBgKR0YlZMZr6561MYrIOYXoyqcATdTjqSlDKjFjr3xvv1kHGfOeQL8FaR3PtKO0t06e7uOxlJMpxxiqRnPzdul4txjzaBmDLjlqq3c9JUnsf2U/4VG36UaeRvV13oWyCqcrLxuU29ODf3Bbz/Hpn2V9fPefUOsaQ8zFst4kxsLcL2B/M7/cElun8WEYXvEEz5eKXOLrJ97zrakqpXt0OiMpyp3NVdlFuDQ8HRF31MN5QqX9wAfAL7lG9K3Av9ct1atMIp6dRQhYgtt0VAuWaUts7mvyiF/KV3MqBxPe1HuhTfgas6RdGh4msn4fNNfLnRIvOJq4/EMbRGbkG0xHs+wpj3sZWW2BEFy9e7LMeY2W1xJvtOHornJzkIlqsulkX2t96Qof7Ky1H4eG4+ztj1KNGQzMp3yVsyW97znO3Ysxs1XbuEzjx8m67pYfpVTUXj3G8+v+nw7rs6p4AreJKSamKpKKMvmoqo/UNXrgM+JSKeqDqnqrXVr1QrinsdeYjpVOgUIeOqJiC1s6G2jpzWMqpf+xRLJVSUsRdiWonrlQr3zyHSS4+OJXCBYvk612dOpB3r1yz/2XS756KNzyg8sRjrrgsyNNwkI28Lm3jY2r20jZAmnplIMdLbQ2RJibXuUiK8mgMqMuc0YV3L7ta9hoLOF89a2ceE5XWzpa5+nHq3GvtTIvgZ2kXzqNSlaaj+Dtna2hNna38H29V2c09PKtoHOitpx69UX8O43nk9r2CbresL03W88P2c/rZQta9tyzgFBTSTwxqN6X8OyhIuI7BCRnwH/CjwnIs+KyOV1bdkK4XM/+EVZ+/W0hhmJpRibSXvqMX9W4lLa0N8ZtXlVf0dRg2++QfjUZCI3G1/f1TLPYN/MyQ6DGeOR0RhTySyJjOfOfXQsVpbTQdh3wRPwVQmznNvdSldrmM6WMOcPdDDQGeWBwTewbaCTRMahvzPqF+3SinKPLWdNkXJZ7BpX69TRyL4u56Roqf1calvzBf8TQ2e454bX8Yv/8Vv860d/syrBEnzf6Mxcb1J8u2yQQaKelKsW+wLwTlX9IYAfS/JF4JJ6NWwlsPfgMIlCg0kRFBiOpf0gSpljXMsnP8GLAH9142VFBUChDry/s4XuNndBg32zJjsMZoxjsWyuvrurylQiy/ru0KL2jwvWdXFkNMZ0MkvaUdoiNumsg21bdOWpVPJnvPnG3HO6o5yeSpGtIPdYs6oZF7rG1Tp1NLKvy5k4cqn9LNbWK7au4d59Q54NZ4G211r1GJQViKWynpck+E4unsDs7/QKxVUTU1UJ5QoXJxAs4MWSiEi2Tm1aMdz1yMGK9g/yfVniLRjzU8IUZj1WyC1b82+wwLVwOpkl67qM+vrdTb2tc36r0TPpcgn06vn13QMVVTl9CATF+u5QzhNnKuEZVUt558wz5m7urWjQWoklZ6u1XzS6r8s1KapFP6u14SzFm7OYs8WdD7/ARDzjlRUQQS3PwcUWL4PEcl3DBYWLiAQO1T8QkXuBB/DGvd8D9ta1ZSuAapJC5ofBzAt6Y1bIRCyK3pDFXAtV4ZXJJF2tkdz3NMNMuhyCGWN+MFtQ36bcCovlVOAsFB5L9VpbCVUb81lKBcuV1tdqqHU/KxEY1Qr+UgLs5EQcSzwtQMaZjb3LuPDLsRgXb/AmU/f+4HAulT/AFVt6eeCWf1tVf4ux2MrlUwXvP5L3epnSIjYvmVKVogoQfzkSrEhc1xtEC7Pi5p/Qc3vbit6QgWuhJUHlwll320Onp1nXFSVkW00/kw4IZoydLZ4Xneu7z3W1h8vuQylBUc8BsFnVjKVYysw86GcwQy62ol4N1PKaViIwqhX8pQSYo4Eb+vyg7nhGPXXdDw7zxJHxOZ89cWScG+/9l5oJmAWFi6r+ek1+ZRWy9+Aw5YkWcoLFtjyhIH4VwqBa4tr2CPG0w0x61jNmZDoFQEc0VHIG49UvmRt9eXwiyQUDHXzoLbVJMV9v8meMWWeKtKNEQhbnre1Y8gy52viMZophqRVLmZk3m+v1SqASgVGt4C8lwEIiuMo8wRJ4pt7/oyNMJYtbNQoFzlIoN7dYN96q5Sp/0w+AO1R1smYtWWFUYm8JLnF72Obm/20rTwyd4fh4nMmEF3sRDdnzLvZM2iE+Fqe3Lcyr13fltm9Z28bhkRnEFyzBd0dtYdtAJ/F0lp62yIp66At11eXMkBcTANUOiKt5IC12nhczNkPzZHhYSQR1Y05MJLzkmJbQEZ1bCDCgWsFfSoCdP9DBaCzFSCw9Z/+QJVgC0yUES62pxFvsOeCt/vv/jOct9h/q0aiVQKX2logtRMM2l2zsybkW3vPYS9zz+KGSddgVOBPPcMXWNbltt1/7mpwniDqz+YfWd3sG/ZViyC9GuQN7OftVMiDmC6qpRIb2qE13a8uix61Uip2/d3/9GdJZl7SjtEdsbr5yS+4+baYMD81M/n3UEbFz6VtUFXThMhnVqORKrXg+9BYvtummrxzwS2jjlfywvYh/y5Ki9t5aU65weZWq/p957//8bM8tVq69pb8jkhv4g0EKvJXPwVPTZRmuHvrZK7kHfed2r9jRvfuGePrlcURgXWdLzu12pRjyi1GuQChnv3IHxMKB9tRkkkTaIRqy6WwJFz2uWdVm9zz2Evf/6AgzaWeegMin8PzFklkmE1kv2DckJDIOn3n8MOAF9TXa9bpZz3c+hffR4ZEYWUfZ2Nuau49qPUlZbMXz7jeez2ceP5xLhJl1vZLa7ZH5mpJ6UG5W5IQf2wKAiPw7oLycJ6uUcu0tI7E0pye9U9Uatjk0PM2H9zzP4eHyBAvAL0bmrpJ2bvcqP977tssZ6GwhVCKKf6VRLJAt67g8/fL4nKjy/P2mEhmGRmK8fCY+pzhWudHdhZHZ0ZAFMmvzKjyuWTJMFxKkavfyUpETEPc89tK8fQvP80hstq+WWIQsK6ebh8ZmeGjW811I4X0UrBjy76N6rPaCsaBYFdhS0f4Xndtd0zaUotyVyx8CX/FtLwDjzFaHNCzCSCxNWzREKuswHs8wHk/jlO0NMN8wF7DS3EQL1QYiwnQqm5uNFs6QpxIZTkwkCVlz1V8dfnK/IPOs5WUHQyCnHivXSFq4wunriHJyMkEy66Cq845rVvtDkKo9ZHnzxWCmev+PjsxbvRSe51x6kDy9jSXkHEwaeZ816/kupPA+itheVuN03oNeOLlZjhXZrVdfMO/6X3JwmCeGxmr6O8UoS7io6rPApSLS5b+fyv9cRN6hql+uQ/tWBYp38wX3mSykfC2CZUnJG3GluMTmqw1sIVc5cUPPbLqaXZdtYPfTJ3IC4fR0EoD13S25fE/xdBYRz39/eDrp6bEFUGF9dwu2JbmsueUMiIUDbVdrmFTWKVlVslntD0Gq9nzyBUQ+hYI3cIO3827MQH0S0Kj7rFnPdyGF91FfR5QTEwkEZWgkRirrYlvC9ZeeCzTWcWS5rmO5ajHAEyqFgsXn3TVqz6rFcWcTxxWmfVkMC10RqoGFyJ+BjsbSXvSwJYzG0rkkgU8MnZmTH0vVEz6Bzhq8gSWWynLHdRfl8oKFLOFcf7/8gWchlUFAMZVPJGRzzw2vK3rcciZTrIT2iD2vemGhgAgozEO2sbc1J6RddXO6+Zuv3LIsbV+IZj3fhRTeRyFbaA1bIEIy6xCxhTXtYXY/fSI3UWy25Ke1ply12GJUXVZYRK4BPoNXH+Z+Vb2z4PMo8BXgcmAM+D1VPep/9gG8+jIOcKuqPlrOdzaKoPxuucxWopMVoRpYiPwZaC7Vi5/mBebWqgn6dON9+xn2Vy8BwcCyc/sAl23uXbKhuVKVz1KCEeupBilM1e4qCwqIwpVIuc4Ay02j08/ks9D1K3YfRWyvZlNhWYzgO6qNym9254aAWgmXqvzaRMQGPgv8BnAceFJE9qjqz/N2uwkYV9XzReQG4C7g90TkQuAG4CLgXOAxEQmehsW+s2EUnqggkW8kZKGuknG9gj6WJYSEXPT9KxMJBrpaSnowNTv5aoMg1QuQqxBZTCgsNrDUauCpROVTrf2h3mqQQBBUKyCK6eabgWaxK5Zz/QrvoyvverykAKnGA2+lxWBVpBZbgGpXLq8HDvv1YdLAg8D1BftcDwT2nN3Am0RE/O0PqmpKVY8Ah/3vK+c75/HLX/6S73znOwBks1kGBwd56KGHAEgmkwwODvLd734XgFgsRveBLxIZ9uSVpGe89yMveu9T03Qf+CLh0UMAWMlJ7/2Yl57fip+h+8AXCY0f5T1v2sadv3EO/T/9Mp3xE4Rti3WMs/aZL8HECVyF1vhpIvs/z6mXjzCdzJAefZmRR/6Kfp0A4Nlnn2VwcJCjR48C8NRTTzE4OMjx48cB+PGPf8zg4CCnTp0C4F/+5V8YHBxkdHQUgH379jE4OMg/PXmIaz+9j4tu/iSv/Y1dvPmuR9h7cJjvfve7DA4Okkx6q4iHHnqIwcFBslnPnfE73/kOg4ODuXP5rW99i3e+852599/85jex9n8xpzboPvkTWp7+Go6r9HVEGH/2MWL7vpQTCl/60pf4wAc+kFPf2If+mZG9X8mlkT+49x/48z//89zn1guPMvLDB3Kf//SRv+euu+7K/f6nPvUpPvWp2UxGd911F5/+9Kdz7z/+8Y/z13/917n3f/7nf87tf/GpXAr0X7/hj/iz/3t2/0e/fDe/GTmYU5v90+c/wVe/+tXc5+9973t54IEHcu9vvfVW7vrcl3KrzzPf+xwcfTKnBhkcHKzo3hscHOTxxx8HYGJigsHBQfbt28etV1/A4398OW8a/Uf+9s2d3Hr1BZw6dYrBwUF+/OMfA3D8+HEGBwd56qmnADh69CiDg4M8++yzABw+fJjBwUGef/55AF588UUGBwd58UXv3n7++ecZHBzk8OHDNb33JiYmAHj88ccZHBwkFosB8N3vfpe//8sP8sW3v5Yf3vZG/vPGM/z9X36wonvv1ltnS0898MADvPe97829/+pXv8r73//+3Pvg3gu4//77+dCHPpRTYznPf5fJ//VATo318U/ezcc//vHc/p/+9Kdz996m3jbO7P8HJn/yD7nPzzzxTaIvPpJTo43+6AGmnvpOzgOv+4V/5G//9m9z+3/oQx/i/vvvBzwBm9n/d7gv7s2p0tL/8hXu/Kv7cvsXu/e++c1v5t6/853v5Fvf+lbuffeBLxI9+Yz3xnXoPvDFiu+9UtRKuPyvKo/bABzLe3/c31Z0H1XNApPA2gWOLec7ARCRQRE5ICIHMpn51QzrTVfU5uHnTvH1A8dIZl029rbzrp2v4tyeNtJZF9uCjb2trG33ElIqyvBUkmQmiyr87uUba9aWiXiGj+x5nkPDsdzqamg0zvt2P8vPTy49EcPajmhOz694toBtAx246qW4ufDcrpL5wd66YxNvvmh9UbvJzu0DXP/aDVz32g1LLn0b8MpEgr0vjeRsXMmMw+MHh5dk4xqPp5uuBoyhfErVe1ko2v2Wq7biqpJ1NGfPc13l0o09uYlRa9gmlXVzE6NzelpLft+x8Th2Qc0i2xImErNj11gszZf/5WjOdX8sz818uREtw7osIuuA/wGcq6rX+iqpK1T180v6cZFdwDWqerP//j8Dv6aqf5y3z3P+Psf9978Afg34KLBfVb/mb/888LB/2ILfWYwdO3bogQMHym77ebf/U9n7FsPLKRbmTDzLeWvb5qh17rjuIj747efoaQ0jvgfPVCLDaCxFMuvy+vPW1Fw1cON9+3nm5XE/35mfFNNVxILXberlgcE31Oy3StEs+uTA1lOshHS156Ee31krmuW8NzPVXr/g3NZCpbdYG/LVZoXjCTDnGi/kinz0zrdU1C4ReUpVdxRuL9fm8iW8dC9/5r9/Cfg6sCThApwANuW93+hvK7bPcREJAd14hv2Fjl3sOxuOACMxb8ZxajJJX0eUrtZwzuBXzEU2ZMuSBqOFBpFj43Gyrksor1ywiJccsx6z68K2XLF1DbufPtEwfXJ+e0amU6zvis75PFhlFDN8X7KxZ9HBuZkM0/ksJQfb2SSQqr1+tXThDtowMp1kOpmd595cKiborkcOMpN25lzj5aBctVifqn4DPzDdV08tXDi+PJ4EtonIFhGJ4Bno9xTss4fZgM1dwOPqLbf2ADeISFREtgDbgJ+U+Z0NxQKcvLrWQTDgVCKTi0h/6fQUL5+J8+KpKV54ZZJDp6eZSmSqHowWi3Te1NtGyJpbHVPVW3bX2u2zWFs+u/cXZBynIa6Zhe0RgRMTSaaTs+qGwB22MAr+7scO8e6vP7Oom3ih+2+zlJquxiV2pUTN15J6Xb/88sZBBoqF2rDrsg2MxzNF3ZtLqe6GRmfmXePloNxfmRGRtfjOTiLyBjzbx5JQ1ayI/DHwKJ7b8BdU9XkRuQM4oKp78FZHXxWRw8AZPGGBv983gJ8DWeBdqur47Zv3nUttay3JD85XvFgNS4RTU0kc14vb6IiGGJ/xKipalue2W6jArGT2uFik8y1XbeX9u59lPJ5BxfslV6EnGq757LpYWxxXmYxn6OuYLb26XDaJwvas62zhxESCU5NJOqKh3Cx1NJaaHwXvOEwlsmxe057rSyk38WYMeK3GJXalRM1XQjnPUq2vXzWrxieGzrCxt7Woe3MpDzRgntBZDspdufwp3uz/VSLyv/DiTv6kFg1Q1YdU9QJVfZWqftzf9mFfsKCqSVX9XVU9X1Vfr6pDecd+3D/u1ar68ELf2cxkXMVRl5SfDHN9dwujsTThkEXEtmgJWWwb6KS7NZybUVY6eyw1q8kPOPzErkvZNtCBiCAinN/fzid3XVrzAaNYW6Ihi1RBTpzlCpbLb8900rNtua6SclxOTSVzs9S0oxTYU3MF4PJZSYb6aoIUF7uXFqKSmfpy0aiVWDWrxoXOfX4g51QizaHT0xwd84oLjs3MNeyXcu9d2xYu8UnllJv+5WkR+ffAq/12vaiqy+9etYrJup6NI4hIPzGRyHmGFAYaQuWzx85oiJdOT5Px9XHRkEVve5jz1nbk9lmumXWxGVZnS4hsXBtikwja47jKyYkkIhCyvTKxbZFQbhbb7uc0yxcwxR7SZowgL0UpW8IVW9dw4337i87ky43RaDa7WikatRKrZtW40LkPYoLueuQgR8fihG1hY08rqazL8LRX22Vte9RT69riebHlfbcAA10t1IqyVi4i8nbgP+JFyV8G3OhvM9SQNa1h0o7LkJ+uO5X1ypQWCzQsZ/YYzBJ3/MX3ePH0NGn/ZlIgmXU5PZWaUytmuSiVcuVdO19VVKdd79lu0J5Tk0mCsqGKpx7Ln0nefOUWv8Kfm0uTAtDVGmpIxuByuOexl7jko4/yqv/+EJd89NF5WZKL2RKCHG+lZvKlsiQHAunKux7nmrt/wPt3PzvPrpbONsauthBLWYkthWpWjYtlqN65fYCetgjnrW1j20AnXa0R+jtb6O+IMJNycte4uzXM5jWttEdswrbQHrHZvKaVWKp2qfjLtbn8m7zXLcCbgKfx1GOGCgjbkls9FDKVzJKJ51vUIatKb8SadxMtNnvM1+fGU9nZzLd5v2eLp8O9leVloajrwrYsR1Ry0J5bvvYUineN+jq8Gjmqs95yRaPgf33WW6zZMlMHafg9O9FsGn5gTjR+4Yr1xvv2LziTL3b9ClclQT2T9uisIMm6LtPJLP2ds21sBhVio+rVVOOBVk7GgvwV0XQyw8h0ilTWwbYsPnb9xezcPpBza97aP6u5CNyaa0W5arE59hUR6cGLfDdUSKmlooVnewlULq7O1ryeSjqcP9CSu+luvG8/h4anmU5m6W0L09cRnaPOuPbT+zh4ehpVaAlZc1L2i0A0ZKOqOFqem/Heg8Pc9cjBXPXNLWvbuP3a1yxpAC1XBbdcKotyc5WVSpPSDMKkkErS8OdTSl1z6PRUSVVZoUDKr2cSpCyK2lbOrhjQDCrERrmJV5vaZqFn557HXuKVySTHxxNY4nl8RkIWlggilZekWArV+qTNAI1PmdpA/vTqbfzlY4cqOkbw8oQVI3jkcuVIHZesam7FEVz0YBa/vquFsJ3izIznvrxtXRdXbF3DV/f/0vP48o9LZt3ZbMzMZmQu181478HhnBdZIPgOj8zwvt3P1sXYX0i9U67n2wU6oyEm/WjnWj5wjYoJqSQNfz7FZvKjsRTTKWeeqixYQZZTz6S7LcyZmUzTxfrUM3/ZYte+Gjtnqe8MVqqBi0kwdmSyLrYtrOtsIe043PrgM3S1hnM1lYqVlqgFZQkXEfkOs04xFnAh8I2atWIFcuvVF1QsXMrJ7plxFVcd8mVQUAQr0I/O1oxoySWCBPxYETdX1jRY/RTi+CnVy3EzvnffENPJLLYluch9cZVYanlcT+upsihUuSUyDgKErdo9cI1MNhg4IKgqWddFfVtS2yJuqcVmtePxDGvawyVXkKXqmYQsyRVdC9s279q5mSeGzjSdCrEeziz1uPZ7Dw7zvt3PEktlcVzPPT6Y6M2uVG0c0Zxgd4FN3a2IwOh0GgU2r2nzhbubU5XVmnJdkT8JfMr/+7+Bq1T19pq3ZpUSsSvL65kfYCl4rslhWxganSlaBvil4Zjv7eT6Bmdykiww4Fvi2RJsi4rcjIPI/fwCZ/WM3C+kniV2i7mCdrWG6W2P8rHrLwbgg99+bklOBI2s23HzlVtwXG+QcXV2chOyWDRYr9DI39kSYm178awFULyeSU9bmC197XMcNG69+oJFa+ysFupx7e98+AUm4hnU9bQc6np5Ae98+AVm0rOejN5kcPa4rtawV3JZPE/R5bgXy7W5/KAuv36WUGzFIgt8ls9AZ5TOFs+wDN6svS0Syhnqghsq6yjRkI2TdvxIVwiLkHU9D7Fo2GZzbyuxtFO2ambvwWEva4ALjrqELQvbkrpF7hejniqLkraF4ek5VTOfeXmcm77yJNv6Oyq2NTWykuKtV1/ANw4c4+RkEle9CUZfe4ROP15qoX4UM/IH7toj06lcTZ6+jkjODlOoZvnQWy5c8cJjKSrNelz7I2NxTzPhSw4RUFc5Mhaf5yofsrx6Mp7tRUlmHSwR+jpmJwn1vBcXFC4iMk3psVFVtasurVpFeEZ5IWypX55XfY8ki4zrzjm7gV0keP0ra9tyxtDRWApbhKNjcWxfmNj+ikiAk5MJelvDOfdlV70Zqm0JbWGLSNgm42rZy/NgSd8WsZlJZ3FcL94m5EuuekTulyIY6IIH/YPffo5N+5YuZEqp3NJZl+5WwXGVVyZTiHizxKNnvCzR/R1RplPZsgabRnki5RDhwnO6cglQgTkecOWSn8HBTxhBOutycjKJo5qLn5hMpOnviHJsPJ6bEa9UAbNUtVbhtZ9KZDg9nUTVE9aV3r97Dw7POkQ4ngo3ZElOq1BYMC7QWJzbFWUykaE9EqItYtOVJ/DqeS8uqBZT1U5V7Sry12kES3l0RG22rG0DkVyEt2156fMLT37OqOXfGLavrx6ZTjISS9PdFmJjTwsZR3HxLl7U9hwALISZtMPG3lYitnfDBeqvc3ta6W4NV7Q8D5b0/Z0tbOxpo8W3DDtK3SL3F6IeUdSlVG5BVtmRaU+wWOLZm9JZl4l4hiOjM2W3YV7U9PA0R8fiTMTTyxKdXqsywTu3D7C2PULIltzkKBzyVrJTiSwiwnQiw2gszQunpjkTS3F0LMb7dj/LtZ/e11QR+eWyVLVW4bU/MZEg6yjru6IV37+BrSUfxbPRZhxla187t159Ae9+4/m0hm2yrrcqec+btvGjD1zND297I/fc8DoiIXvZYrIq8hYTkQG8OBcAVPXlmrdolZHMumRc5dXrOhmb8by7IjaIeIP1VJF6EJ4A8jIgHx+PE087tIYtTk+lch4gtkDItujvjHJywgv+SzuKbQnru1vnJNVbqCJeKfKX9F2t4VzMx2QiwyPv/fc1ODOVUQ+X5FIqt3v3DTE8nfRUP76OIfCyswQc1dxgs1gbgt+48+EXODqWIGxZbOxpIe24y2LYr6XLaSztcH5/R24VdPDUFJZfqno6mWEkls7tm8i6xCe9lCNTiQwXrOtsmoj8clmqWiv//nr65XFClrC+2xs+T00mSWYcbvrKAXpaQ1ywrmvR3ICxVBZboNDhVIFrL14PLFxRdLmrepbrLXYdnjH/XGAY+BXgBbwSw4aFUM0NiBHbRiRD2oHLNnczPpMikY6Rmev6T9aF7evac6n1L/noo8RSzhyXYkd9l9LpFK0hYTrtGW1HplPcfOWWOTdMNaqZhqtzCigVGHZ8PMHeg8NLEjDFjv3wnuexRchk5xrDHVdpyXOqKGew2bl9gHv3DTGQzDCVyHJyMknEtuhqDVUkHKvR/9dyQCm8J4Ia8RHbYmQ6NVd/nvcm7XjehZ0t4brEKNWLWjwDwf0VTPBiqSwnJ5Io6gkJVaaSWY6MxhYUvMfG4zjubGhCPlZeMHQ9XJ+rpVxvsY8BbwBeUtUteBH6++vWqlVEcDNMJTKcnEzguorjugxPJzk0EqOnLZzTYcOsh9jt176GvQeHuebuH+RWN8WSJKayDpMpBxR+ZU0r/Z3RXArugEo9rvYeHGZ8JsXRsRk/1X+66DHLmYQwUO9MJzOcnEiSdXVOYFgtfzvwlurriOD4giVsedfF8StnBpQ72BwanmZ0Ok3W9VaXWVcZnfbUZOVQC7VgOa7wC1F4H3W2hHDVS3+Tyjpz3d4LvAtHpr1VTDNE5JdLLT0Vg/t3ZDqFqzonS0fGUU5NpRieTnLnwy+UPN62PJWkp/KeHSuCc9pspRDKFS4ZVR0DLBGxVPWfgXmVxwzzCYTLaCyFhZdtOBqyPf2tZTGd8ry32vwYlmjY4tXrvJQM79v9LIdHZhb8/iBGLRKy6GqNFNULV1KLIrhBM66ysacVBI5PeDPt/GNqeSOXI6TKzf9VK3ZuH2DTmnbO6Y7S7ukxifjeY5OJTMWDTTrrgm+/cRwl43jq0vF4pqxzVq3+v5bXqfA+2tLXwbvfeD49rRGcub4pc+oChS3JxVw0Q0R+udSyhktw/yYyzpyMGflksi6HRmIl7/9gUqNKLmbJsoSw7Xno3fK1pxieTpJ1dNnd3otRrs1lQkQ6gB8Cfyciw3hR+mc1fe1hRmcWTg6teDl70o4XKY8K/Z2eK+C6rijHJ5KEbGFLX3tOH377ta/J07EK2RJzzsA2E9gBAorNDqtNt9LVGiGeztLTFplzfK1sIOV65JSb/6uWHBuPs7Y9SjRk57nfeue60iBLV12yjpJxCqLjVcuyQyyk/19IFVJrW1XhfbT34DC7nz7Buq4oYzPpeXnzAueUsCVNl9SzHGqlRgru39//0pMl93GBqGWVrAX0yV2X8sF//BnHJ7xKktGQRWvYm6AOdHrPqSXCyckE4NlKG7lSLFe4/DNeeeF3A2/zX99Rr0atFD75u69d8GYBLxraM8wncgGRgXtxyLa4YKCDnrbIHH34vx6fWLDGdUB/R5RYKpvTewcsZXZYrhGz2sGukEoGv3Lzf9WKTb1tHB2LMRbLeO7IluC43uqjkqjmvQeHyTh+TELe2GsLtITt3OxyMZfmo2MxphKz1zsaskhmXG76ypOELYt1eV5IpdKyQG1VU8H1625tIRqyOTWZIOUoAmxe46V7H49naIuGGOhsaZqI/Ebwr8cnFvxc1Z9wlrg2O7cP8KPb35R7vo6Px/0VVYS+jhamElmyriLqaUq6/KwTjVoplitcQsB38SpBfh34uq8mO6sp5yEJKiZ0RG1iKYdU1qFDZ6sbfugt2+d8zz2PvcTdZaSVWdMWpr8zimXB8HSazpZQLs3GUmaH5RoxS+WfmkpkuekrBwjbwrrO+YNdIZUOfsuZZPCWq7b6KyXFwouGBuhtWzwIMZ979w2xpj3MWCyTUw8Jnsq0vzM6p7+lBPMVW9fwk6Nncql9klmHmbSD7evfU47LL88kaPHr9JRKywJe4aiZlMOVdz2+5FxnxbwKpxJpTvmejVv6OrizDgKlUfnalsLf/mBh9VRLyCJkW4tmJs5fTeV7ghZ6jjZ6pViWzUVV/1xVLwLeBZwD/EBEHqtry1YBlkAi43JkNMY53a30+on78qsbFj4Q9//oyILf2d8R4U+v3sar13cxmchw3lpP772lr6Mmtb3LNWIW7hfE4mRdT3WkLpyc9PS/C+l9K43DqKUefDF2bh+gI2oTsS0cP6XJud2t9HWUnl0WI1CvndvTkoueDlZCnS2zs8uF7CNPDJ2hvyNCxLY87zUN3KI978L8RKWnJlMcOHqGK+96nPGZFFOJTO46jcaSDE+naYvYNTH6Frt+Idviss29dUvx0myG63KJZxZOGJrMuvxiZIZjZ2bK7kv++e9sCfv3mBeXVc9noxwqzYo8DJwCxoAltVhE1uCtgs4DjgJvVdXxIvu9A/ig//YvVPXL/vbLgS8BrcBDwLtVVUXkE8D/AaSBXwD/RVUnltLWanHVk95eDYsW+jtbaPfVA4GbcSEzQfoW5kbsg2dn+IQfvFhY9yT/fWAgr2ZWV67rav5+h4an/UzMSsb12mlZAq63mtnS117TlchyulNesK5r3sw/ns6WrWoIUuicmkwSDVmsbY8w4WdeLrRDLKQiPDYep68jSr8/qw1iTFLZ+fY4BbJ+NoZExrufIrbFZCLDTMqhvyOS+55KbTClqkuWun71WGE0qnLkUgnUooXPdT4hC05NpcrOOl74/NiWMNDVWKESUG4lyneKyF7g+8Ba4A9U9ZIl/vbtwPdVdZv/vfMSYfoC6CPArwGvBz4iIr3+x58D/gDY5v9d42//HnCx376XgA8ssZ1LwsWbkQC5QeYnR88U9YoqNlvJdznc8StrysoHttRZ3c7tA2UlF9y5fYBbrtrqeTDhCRXEi2twXEX8ALtmWYlUw1LcUYNr0R61/XILLhOJDO0RG0skZ4cI+rtQRcTCFUKwgimF57LqDcDdrWF62iL88LY30tUanpNbKv83yu1P/r21++kT7LpsQ8kKovVYYSylcuRyus8XssEPoCx22TzjvE3Y9vp1ZibNLV97atE2NvPzU+7KZRPwHlX9aQ1/+3pgp//6y8Be4LaCfX4T+J6qngEQke8B1/iCrktV9/vbvwL8NvCwqn437/j9wK4atnkeC81CAlQ1F+cCXsqWQltE8CB2Rm0mErNxLcGXd7eGyhrQlntWd+++IdJZB1eVbHY2vCHjuH4WZqnJSqRROvalBCEWGrtHplMksw6uwr1vu3yOW/eN9+1nZDrF6HRqjtNHIJgLZ6hdraFcXfTFyB94lxIYWOreemLoTNGVeL3uxWr7sJBnYtDeet5ff/Hbv8qtDzxNLO3kEokG2o0gP5jjas5VOZ11efLoGZ5+eZx37XzVgpH3zSBMCik3K3I9Zv/rVPUV//UpYF2RfTYAx/LeH/e3bfBfF24v5P/CU70VRUQGgUGAzZs3l93wfNa0hRmLL+yO7PreGwCCt2wtfNCCB3HTmnbCk4k5qTQ29baW7Z0UGFiDKPa04+bqk1RKOQP6S6enmEpmsfBWaYGg9bIIKFvXtC+5amUja6JA9Q9vvrG7syWcy24dXIvCiqLru6KcmPCqCG7oUUK2lRPMO7cPsOv4xJwSy9ddsp5//OkrRdWo+QGN+Qb8pRREq9T5ol6eatU6dRQKO8dVhqeS3PyVJ7EtizXtYda2L+6EUi07tw9wz42XzZmoeMHKcU9d5lcKzSdkCY4qn937Cy7Z2NOUQqQU1VaiLAvf6L++yEd/lv/Gt5UsNYC48Lf/DMgCf1dqH1W9D7gPYMeOHVX9/kBXC2fimQVXLyLezKolZDHQNTsrzX/Q8h/E9d2trO9uzQ1EP7ztjWW3p5j7bMZVppPZitKklDugB3EN4ZCNOC6OXz3TtoTPv31HVQ9DoVAbn0mtSB17qRl2RzSUO7fxVBZXldFYGlvA9S3zJyeS7DhvTU6wBPEk/Z1RNvsD6lMvT7Kxt5WTEwmvkml+4KLtJT0dm0kxPJ2mvyOy5IJola4Y6pVCqNrVZGEKocCzynFBRBmLZYiG7IrT1FSyqi4WJxRkm1aZm97Fy3jsBe5mHLfp7/dCyo3QrwpVvVpVLy7y923gtIicA+D/W0yxeAJPJRew0d92wn9duB3/+34f+N+B/6SqNRVahUynsmxe0zqnME8+AqzrjNIStnEVTkwkGBqJMZ3MzHnQapW99partnJmJoPixRoUus+WS7kR4ZGQBeoNirYtXrZcS+htC1ctWAr19IdGYmSduTO6lZBGpJS9RlVz5zbjetcp6yppV4mELEKW4PrHFwuGDK5HxnEYjaVy6lNLPHfWrqjNq/o75hnwCwuiVerNVan9qZ6F3sq1C+aT/4wFGa/FX+PZfiqhIE1N1nF5+uXxRW0zS7Ur7dw+wCd2Xcq2AS8haKAeC1ue1x14TgBR22r6+72QugqXRdgDvMN//Q7g20X2eRR4s4j0+ob8NwOP+uq0KRF5g3gpWt8eHC8i1wD/DbhOVet+NTb1thGyLTavaZsTyAieYOnriBANWSTSXtoHwZuFHB9PMJXI5B60Wj2ItXSfLcdoum2gk77OiLd8d5WQJfR1Rtg20FlRuwOKCjXL4rT/0AfUOjisHobeUsbWWNrJnduIr/oCQL3BTpB5grzwegxPJXllMkUi46k9w37Ay5a+du658TIefs9VSzbgL9SfU1NJr1hdKsO9+4aKnq9mMzbnP2OprFf+2cUT6IFaKu24TCUynJhIIrCowCg1CbvrkYNl3087tw/w8Huu4sW/uJYvvuPfEA1ZIIKiuKqoQndbeMWkzQmoq1psEe4EviEiNwG/BN4KICI7gD9U1ZtV9YyIfAwIwuDvCIz7wDuZdUV+2P8D+GsgCnzPTw2+X1X/sF6dCPS/YVs4pzvKy2cSXuVHW+huDTOVzM7aT1SxrNlBf237bEqVpRiOC1mq+yyUr9II+r++O1SToMZievogTU69AifradMpZq/ZtG/23PZ3Rjk65g30IuC63oB3rl9uIXdM3vWYTmYY9oWtt2L2BqL+jgiqOltQrbeNDr86Ya1UU0FfPrznebpbvbo3C52vauxV1ThvlHPM3GcsgYiXmw68Ynu4nrrw9LSXXmV99+xqr5SarNj9mnVcjo7FOW9tW8X3087tA7xr56v47N5fkHFcorZFd3uYsG2vqLQ5AFJnrdGKYceOHXrgwIGqjs1Px9ARDTES82wEw1OpebUXbIFNa9ro8A2rldhTKmlPMFjmD8aVzBor+Y78/i+1RkRQTrdQMEZsa16anFrNgEv95kLxSEuh8Ny+dHo6V0iuNWzT1xElZMuc388/5pWJBHG/TkPY8orFua6XMzurynlr23PXbCrh2QO7/TxT+dcRKveQ2ntwmFsffIZ42iHkF7Nz8dRKW/raefg9V83Zt5rvr/TercUxo7EU4/EMnS0hppNZ1ndF6WqN5PYvZf8sdu8cOj0NwpzVe6X3096Dw9z1yEGGRr0UjlvWti3ZMaZeiMhTqjovkXEjVy6rhmJGulsffAZH5wdOuXh63XrWoK/FKqiS76ilK2QpT6DCNDm1ZLnr3Bee24HOKKenU7ko+1TWQZk7U80/5mhQR11m9fLiB1RGQ9YcxwegqGAGKl6tBQPyTNrzDkz6AZxhy6uI+MKpaXb8xffYNtCZC66sdDVYjftyNccUXoP8NDWBwJhKZBiNpXJF485bM/95LXq/ui4be1qX7LEZVJYNvnclFVoDI1zqws7tA4Rtz1Sos+p071+FeNobMK/YuqbqSPpy2lAPlU69qaV6sFwaURgtOLfBgN3XEWEyniGZdcjGlXft3FxSxXTjfftzHoGuesGqjnpZ7NZ1zbevTCYyc1YU4M24qx3EW0I28bSTmzDlF7uLp7IMTyf57N5fsKY9THdrZZkAqhH01U4OSt3ft1y1lfftfpaJeCZXaynrKGMz6Xkel8Xu17AlTCYzS/LYXKlZCPIxwqUO7D04TCzlLOiefPnm7qpmdstBo5MCLrdQCwaTExMJHL+YV0c0xIfecmHdf/vefUNkHIepRJaMq7SEvADJoLJgqfZ+eM/zrO2AyXiGlOMSsiw29bbkVjIB+XnL8q/poeFp1nfNTZBYOCAXHvPS6SnO6W6lIxryqqAWELG9QbQtEsJxlcl4hr6OlpLfX4xmqJq6c/uAl3E8mcVRJeKXE7et4tmrC+/Xex57ibu/fyiX/80CxPegLHZ8sedtuVfT9aCR3mKrliALbqiIf7JXRzvKo88PMzyd5OUzcY6Mziya4HG5WKlJAZeKV2vH062jc4MQ60k1FSoDL6zz1nawtiPK689by71vu5yPXX9xUY/DK7aumXdNp5NZxmZKe+AVuw9iKYcTE3HG45l558dzVpOcx2Q0ZJEqcB8vZ8CvxmuyHi7P06ks5w90sH19F1v7O+hs8WxWh05PLegFFsQjBSmbXPWSi/b4XnuFwqHU8xY4YuSzkgqtgVm51IX8IlOBJ1DA2nYvs2084/iVDYWso5ycTHBud0vDZyarYTleKffuG6KrNcz67tbctuXqc36FSvA9xkS97QtQanVXTKVY7JoGGbrbIsW9/Iods6Y9zKmpFCFLCPmqngBHwfZLCAB0toTIxrViD79q1KL1UKWWKikxnXLmCYJ8bcNdjxxkeCrpqcMFIpaF4KnCiwmHUs+biJBx3GUpLVEvjHCpA8GNCbMz4KD29Xg8A3ieQV7QlB845cLJiQStkVBN6mxUS72X441WuRWjkSqIsC0kMp4Lcn4xsYhd3dqpmND54Lefm9e/vo4oWcf1C9nNH5CLnZO17VFOT6WI2BZpx6UlZHmZsB3PY2xtR5iOqDc4RkI279q5mSeGzlQ84FejFq21KrWYoX48nmFNe7jkxGvvwWFeGo5hi3dd0476xnxIZYvn2Ct1700mMnzs+ouX1fZYa4xwqQPBjTk8lczV3AAIiVc8LOMo/R1hJhMOLprLKeQo9EXthtpgaqG/zhcgnVGviFks7dARsRmbSdPVGm4qO1MjDPoBF6zr4shojOlklmTWRf2gubSjFaXrWYhS/du2rquka+xCqWv6O6O0RUI5bypHXaK20NMamZdSppTdqNkpthqaTGRY2146IPXOh19AVUm7viefH1iccaGrxS7qGr3QvdcIh5paYmwudSDQiStgWUI0ZBH1I6gjtkXI8mrTn9vTkrsBvZobXl34hdKt1Jul6q/zdci2wKHhGIdHZrAFjp7x9PVeOv7G9bGQeqYpKee3I74R38LzLArZQlvEXpKtKz/jwEQ8zWRewbBq7RhTiQxdUZujYzMcfGWS4xNx0o7rRbK3hTk5mWA0luInR89w64PPcM9jL1V3UpqEwhQz2wY6S9pB9h4c5tBILKepcBXfjga2Bffc8LqSnmmNuvfqjREudWLndq/e++Y1bVywrpML1nexfX0X67tb2DbQScbxDLhb+trZvKYNEeGc7oW9d5ar3UtJ2ZGvQx6NpbEtz640GkvjuJ46cCQvlUszeMA0Mk1J8NszqdmiXud2t9Lf2VK14C00EgcCIIi1KKd/heckYlte5olIiI09rTgKjusFT27obcV1YTKRJZlxCVneoPuZxw+veAGTz0KC4N59Q4Qtr0xx2JJcrkFXYVt/x8LxNk2UIqeWGLVYHVkoIBDmLrkDPXY+jfIOWcpyPF+HHASfBa+9PFrunH42iwdMrVUQlWbK7WoN5yYZAdUK3mKp5aeTWaaSWS7b3FuV7ePG+/aTdtzcd9qTSUKQu75B7jcFXNcL7sy6Lvf/6EjJOiTFzlHQ/mayyQUs5DjwwW8/x7quKK9MprD8CZXrKo7C7de+ZtHvbZY+1hIjXOrIYl4shVH91dSoaDbydcgR2/IKH6k3I+/riHJiIpFLG7JS+7gY1eQqq6Xdp1RqeYWq7VyFhudgopDKuv73z5JxlYzrqY8yTvHAwWLn6H27n0Wg6Wxy+ZQSBMH1O7enJReVb4vwqv72ovs3o2NLrTFqsTpTqLdd7cvjQHUwMp0k6yqprOvPeC1CttDTFmZLX3vd+tjIMrYB5ZYryKeWuvdSqeUjtlW1nauwJERfR9SL4fBzmpXCEorajoqdo1gqy3QyW9F5K6RR1z+4fvmq7oGuFm67ZnvRNp4NsWRm5dJErIbl8c7tXsXEz+79BVnX8yJS4Ezc87T55K5L69bHRlesDKjGtbmWsRr56thU1sES7xoE0fLVqNsKVbzBRGHST4wZtmW2dEAea9sjOQGR35di58hxvVo3+VTS1kZe/0qu39kSS2aEi6HmPDF0ho29rfOyDPe2R+v68FT60NZLNVGtiiv47aBNwYy9mpiPYqnlu/zBvBp1W7HB80NvuZB79w3l+jo8lZxTd6e/I5KrqFooIIqdI9sS0LnxPZW0tdGDdrmTw9WQ2qUcjFrMUHPKLTTWyN+tp2qiWhXX3oPDvG/3szxzbJzTU0meOTbO+3Y/W1WbAnXsu3a+ClU4PhHnF8PTjMaSVavbiql489WgsVQW8AKHA8ECpesAFZ6jjmiIzpZQ1arBRt13lVKrqrPNjhEuhprTqIenkt+txC5SqR6/WvvZnQ+/wEQ8g/ouvurCRDzDnQ+/UEbv5xPkueptC9MSskk7ypmZDLsu21CzmfzO7QPsumwD435G56jtueGOxzOcmoxzaHiao2NxJuLpOeet2Dn65K5L+cSuS6u2O66UQXs1x7bkY9RihppTygW7ng/P3oPDTMTTHB2LE7aFdZ1RQn4J4WK/W65qolo9fjX2syNBnRZrNs+YusqRsepm3oEA7W5tod+vWxVPZxfMuFwNhWpQz0MtwUgsQ0vIYmNPC2nHnXfeSp2jagVfI+67aqhHLrRmxAiXs5x62B2W++HJFwAbe1o4PZXi+ESCbf0dfOgtFy7oOrqYXaTRevylUKluv9p7ofB3OlvChKykF0C4bm41xnqet5U0aK8G553FaJhwEZE1wNeB84CjwFtVdbzIfu8APui//QtV/bK//XLgS0Ar8BDwbs1zNRGR/wp8EuhX1dG6dWQFs9y14+tFoQDoao0s6kBQ7ix3OY2vW/vaOTQcQ3Q2iaUX4d1e1fdV4liwlHuh2O+kHJeWUO3tH4sJwEYN2mdD3EqlNNLmcjvwfVXdBnzffz8HXwB9BPg14PXAR0Sk1//4c8AfANv8v2vyjtsEvBl4uZ4dWOlUE4+xEI2KMajGkFuuXWQ59fi3XbOd3rawX/nQS9nS2xYuGitRDpXo9pdyLxT7nZBl0dkyd+661PPWrPEhzdquRtNItdj1wE7/9ZeBvcBtBfv8JvA9VT0DICLfA64Rkb1Al6ru97d/Bfht4GH/uLuB/wZ8u26tr5ByZzbLOQOq5ay8kplvrfu4FNffxX53OfX4O7cP8Ildl9ZMrVOJmmgp90Kx37n+0nPZ/fSJmp63whVq1lGGp5Pc8rWnKkprU2tWsuq0njRSuKxT1Vf816eAdUX22QAcy3t/3N+2wX9duB0RuR44oarP5udpKoaIDAKDAJs3b66iC+VR7sC73EFgtUw5Uu4DVus+7j04zPhMiqNjM4Qti3VdCxvyK2W59fi1VuuU+31LvReK/c4lG3tqdt72HhzmwNEzZFxFgJAFiueZ5qo2NFXM2RK3Uil1FS4i8hiwvshHf5b/RlVVRBYqOV/u77UB/x1PJbYoqnofcB/Ajh07lvz7pSh34F3uGVAtZ+XlPmC17ONcQ34rp6dTHJ9IcsFABx96y/aautuu9hloPVZotTpvQfxPNq/yZcYF8EqDR0N2Q1cLjawH1MzUVbio6tWlPhOR0yJyjqq+IiLnAMUUlCeYVZ0BbMRTn53wX+dvPwG8CtgCBKuWjcDTIvJ6VT21hK4siXIH3uWeAdVyVl7uA1bLPpYy5Pe0RVa9MKg19Vqh1UIFeu++IWKpLLZAVudmMss6yoYer4BXo1YLK8UFerlppFpsD/AO4E7/32L2kUeB/5FnxH8z8AFVPSMiUyLyBuDHwNuBv1LVnwG5O1dEjgI7Gu0tVu7A24gZUK1ml+U+YLXq43sffJonhsZy77uiNr/S19HU6ojl9iiq9PdqvUKrlQr02Hgcx1VCtoW4iqNKsIixLaGzpfq0NrVgJblALyeN9Ba7E/gNETkEXO2/R0R2iMj9AL4h/2PAk/7fHYFxH3gncD9wGPgFs8b8pqNcr52VHLlbrvdVLfr43gef5ls/fWXOtqmUwy9HY02rjlhuj6Jm8GCqlTfipt42bEtQ9erEREM2IX/kCso3NPpZKTf7+dmEFGYhPVvZsWOHHjhwoG7fH8wiF5vZlLvfSmapfXzVf3/IK5XM/GTvm9e0LWupgnJXBzfet3/eii2ezjLQ2VKyjv1SWO7fK8aVdz1OT2t4TgE0VWUykeGHt72x7O8JbC4T8cycCo9tYYsNvW3EUtmy7iMTi1IfROQpVd1RuN1E6C8T5aoczgbj8VL76AQ6EQEp0MEvt2ApV+2z3Pa0ZvBgqpUKdOf2AT6561LufPiFXCqcbf3t3HbNXKeNIM6qmPBolnIMZxNGuKwQ7nnsJe7/0RFm0g7tEZubr9xSsnzsase2ZK6AwYtmty1Z1oGiEs+35banNYMHU74dLuu4nJ5KkXFdwpYUrU65EItNSBYTHpW4ypvVTW0wWZFXAPc89hKfefwwiYxDyPIGic88fph7Hnup0U1rCNdd4nm3q87+5W9fLirJDLDc9rRmsN8FdriIbXF8IgkCG3taybhac/vPYvadcq5VM9ipVhNGuKwA7v/RESyBkGVhieX/620/G7n7hsv4ndee4xWXwlux/M5rz+HuGy5b1nZUkhqm2jT81RL8XtgSDg3HOD6eoD1iL35gHdrR0xbhvLVtbBvopKs1suQ0Q8VYTHiUc61qnQ7pbMeoxVYAM2kn5x0TYIm3/Wzl7hsu4+4bGtuGSuMbGmFPi2dcNva25tq3Uso+V8piasByrlUz2KlWE2blsgJoj9i4BW5RrtKQmahhluVejVRKs8zElyP5ZzE14FQiw/hMiivvepx79w2x67INC16rlVJsbKVgVi4rgJuv3MJnHj9M1nX9XEre381Xbml00856mtm7r1lm4ssRwV4YyNgRDaFAxtWc/WT30ycWFP4m0r62mJXLCuDWqy/g3W88n9awTdb1Boh3v/H8s9ZbzFAezTITX64VXn4gY09bhO7WcEWrtmZfia40TBClT72DKA2G5SbfPTd/Jn42DJi1CuA0LI4JolyhGL97Q7WczTmvmiHO52zHCJcmxkQVG5ZKM9uE6klgPxmZTjKdzJLKutiWcP2l5za6aWcNxubSxDSLt4/BsNLYuX2AXZdtYDyeIZl1iNjCmvYwu58+YYIilwmzcmlimsXbx2BYiTwxdIaNva3zkneWU1DMqKOXjlm5NDHN4u1jMKxEKknPk49JA1MbjHBpYpohP5TBsFKpdnJm1NG1wQiXJsb43RsM1VM4ORuNJTk+nuCl01PceN/+kiuRalc8hrkYm0uTU06qcaMbNhjmk++KfWh4mulklt62MH0d0QU9L40bc20wK5cVjNENGwwLE0Tt97VHEIGxmTRHRmfIOlpS1WXU0bWhYcJFRNaIyPdE5JD/b2+J/d7h73NIRN6Rt/1yEfmZiBwWkXskLxRXRP5ERA6KyPMi8j+Xoz+NwOiGDYbF2XtwmEMjMVxXsUXIOsrJyQRZxy2q6jLq6NrQSLXY7cD3VfVOEbndf39b/g4isgb4CLADr5rtUyKyR1XHgc8BfwD8GHgIuAZ4WER+HbgeuFRVUyKyau+IY+NxbIGhkRhpxyViW/R1RIxu2GDI4959Q4QtCwVEBBHAhdPTKV63qeic9qwNPq0ljVSLXQ982X/9ZeC3i+zzm8D3VPWML1C+B1wjIucAXaq6X73kaF/JO/6PgDtVNQWgqqtWR9QRsTkxkSTrzM7ITkwkTSp+gyGPY+Nx1nVFUQVXleB/RtVVXxopXNap6iv+61PAuiL7bACO5b0/7m/b4L8u3A5wAfC/iciPReQHIvJvSjVARAZF5ICIHBgZGam2Hw0jpwmUvL/87QaDgU29bYRsi3N7WghZguMqlggXDHSY1UkdqatwEZHHROS5In/X5+/nrz5qlZ45BKwB3gC8H/iGlBhtVfU+Vd2hqjv6+/tr9PPLx3Qqy5q2MBnHJZlxyTgua9rCxFLZRjfNYGgaAgO9bQlb+trZvKaNga4Wbrtme6Obtqqpq3BR1atV9eIif98GTvvqLfx/i6mvTgCb8t5v9Led8F8XbgdvFfMP6vETwAX6atuz5qAzGmJsJu3pkvGk89hMmo6o8TA3GAIKDfQR26ItbPHBbz+3YLyLYWk0Ui22Bwi8v94BfLvIPo8CbxaRXt+b7M3Ao746bUpE3uCvSt6ed/w/Ar8OICIXABFgtG69aCDTiTSOgqonWFTBUW+7wWCYJXBJ/tj1FzOTduZUqDTu+/WhkVPcO/FUVjcBvwTeCiAiO4A/VNWbVfWMiHwMeNI/5g5VPeO/fifwJaAVeNj/A/gC8AUReQ5IA+/QVVoRbWQmgy2zgkXEW8GMzGQa3bSqMAGhhnqT774P0BYJlZ3M0lAZDRMuqjoGvKnI9gPAzXnvv4AnMIrtd3GR7WngbTVtbBNjW4JtzS5AHddtYGuqx9SuMSwHJtP48mEi9FcwW/vacfPcK11VXPW2rzRMQKhhOTCZxpcPI1xWMLdds53etjACZB0XAXrbwivSC8YkCzQsBya1y/Jh3IpWMDu3D/CJXZeuihrpQbJAx1VGplOkHRdbPNdRQ+NYbXaw/GSWK/2ZaXZkldq6K2bHjh164MCBRjfjrGXvwWHev/tZxuMZLD8qyVXoaQvzyV2Xmoe/AeTbwVrDNomMQ8ZRk2fLMAcReUpVdxRuN2oxQ1Owc/sAa9sjhGxBgbBtsaGnle7WsLG7NAhjBzMsBaMWMzQNsbTD+f0dc9LXqKqxuzQI41llWApm5WJoGownT3NhrodhKRjhYmgajCdPc2Guh2EpGOFiaBpMkabmwlwPw1Iw3mI+xlvMsBRWm8uuwVAuxlvMYKgTgcvu8HTSJEM0GHyMcDEYlohx2TUY5mOEi8GwREzqGoNhPka4GAxLxLjsGgzzMcLFYFgixmXXYJiPES4GwxIxLrsGw3xM+heDoQbs3D5ghInBkIdZuRgMBoOh5jRMuIjIGhH5nogc8v/tLbHfO/x9DonIO/K2Xy4iPxORwyJyj/jZDkXktSKyX0R+KiIHROT1y9Ung8FgMHg0cuVyO/B9Vd0GfN9/PwcRWQN8BPg14PXAR/KE0OeAPwC2+X/X+Nv/J/Dnqvpa4MP+e4PBYDAsI40ULtcDX/Zffxn47SL7/CbwPVU9o6rjwPeAa0TkHKBLVferl7/mK3nHK9Dlv+4GTtan+QaDwWAoRSMN+utU9RX/9SlgXZF9NgDH8t4f97dt8F8Xbgd4D/CoiHwST3j+21INEJFBYBBg8+bNlffAYDAYDEWpq3ARkceA9UU++rP8N6qqIlKrDJp/BLxXVf9fEXkr8Hng6mI7qup9wH1+W0dE5JdV/mYfMFrlsSsV0+ezA9Pn1c9S+/srxTbWVbioatFBHUBETovIOar6iq/mKpbl7wSwM+/9RmCvv31jwfYT/ut3AO/2X38TuL/MtvaXs18xRORAsaygqxnT57MD0+fVT73620ibyx48QYD/77eL7PMo8GYR6fUN+W8GHvXVaVMi8gbfS+ztecefBP69//qNwKF6dcBgMBgMxWmkzeVO4BsichPwS+CtACKyA/hDVb1ZVc+IyMeAJ/1j7lDVM/7rdwJfAlqBh/0/8DzIPiMiISCJb1MxGAwGw/JhioXVABEZ9O03Zw2mz2cHps+rn3r11wgXg8FgMNQck/7FYDAYDDXHCBeDwWAw1BwjXCpARK4RkRf9fGbF0tVEReTr/uc/FpHzGtDMmlJGn/9URH4uIv8qIt8XkaI+7yuJxfqct9//KSLqO6GsWMrpr4i81b/Oz4vI3y93G2tNGff1ZhH5ZxF5xr+3f6sR7awlIvIFERkWkedKfC5+nsbDfp8vW9IPqqr5K+MPsIFfAFuBCPAscGHBPu8E/tZ/fQPw9Ua3exn6/OtAm//6j86GPvv7dQL7gP3Ajka3u87XeBvwDNDrvx9odLuXoc/3AX/kv74QONrodteg31cBlwHPlfj8t/C8bgV4A/DjpfyeWbmUz+uBw6o6pKpp4EG8/Gj55OdL2w28KcjWvEJZtM+q+s+qGhSL38/c4NaVSDnXGeBjwF147u4rmXL6+wfAZ9XL74eqFgt4XkmU0+dVl6NQVfcBZxbY5XrgK+qxH+jxA9yrwgiX8imV56zoPqqaBSaBtcvSuvpQTp/zuYnZeKOVyqJ99tUFm1T1n5azYXWinGt8AXCBiPwvv5zFNaxsyunzR4G3ichx4CHgT5anaQ2l0ud9QUwlSkNNEJG3ATuYzY6wKhERC/hL4Pcb3JTlJISnGtuJtzLdJyK/qqoTjWxUnbkR+JKqfkpErgC+KiIXq6rb6IatFMzKpXxOAJvy3ufnM5u3j58hoBsYW5bW1Ydy+oyIXI2XjPQ6VU0tU9vqxWJ97gQuBvaKyFE83fSeFWzUL+caHwf2qGpGVY8AL+EJm5VKOX2+CfgGgKo+AbTgJXhczZT1vJeLES7l8ySwTUS2iEgEz2C/p2Cf/Hxpu4DH1beUrVAW7bOIvA64F0+wrHRdPCzSZ1WdVNU+VT1PVc/DszNdp6oHGtPcJVPOff2P+AlkRaQPT002tIxtrDXl9Pll4E0AIvIaPOEysqytXH72AG/3vcbeAEzqbFmUijFqsTJR1ayI/DFeMk0b+IKqPi8idwAHVHUPXnr/r4rIYTzD2Q2Na/HSKbPPnwA6gG/6vgsvq+p1DWv0Eimzz6uGMvsbJJD9OeAA71fVFbsiL7PP/xX4f0TkvXjG/d9f4RNFROQBvElCn29L+ggQBlDVv8WzLf0WcBiIA/9lSb+3ws+XwWAwGJoQoxYzGAwGQ80xwsVgMBgMNccIF4PBYDDUHCNcDAaDwVBzjHAxGAwGQ80xwsVgMBgMNccIF4NhGRGR2DL8xnULlQowGJYDE+diMCwjIhJT1Y5Gt8NgqDdm5WIwLAERuVNE3pX3/qMi8kG/cNrTIvIzEZmXsl9EdorI/5f3/q9F5Pf915eLyA9E5CkReXShtOcicmtesbYH/W2/LyJ/7b/+koh8zs9mPOT/7hdE5AUR+VLtzoTBMBcjXAyGpfF14K1579+KV9Pnd1T1Mrxiap8qt66PiISBvwJ2qerlwBeAjy9wyO3A61T1EuAPS+zTC1wBvBcvf9TdwEXAr4rIa8tpl8FQKSa3mMGwBFT1GREZEJFzgX5gHDgF3C0iVwEuXk2Mdf72xXg1Xtbl7/nyyAYWSh74r8Dficg/4iWYLMZ3VFVF5GfAaVX9GYCIPA+cB/y0jHYZDBVhhIvBsHS+iZcFez3eSuY/4Qmay1U146fmbyk4JstczUHwuQDPq+oVZf72W/DK1/4fwJ+JyK8W2Scog+DmvQ7emzHAUBeMWsxgWDpfx8uAvQtP0HQDw75g+XXgV4oc80vgQhGJikgPfnp34EWg3y9QhYiEReSiYj/qFy7bpKr/DNzm/65xFjA0BWbWYjAsET9deydwQlVfEZG/A77jq6EOAAeLHHNMRL4BPAccAZ7xt6dFZBdwj4h04z2jnwaeL/LTNvA1fz8B7lHViTLNOwZDXTGuyAaDwWCoOUYtZjAYDIaaY9RiBsMKQEQ+C/y7gs2fUdUvNqI9BsNiGLWYwWAwGGqOUYsZDAaDoeYY4WIwGAyGmmOEi8FgMBhqjhEuBoPBYKg5/z/R7FxylAU/AgAAAABJRU5ErkJggg==\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "sns.residplot(data=merged_results_wo_zeros, x=\"value_sim\", y=\"value_obs\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 306, - "outputs": [], - "source": [ - "# # Note power tests will be inf when expected count is zero (any non-zero count will be impossible)\n", - "# eps = 0 #1e-14\n", - "# non_zero_shares = ms_comp.loc[(ms_comp.mode_share_obs > eps) & (ms_comp.trip_mode > eps)][[\"mode_share_obs\", \"trip_mode\"]]\n", - "# ddof = non_zero_shares.shape[0] - 1 # what to use here? num alternatives - 1? What about zero expected count?\n", - "#\n", - "# non_zero_shares.trip_mode *= num_samples\n", - "#\n", - "# sum_ms = non_zero_shares.mode_share_obs.sum()\n", - "# if sum_ms != 1.0:\n", - "# if np.abs(1.0 - sum_ms) > 1e-1:\n", - "# print(f\"NOT GOOD {sum_ms} should be one\")\n", - "# #print(f\"sum ms is not one but {sum_ms}, re-normalising\")\n", - "# non_zero_shares.mode_share_obs /= sum_ms\n", - "#\n", - "# powerdiscrepancy(non_zero_shares.trip_mode.values, non_zero_shares.mode_share_obs.values, lambd=0, ddof=ddof)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 314, - "outputs": [], - "source": [ - "@numba.jit\n", - "def within(lower, upper, x):\n", - " return (x >= lower) & (x <= upper)\n", - "\n", - "@numba.jit\n", - "def numpy_within(x):\n", - " lower = x[0]\n", - " upper = x[1]\n", - " val = x[2]\n", - " return within(lower, upper, val)\n", - "\n", - "def all_within_ci(trip_df, num_samples, alpha=0.9):\n", - " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=alpha, method='goodman')\n", - " vals_with_ci = np.hstack((ci, trip_df[['value_obs']].to_numpy()))\n", - " return np.apply_along_axis(numpy_within, 1, vals_with_ci).all()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 328, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Have 17 with contribution outside 95.00% CI, corresponding to 1.14% of all non-zero observations.\n" - ] - }, - { - "data": { - "text/plain": " lb ub value within\n3 0.000031 0.001291 0.000030 False\n3 0.000031 0.001291 0.000025 False\n8 0.003744 0.008367 0.003717 False\n8 0.000009 0.001113 0.000008 False\n1 0.000009 0.001113 0.000006 False\n1 0.000009 0.001113 0.000008 False\n3 0.000062 0.001460 0.000058 False", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
lbubvaluewithin
30.0000310.0012910.000030False
30.0000310.0012910.000025False
80.0037440.0083670.003717False
80.0000090.0011130.000008False
10.0000090.0011130.000006False
10.0000090.0011130.000008False
30.0000620.0014600.000058False
\n
" - }, - "execution_count": 328, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "alpha = 0.05 # default. this corresponds to a 95% CI, so 95% of all values should lie within our approximation\n", - "vals_within_ci = merged_results_wo_zeros.groupby(\"trip_id\").apply(lambda group: all_within_ci(group, num_samples, alpha))\n", - "trips_not_within = vals_within_ci.loc[~vals_within_ci]\n", - "print(f\"Have {trips_not_within.shape[0]} with contribution outside {1 - alpha:.2%} CI, corresponding to \"\n", - " f\"{trips_not_within.shape[0] / merged_results_wo_zeros.shape[0]:.2%} of all non-zero observations.\")\n", - "\n", - "bad_res = []\n", - "\n", - "for bad_trip_id in trips_not_within.index.values:\n", - " trip_df = merged_results.loc[merged_results.trip_id == bad_trip_id]\n", - " ci = multinomial_proportions_confint(num_samples * trip_df.value_sim.values, alpha=alpha, method='goodman')\n", - " vals_with_ci = np.hstack((ci, trip_df[['value_obs']].to_numpy()))\n", - " x_ = pd.DataFrame(vals_with_ci)\n", - " x_['in_range'] = np.apply_along_axis(numpy_within, 1, vals_with_ci)[:np.newaxis]\n", - " bad_res.append(x_.loc[~x_.in_range])\n", - "bad_res = pd.concat(bad_res)\n", - "bad_res.columns = [\"lb\", \"ub\", \"value\", \"within\"]\n", - "bad_res" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "## CI is not great for this\n", - "\n", - "we expect about 5% outliers for 95% CI, and I can see for 1e5 samples the number of outliers actually nearly doubles for the 95CI which makes sense - it is the large sample limit where all this is exact.\n", - "\n", - "Let's plot a histogram of the std deviation, and take a single trip and plot the diff from expected value with number of draws (i.e. prob vs num_draws)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "### histogram of residuals" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": 440, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "5.4540809129771874e-05\n" - ] - }, - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEICAYAAAC9E5gJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQB0lEQVR4nO3df6zddX3H8efLVkRx2iI3DWsbbzcbHZo48aawsZhFtlLA2GZRg3HSmGb9Y8zhfkRh/zRDSTBZREkmSWO7lc2IDF1ohMkaKHEm48ctIFgq444ftg3YqwWUGXXF9/44n7ID3tvee8/pPbf3Ph/Jzfl+39/P95z3N4S+7vfH+dxUFZKkhe1Vg25AkjR4hoEkyTCQJBkGkiQMA0kShoEkiSmEQZLtSQ4l+W5X7fQku5I81l6XtnqSXJdkLMlDSc7u2mdjG/9Yko1d9Xcnebjtc12S9PsgJUnHNpUzg38E1r2idgVwR1WtBu5o6wAXAqvbz2bgeuiEB7AFOAdYA2w5GiBtzJ907ffKz5IknWCLjzegqr6VZPgV5fXA77flHcBdwKda/YbqfJPt7iRLkpzZxu6qqsMASXYB65LcBbyhqu5u9RuADcC/Ha+vM844o4aHX9mWJGkye/bs+WFVDU207bhhMIllVfV0W34GWNaWlwP7u8YdaLVj1Q9MUD+u4eFhRkdHp9+5JC1QSZ6abFvPN5DbWcCszGmRZHOS0SSj4+Pjs/GRkrQgzDQMftAu/9BeD7X6QWBl17gVrXas+ooJ6hOqqq1VNVJVI0NDE57pSJJmYKZhsBM4+kTQRuCWrvql7amic4Hn2+Wk24G1SZa2G8drgdvbth8nObc9RXRp13tJkmbJce8ZJPkKnRvAZyQ5QOepoGuAm5JsAp4CPtSG3wZcBIwBPwU+BlBVh5N8Grivjbvq6M1k4E/pPLH0Wjo3jo9781iS1F85WaewHhkZKW8gS9LUJdlTVSMTbfMbyJIkw0CSZBhIkjAMJEnM/BvIJ7XhK259afnJay4eYCeSNDd4ZiBJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJoscwSPIXSfYm+W6SryQ5NcmqJPckGUvy1SSntLGvaetjbftw1/tc2eqPJrmgx2OSJE3TjMMgyXLgz4GRqnoHsAi4BPgscG1VvQV4FtjUdtkEPNvq17ZxJDmr7fd2YB3wxSSLZtqXJGn6er1MtBh4bZLFwOuAp4H3Aje37TuADW15fVunbT8/SVr9xqr6eVU9AYwBa3rsS5I0DTMOg6o6CPwd8H06IfA8sAd4rqqOtGEHgOVteTmwv+17pI1/U3d9gn0kSbOgl8tES+n8Vr8K+HXgNDqXeU6YJJuTjCYZHR8fP5EfJUkLSi+Xif4AeKKqxqvqf4GvA+cBS9plI4AVwMG2fBBYCdC2vxH4UXd9gn1epqq2VtVIVY0MDQ310LokqVsvYfB94Nwkr2vX/s8HHgF2Ax9oYzYCt7TlnW2dtv3OqqpWv6Q9bbQKWA3c20NfkqRpWnz8IROrqnuS3AzcDxwBHgC2ArcCNyb5TKtta7tsA/4pyRhwmM4TRFTV3iQ30QmSI8BlVfXiTPuSJE3fjMMAoKq2AFteUX6cCZ4GqqqfAR+c5H2uBq7upRdJ0sz5DWRJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ9BgGSZYkuTnJ95LsS/I7SU5PsivJY+11aRubJNclGUvyUJKzu95nYxv/WJKNvR6UJGl6ej0z+ALwzap6G/BOYB9wBXBHVa0G7mjrABcCq9vPZuB6gCSnA1uAc4A1wJajASJJmh0zDoMkbwTeA2wDqKpfVNVzwHpgRxu2A9jQltcDN1TH3cCSJGcCFwC7qupwVT0L7ALWzbQvSdL09XJmsAoYB/4hyQNJvpTkNGBZVT3dxjwDLGvLy4H9XfsfaLXJ6r8iyeYko0lGx8fHe2hdktStlzBYDJwNXF9V7wL+h/+/JARAVRVQPXzGy1TV1qoaqaqRoaGhfr2tJC14vYTBAeBAVd3T1m+mEw4/aJd/aK+H2vaDwMqu/Ve02mR1SdIsmXEYVNUzwP4kb22l84FHgJ3A0SeCNgK3tOWdwKXtqaJzgefb5aTbgbVJlrYbx2tbTZI0Sxb3uP/HgS8nOQV4HPgYnYC5Kckm4CngQ23sbcBFwBjw0zaWqjqc5NPAfW3cVVV1uMe+JEnT0FMYVNWDwMgEm86fYGwBl03yPtuB7b30IkmaOb+BLEkyDCRJhoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEn0IgySLkjyQ5BttfVWSe5KMJflqklNa/TVtfaxtH+56jytb/dEkF/TakyRpevpxZnA5sK9r/bPAtVX1FuBZYFOrbwKebfVr2ziSnAVcArwdWAd8McmiPvQlSZqinsIgyQrgYuBLbT3Ae4Gb25AdwIa2vL6t07af38avB26sqp9X1RPAGLCml74kSdPT65nB54FPAr9s628CnquqI239ALC8LS8H9gO07c+38S/VJ9jnZZJsTjKaZHR8fLzH1iVJR804DJK8DzhUVXv62M8xVdXWqhqpqpGhoaHZ+lhJmvcW97DvecD7k1wEnAq8AfgCsCTJ4vbb/wrgYBt/EFgJHEiyGHgj8KOu+lHd+0iSZsGMzwyq6sqqWlFVw3RuAN9ZVR8BdgMfaMM2Are05Z1tnbb9zqqqVr+kPW20ClgN3DvTviRJ09fLmcFkPgXcmOQzwAPAtlbfBvxTkjHgMJ0Aoar2JrkJeAQ4AlxWVS+egL4kSZPoSxhU1V3AXW35cSZ4GqiqfgZ8cJL9rwau7kcvkqTp8xvIkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAk0UMYJFmZZHeSR5LsTXJ5q5+eZFeSx9rr0lZPkuuSjCV5KMnZXe+1sY1/LMnG3g9LkjQdvZwZHAH+qqrOAs4FLktyFnAFcEdVrQbuaOsAFwKr289m4HrohAewBTgHWANsORogkqTZMeMwqKqnq+r+tvwTYB+wHFgP7GjDdgAb2vJ64IbquBtYkuRM4AJgV1UdrqpngV3Aupn2JUmavr7cM0gyDLwLuAdYVlVPt03PAMva8nJgf9duB1ptsrokaZb0HAZJXg98DfhEVf24e1tVFVC9fkbXZ21OMppkdHx8vF9vK0kLXk9hkOTVdILgy1X19Vb+Qbv8Q3s91OoHgZVdu69otcnqv6KqtlbVSFWNDA0N9dK6JKlLL08TBdgG7Kuqz3Vt2gkcfSJoI3BLV/3S9lTRucDz7XLS7cDaJEvbjeO1rSZJmiWLe9j3POCjwMNJHmy1vwGuAW5Ksgl4CvhQ23YbcBEwBvwU+BhAVR1O8mngvjbuqqo63ENfkqRpmnEYVNW3gUyy+fwJxhdw2STvtR3YPtNeJEm98RvIkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkMYfCIMm6JI8mGUtyxaD7kaSFZE6EQZJFwN8DFwJnAR9OctZgu5KkhWPxoBto1gBjVfU4QJIbgfXAIyf6g4evuPWl5SevufhEf5wkzUlzJQyWA/u71g8A58x2E93BcCJ0h81UQsigkjRb5koYTEmSzcDmtvpCkkdn+FZnAD/sT1dTl89Orz7dMV0GcnyzZD4fG3h8J7u5fnxvnmzDXAmDg8DKrvUVrfYyVbUV2NrrhyUZraqRXt9nrprPxzefjw08vpPdyXx8c+IGMnAfsDrJqiSnAJcAOwfckyQtGHPizKCqjiT5M+B2YBGwvar2DrgtSVow5kQYAFTVbcBts/RxPV9qmuPm8/HN52MDj+9kd9IeX6pq0D1IkgZsrtwzkCQN0IIKg/k+5UWS7UkOJfnuoHvptyQrk+xO8kiSvUkuH3RP/ZTk1CT3JvlOO76/HXRP/ZZkUZIHknxj0L30W5Inkzyc5MEko4PuZyYWzGWiNuXFfwF/SOdLbfcBH66qE/4t59mS5D3AC8ANVfWOQffTT0nOBM6sqvuT/BqwB9gwX/77JQlwWlW9kOTVwLeBy6vq7gG31jdJ/hIYAd5QVe8bdD/9lORJYKSq5vJ3DI5pIZ0ZvDTlRVX9Ajg65cW8UVXfAg4Puo8Toaqerqr72/JPgH10vrk+L1THC2311e1n3vymlmQFcDHwpUH3ooktpDCYaMqLefOPyUKSZBh4F3DPgFvpq3YZ5UHgELCrqubT8X0e+CTwywH3caIU8O9J9rSZEk46CykMNA8keT3wNeATVfXjQffTT1X1YlX9Np1v4K9JMi8u9SV5H3CoqvYMupcT6Peq6mw6My9f1i7ZnlQWUhhMacoLzV3tWvrXgC9X1dcH3c+JUlXPAbuBdQNupV/OA97frqvfCLw3yT8PtqX+qqqD7fUQ8K90LkufVBZSGDjlxUms3WDdBuyrqs8Nup9+SzKUZElbfi2dBx2+N9Cm+qSqrqyqFVU1TOf/uzur6o8H3FbfJDmtPdRAktOAtcBJ90TfggmDqjoCHJ3yYh9w03yb8iLJV4D/BN6a5ECSTYPuqY/OAz5K57fKB9vPRYNuqo/OBHYneYjOLy67qmrePYI5Ty0Dvp3kO8C9wK1V9c0B9zRtC+bRUknS5BbMmYEkaXKGgSTJMJAkGQaSJAwDSRq4fk8ymeTFrqfupvQIvU8TSdKA9XuSySQvVNXrp7OPZwaSNGATTTKZ5DeTfLPNd/QfSd52InswDCRpbtoKfLyq3g38NfDFaex7apLRJHcn2TCVHebM30CWJHW0CRl/F/iXzkwsALymbfsj4KoJdjtYVRe05TdX1cEkvwHcmeThqvrvY32mYSBJc8+rgOfaLLYv0yZpPOZEjV0T5z2e5C46U74fMwy8TCRJc0ybnv2JJB+EzkSNSd45lX2TLE1y9CziDDrzeh33LwIaBpI0YJNMMvkRYFObAG8vU//LjL8FjLb9dgPXTOXPw/poqSTJMwNJkmEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCfg/eoht2K7d/0kAAAAASUVORK5CYII=\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "merged_results['diff_squared'] = np.square(merged_results.value_sim - merged_results.value_obs)\n", - "print(merged_results['diff_squared'].max())\n", - "plt.hist(merged_results.diff_squared.values, bins=100);" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 448, - "outputs": [ - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEFCAYAAAAYKqc0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAANl0lEQVR4nO3df6zd9V3H8edrMDbDXDXSGEJhZZbgGhOd3rBEjDHGH0VWIPiLOv3DNDQYWWaMMV3iP5oY8R9jljBNI4QYFwiyaajUsSWD4BLmaJE5WMVUZKGNSTvJqvgPAd/+cc/wer2nnHvP9/Tb8+7zkdxwz/fe8z3vbwsvPnmf9/l8U1VIknp5x9gFSJKGZ7hLUkOGuyQ1ZLhLUkOGuyQ1dOnYBQBcccUVtXPnzrHLkKSlcuzYsW9U1faNfnZBhPvOnTs5evTo2GVI0lJJ8vVpP7MtI0kNGe6S1JDhLkkNGe6S1NDg4Z7kA0n+NMkjSX5t6PNLkt7eTOGe5P4kp5M8v+74niQvJjmR5CBAVR2vqruAXwBuHL5kSdLbmXXl/gCwZ+2BJJcA9wI3AbuBfUl2T352C/AYcGSwSiVJM5sp3KvqKeDVdYdvAE5U1UtV9TrwEHDr5PcfraqbgI9MO2eSA0mOJjl65syZrVUvSdrQPB9iugp4Zc3jk8CHkvwYcDvwLs6xcq+qQ8AhgJWVlS1vKr/z4GNvff/yPTdv9TSS1Mrgn1CtqieBJ4c+ryRpdvNMy5wCrl7zeMfk2MyS7E1y6OzZs3OUIUlab55wfwa4Lsm1SS4D7gAe3cwJqupwVR3Ytm3bHGVIktabdRTyQeBp4PokJ5Psr6o3gLuBx4HjwMNV9cLiSpUkzWqmnntV7Zty/AhzjDsm2Qvs3bVr11ZPIUnawKjbD9iWkaTFcG8ZSWrIcJekhkYNd0chJWkx7LlLUkO2ZSSpIcNdkhqy5y5JDdlzl6SGbMtIUkOGuyQ1ZLhLUkOGuyQ15LSMJDXktIwkNWRbRpIaMtwlqSHDXZIaMtwlqSGnZSSpIadlJKkh2zKS1JDhLkkNGe6S1JDhLkkNGe6S1JDhLkkNOecuSQ055y5JDdmWkaSGDHdJashwl6SGDHdJashwl6SGDHdJashwl6SGDHdJashwl6SGDHdJasi9ZSSpIfeWkaSGbMtIUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOGuyQ1ZLhLUkOXLuKkSW4DbgbeC9xXVZ9bxOtIkjY288o9yf1JTid5ft3xPUleTHIiyUGAqvrrqroTuAv4xWFLliS9nc20ZR4A9qw9kOQS4F7gJmA3sC/J7jW/8juTn0uSzqOZw72qngJeXXf4BuBEVb1UVa8DDwG3ZtUfAn9bVc9udL4kB5IcTXL0zJkzW61fkrSBed9QvQp4Zc3jk5NjHwV+Avi5JHdt9MSqOlRVK1W1sn379jnLkCSttZA3VKvqE8AnFnFuSdLbm3flfgq4es3jHZNjM0myN8mhs2fPzlmGJGmtecP9GeC6JNcmuQy4A3h01idX1eGqOrBt27Y5y5AkrbWZUcgHgaeB65OcTLK/qt4A7gYeB44DD1fVC4spVZI0q5l77lW1b8rxI8CRwSqSJM1t1O0H7LlL0mKMGu723CVpMdw4TJIaWsic+6yS7AX27tq1a5Dz7Tz42Fvfv3zPzYOcU5KWkW0ZSWrItowkNWS4S1JDhrskNeScuyQ1NOq0TFUdBg6vrKzcOfS5nZyRdDGzLSNJDRnuktSQ4S5JDfmGqiQ15CdUJakh2zKS1JDhLkkNGe6S1JDhLkkNGe6S1JCjkJLUUNu9ZaZxzxlJFwPbMpLUkOEuSQ2N2pY5X9a2YiTpYuDKXZIaMtwlqSHDXZIacs5dkhpyy19JauiimJaZZv0UjR9qktSFPXdJauiiXrmv59YEkrpw5S5JDblyn8JVvKRl5spdkhoy3CWpIcNdkhoy3CWpIcNdkhpybxlJasi9ZSSpIdsyktSQ4S5JDfkJ1RlMuwern1yVdKFy5S5JDblyH4h70Ui6kLhyl6SGDHdJashwl6SG7LmfR/blJZ0vrtwlqSHDXZIaMtwlqSF77gs27dOtFzvff5AWy5W7JDU0eLgneX+S+5I8MvS5JUmzmSnck9yf5HSS59cd35PkxSQnkhwEqKqXqmr/IorVcth58LG3viSNY9aV+wPAnrUHklwC3AvcBOwG9iXZPWh1kqQtmSncq+op4NV1h28ATkxW6q8DDwG3DlyfJGkL5pmWuQp4Zc3jk8CHknwX8PvAB5N8vKr+YKMnJzkAHAC45ppr5ihjPNPaDrYjJI1t8FHIqvp34K4Zfu8QcAhgZWWlhq5Dki5m80zLnAKuXvN4x+SYJGlk86zcnwGuS3Itq6F+B/BLmzlBkr3A3l27ds1RxvKb5QM9fuhH0mbMOgr5IPA0cH2Sk0n2V9UbwN3A48Bx4OGqemEzL15Vh6vqwLZt2zZbtyTpHGZauVfVvinHjwBHBq1IkjS3UfeWuZjbMrNM1Dh1I2mrRt1bxraMJC2GG4dJUkOGuyQ1NGq4J9mb5NDZs2fHLEOS2rHnLkkN2ZaRpIYMd0lqyDn3JTfPtgRuaSD1Zc9dkhqyLSNJDRnuktSQ4S5JDRnuktSQ0zIXmHl2izwf0y9DTedcaJwcUjdOy0hSQ7ZlJKkhw12SGjLcJakhw12SGnJa5iIw7yTIhTDl4jTLMPxzvHg4LSNJDdmWkaSGDHdJashwl6SGDHdJashwl6SGDHdJasg596Zm2Tly2nHnn5fT+fw79N+XC59z7pLUkG0ZSWrIcJekhgx3SWrIcJekhgx3SWrIcJekhgx3SWrIcJekhgx3SWrIcJekhtxbRlu26Hurbvb80/Y7GWoflPX1zHKuzV5D531a3I/m/HJvGUlqyLaMJDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDU0+M06klwOfBJ4HXiyqj419GtIks5tppV7kvuTnE7y/Lrje5K8mOREkoOTw7cDj1TVncAtA9crSZrBrG2ZB4A9aw8kuQS4F7gJ2A3sS7Ib2AG8Mvm1N4cpU5K0GTO1ZarqqSQ71x2+AThRVS8BJHkIuBU4yWrAP8c5/ueR5ABwAOCaa67ZbN1aoEXfG/VcrzfUfUmn/c6Qr7Xoe4LOc0/YoX5nURbx2tP+rmb5szsffxaz1Deked5QvYr/XaHDaqhfBXwG+NkkfwIcnvbkqjpUVStVtbJ9+/Y5ypAkrTf4G6pV9V/Arw59XknS7OZZuZ8Crl7zeMfk2MyS7E1y6OzZs3OUIUlab55wfwa4Lsm1SS4D7gAe3cwJqupwVR3Ytm3bHGVIktabdRTyQeBp4PokJ5Psr6o3gLuBx4HjwMNV9cLiSpUkzWrWaZl9U44fAY5s9cWT7AX27tq1a6unkCRtYNTtB2zLSNJiuLeMJDVkuEtSQ6mqsWsgyRng61t8+hXANwYs50Lj9S23ztfX+dpgOa7vfVW14adAL4hwn0eSo1W1MnYdi+L1LbfO19f52mD5r8+2jCQ1ZLhLUkMdwv3Q2AUsmNe33DpfX+drgyW/vqXvuUuS/r8OK3dJ0jqGuyQ1tNThPuUeri1Mu29tB0muTvJEkq8leSHJx8auaUhJ3p3ky0m+Mrm+3x27pkVIckmSf0jyN2PXMrQkLyf5apLnkhwdu56tWNqe++Qerv8M/CSrd4F6BthXVV8btbCBJPlR4DXgz6vq+8auZ0hJrgSurKpnk3w7cAy4rdHfXYDLq+q1JO8Evgh8rKq+NHJpg0rym8AK8N6q+vDY9QwpycvASlVd6B9immqZV+5v3cO1ql4HvnUP1xaq6ing1bHrWISq+reqenby/X+yumX0VeNWNZxa9drk4TsnX8u5ipoiyQ7gZuDPxq5FG1vmcJ92D1ctkcmN1z8I/P3IpQxq0rJ4DjgNfL6qWl0f8MfAbwP/PXIdi1LA55IcS3Jg7GK2YpnDXUsuyXuATwO/UVX/MXY9Q6qqN6vqB1i9/eQNSdq01pJ8GDhdVcfGrmWBfqSqfhC4Cfj1SZt0qSxzuM99D1eNZ9KL/jTwqar6zNj1LEpVfRN4AtgzcilDuhG4ZdKXfgj48SR/MW5Jw6qqU5N/ngb+itU28FJZ5nCf+x6uGsfkDcf7gONV9Udj1zO0JNuTfMfk+29j9U3/fxq1qAFV1cerakdV7WT1v7svVNUvj1zWYJJcPnmjnySXAz8FLN3U2tKGe/d7uG5039qxaxrQjcCvsLrie27y9TNjFzWgK4Enkvwjq4uQz1dVu3HBxr4b+GKSrwBfBh6rqs+OXNOmLe0opCRpuqVduUuSpjPcJakhw12SGjLcJakhw12SBjb0xn9J3lwzWTbTyLfTMpI0sKE3/kvyWlW9ZzPPceUuSQPbaOO/JN+T5LOT/Wr+Lsn3LrIGw12Szo9DwEer6oeA3wI+uYnnvjvJ0SRfSnLbLE+4dAsFSpI2YbJJ3g8Df7m6+wYA75r87Hbg9zZ42qmq+unJ9++rqlNJ3g98IclXq+pfzvWahrskLd47gG9Odgr9PyYb551z87w1G5m9lORJVrfJPme425aRpAWbbGn9r0l+HlY3z0vy/bM8N8l3JvnWKv8KVvdmetu7lhnukjSwKRv/fQTYP9mQ7AVmv3PcB4Cjk+c9Adwzyy0pHYWUpIZcuUtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ4a7JDVkuEtSQ/8D+hlGbxrz6lMAAAAASUVORK5CYII=\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "# remove where both are unavailable\n", - "plt.hist(merged_results.loc[(merged_results.value_obs != 0) & (merged_results.value_sim != 0)].diff_squared.values, bins=100);\n", - "plt.yscale('log')" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "### single trip-mode convergence to expected value" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 431, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "estimation bundle trip_mode_choice not in settings file estimation.yaml\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trip_mode_choice tour_type 'othdiscr' (10000 trips)\n", - "Done\n" - ] - }, - { - "data": { - "text/plain": " trip_mode drawn exact\n0 WALK_LRF 0.3210 0.322184\n1 WALK_HVY 0.2812 0.278911\n2 TNC_SINGLE 0.2170 0.219172\n3 WALK_LOC 0.1780 0.176455\n4 TNC_SHARED 0.0018 0.002318\n5 WALK 0.0008 0.000713\n6 SHARED2FREE 0.0001 0.000179\n7 SHARED3FREE 0.0001 0.000063\n20 TAXI 0.0000 0.000005", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
trip_modedrawnexact
0WALK_LRF0.32100.322184
1WALK_HVY0.28120.278911
2TNC_SINGLE0.21700.219172
3WALK_LOC0.17800.176455
4TNC_SHARED0.00180.002318
5WALK0.00080.000713
6SHARED2FREE0.00010.000179
7SHARED3FREE0.00010.000063
20TAXI0.00000.000005
\n
" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": "(1e-08, 1)" - }, - "execution_count": 431, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAECCAYAAAD9z2x7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7D0lEQVR4nO3dd5hU1fnA8e+Z2d7ZZWGB3aV3aYKgYMFCsaLGSiyx15j4S6KmWBI1lhhji7ELJlFBJDawK9JUuvReF9hGWXYXtp/fH3dmdvrO7PTd9/M8+2Tnzp07Z2/wnTPnPec9SmuNEEKIts8U6QYIIYQIDwn4QgjRTkjAF0KIdkICvhBCtBMS8IUQop2QgC+EEO2EBHwhhGgnJOALIUQ7EReuN1JKpQIvAnXAPK31f8P13kIIIQLs4Sul3lBKlSql1jodn6yU2qSU2qqUus9y+GJgltb6JuCCQN5XCCGE/wId0pkGTLY/oJQyA/8EzgYGAVcqpQYB+cAey2mNAb6vEEIIPwUU8LXW84GDTodHA1u11tu11nXAu8AUoAgj6Af8vkIIIfwXijH8bjT35MEI9GOA54AXlFLnAh97erFS6mbgZoDU1NSRAwYM8L8FDbVw7JDx01ADygRJGZDcARIzQSn/rymEEDFi+fLl5VrrXOfjYUvaaq2rget8OO8V4BWAUaNG6WXLlgXyprBvJayeCWvfh+pSSKqDQVNgyGXQfRyY5MuGEKJtUUrtcnc8FAF/L1Bg9zjfciz8lIJuxxs/Ex+BHfNg9Xuw5n1Y8RZk5MOQn8HQy6Hz4Ig0UQghwiUUAX8p0Fcp1RMj0F8BTA3B+/jHHAd9zjJ+6p6GTZ8aPf/FL8CiZ6HTYBh6GQy5BDLzW76eEELEGBXIBihKqXeA8UBHoAR4UGv9ulLqHOAZwAy8obV+tDXXD3hIxxfV5bDuf7B6BhQtBRT0OBmGXGoM/SRnhfb9hRAiyJRSy7XWo1yOR/OOV2EJ+PYObrcM+cyEA1vBnAD9Jhnj/f0mQVxi+NoihBCtJAHfH1rDvhVG8LclezONHv/Qy6FwrCR7hRBRq10F/KamIP5NTQ2w/TvU2pmw4RNUfTU6Ix815BJjzF+SvUKIKOMp4IdtWmY43fP+amYtLwryVaeQzCQmmFZwXe0Shi9+HrXoGUn2CiFiRpsM+BMHdaagQ0pIrn2kZiAXLRzL1MHJPNpvC2rNe/DVg/DVQ5LsFUJEtbYZ8AfnMXFwXsiun52awN8+30Re1zO468Zb4MA2WDPLSPZ+fBfM/a2R5B16OfSdKMleIURUaJMBP9RuH9+bbaVVPP3lZnrlpnLe0N4w/l447R67ZO8s2PCxJHuFEFGjTSZtw6G2oZGpr/7I2r0VzLzlJIYVZDme0NhgWdlrJHupr7as7JVkrxAitNrVLJ1wKa+qZcoLi6hvbOLDO8fRJTPZ/Yl11ZaVvTNg69egG6HzccZ4vyR7hRBBJgE/RDYWH+FnLy6mR8dU3rv1JFISWhglqyozVvaumSkre4UQISEBP4S+2VjCjdOXMXFQHi/+/HhMJh/LL1uTvatnwMFtzSt7JdkrhAiABPwQe23Bdh6Zs4E7Tu/N7yb5WcPfYWXvLKgusyR7LzTG+yXZK4TwQ7taeBUJN5zck62lVfzz2230zk3j4uP9GJdXCrqNNH5sZZxnGr3/FdMl2SuECArp4QdRXUMT17zxIyt2Headm8cwsnt2gBesho1zjfF+SfYKIXwkQzphcqi6joteXERlTQMf3DGOguwgrfj1lOwdehkMvECSvUIIGwn4YbS1tIqLXlxE18xk3r99LGmJQR45c0n2JkK/iZLsFUIAEvDDbuGWcq59cwmn9cvl1WtGYfZ15o4/bMle6569kuwVQkjAj4h//7CL+z9Yy40n9+RP5w0K7Zs1NsD2ecaQj8vK3suhc4jfXwgRNWSWTgRcfWJ3tpVW8drCHfTplMYVowtD92bmOOh7lvFjn+xd/DwsesYu2XspZHYLXTuEEFFLevgh1tDYxPXTl7F4azkf3DGO47plhrcB1mTv6hmwdxmS7BWi7ZMhnQiqOFrPmU/PI79DCrNvG+v7StxgO7AN1rxnjPnbkr2TjOAvyV4h2gxPAV8yemGQmRLPH84ZyKo9h5mxbE/kGpLTG8bfB79cDjd9A6Oug93fw4yr4Km+8NFdsHMhNDVFro1CiJCRHn6YaK25/JUf2FRcyTe/OY2ctCjpTUuyV4g2R4Z0osDmkkrOeXYBFx/fjScvGRbp5riyJntXz4Bt3zSv7B16GRx3iSR7hYgRMqQTBfp1TueGU3oyc1kRy3YejHRzXCWkwtBL4apZ8JtNcPbfIC4JvnwA/jEYpp0HK96CY4cj3VIhRCtIDz/MqmsbmPD0d2Qkx/PJL08mzhwDn7mS7BUipkgPP0qkJsbxwPmD2FhcybTFOyPdHN/YJ3tv9JTsXSTJXiGinPTwI0BrzXXTlrJ0x0G+/s148jKTIt0k/7lL9mYWGMneIZdJsleICJKkbZTZdaCaCf+Yz8RBnXlh6vGRbk5gJNkrRFSRIZ0o0z0nlTvG9+GT1ftZsKUs0s0JjEuy90lJ9goRhaSHH0E19Y1MfmY+Sik++/UpJMaZI92k4PKY7L0c+k6QZK8QISI9/CiUFG/mL1OOY0d5Na98tz3SzQk+d8neXYthxs8l2StEBEgPPwrc8d8VfLWhhC/vPo3CnCDtkBWtrMne1TNg4ydQf1SSvUIEmSRto1hxRQ1n/n0eo3tm88YvTkCpCBVXCze3yd4hRj5Akr1CtJoM6USxvMwk7p7Qj283lfH5upJINyd83CZ7EyXZK0SISA8/StQ3NnH2swtIjDMx565TIt2cyLIle2fAwe2S7BXCT9LDj3LxZhNXjSlk3b4jrN93JNLNiSxbsneF+2Tvx7+SZK8QrSABP4pMGd6NBLOJWcuLIt2U6KAU5I+Es58whnx+/j70nWRM85x2Djw7FL56CErWR7qlQsQECfhRpENqAmcN6sQHq/ZS1yC9VwfWPXt/9ir8dgtc/CrkDoBFz8G/ToJ/nQyLnoWKvZFuqRBRSwJ+lLl0ZAEHq+v4ZmNppJsSvRLTjLINXpO9/4aaiki3VIioIgE/ypzStyOd0hOZtTyCWyHGkrRcGHML3PS1MeY//j44shc+uhP+1hdmXmMUd2uojXRLhYi4uEg3QDiKM5u46PhuvLZgB2WVteSmy4wUn1mTvafdC3tXGLN81r4P6z+EpCwYfKGxuKvwJDBJX0e0P/KvPgpdOrKAxibNBytlPNpexbF6/v7FJiqO1ns/0ZrsPedJ+M1G+PksY6MW52Rv6YawtFuIaCEBPwr16ZTGiMIs3lu+h2hdJ1HX0MSr87dzoCo8QyV1DU1c+tJinv9mK3PW7Afgu81lzNvkPtdRWlnDK/O30ajijLn77pK9L55oSfY+B0f2heXvECKSJOBHqUtG5rO5pIrVRdGZeHx0znoenbuBD1eFJ1D+5ZN1bC6pAmDtvgqW7zrItW8s4RdvLnU5t6a+kVOf/Ja/zt3Iqj2Hmp9wSPZutCR7E+DL++HpQZLsFW1eWAO+UupCpdSrSqkZSqmJ4XzvWHP+sK4kxkXnnPwPVu5l+ve7ANhcUhny95u1vIj//LCbW07txdjeOfyw7QB3/Hel7fkjNY5DPI/N3UBNvTGt1eMitrROlmTvN0ay97R7Jdkr2jyfA75S6g2lVKlSaq3T8clKqU1Kqa1Kqfu8XUNr/YHW+ibgVuDy1jW5fchIimfycXl8uGovNfWNkW6OzZaSSn4/ew2je2QzqnuHkAf8raVV/PF/axjbO4ffTerP0PwstpdXc7C6jnsm9wdg7d7mHvlna4uZ/v0urh/Xk6yUeNb5smo5pzec/vvmlb0jf2Gs5J3xc3iqn7Gyd9diWdkrYp4/PfxpwGT7A0opM/BP4GxgEHClUmqQUmqIUuoTp59Odi/9k+V1wotLRxZwpKaBL9dHR0G1pibNPe+vJjnBzAtTRzC4awabS6r8yjNU1tRz0YuL+NQyDu9NY5Pmnlk/kZxg5pkrhhNnNjGyewcA/nDOAC4fVQA0B/ySIzXcM+snhuZnct/ZAxjcNYP1+/0oU+Et2fvm2ZLsFTHP54CvtZ4PHHQ6PBrYqrXerrWuA94Fpmit12itz3P6KVWGJ4BPtdYrgvdntE1je+fQNTOJ96JkWOe/S3azcvdh7j9vIJ0ykuiXl05VbQP7Kmps51Qcq+fG6UtZt8/9OPjfPt/Eyt2H+dYp2frSd9t47ustDsfeXLSDFbsP89D5g+mUbmz0fuaATnx05ziuHduDnLREumQm2Xrxf/54HbUNTTx7xQgS4kwM6pLBxuJK6htb0TM3x7tJ9vaXZK+IaYGO4XcD7FcIFVmOefJL4CzgEqXUre5OUErdrJRappRaVlYW43u9BshkUvxsZD4LtpSxv+JYRNtSeqSGJz/dyLg+OVw43Pi/uF/ndAA2FzcP67zwzRa+2lDKNxtcZ8+s2nOYf/9gjP1vLa2yHV+/7whPfraRt3/cbTu29/AxnvpiE2cN7MSU4V1tx00mxdD8LNueAf06p7O5pIqvN5Qwd00xd53Zl54dUwEY3DWTuoYmtpdVB/bH25K97xs9/8lP2JK9+ulBMP18SfaKmBDWpK3W+jmt9Uit9a1a65c8nPOK1nqU1npUbm5uOJsXlS4ZmY/WMHtFZOfkPzxnA7WNTTxy4ZDmYNvJCPibLOP4O8qrmbZ4JwA7Dxx1eL3Wmkc+WU/HtESmDO/K1lJjKEhrzUMfraNJQ/GRGqpqGwB45svNNGn485TjvG4I069zGtvKqnh0zgZ656Zy0ym9bM8N6poB4PHbhtXW0irfK5SmdYITb6X6mi8YX/t3ZqZOhYoiSfaKmBBowN8LFNg9zrccE0HSPSeV0T2zmbW8KGJz8jcVV/LxT/u45dRett4zQGZKPHkZSbYe/qNzNpBgNtGvcxo7Dzj2qr/aUMqyXYe4+6x+DMvP4khNA+VVdczbVMaSnQc5tZ/x4b69rIotJZW8v6KIa0/qTresZK9t69s53ejFl1dz7+QBJMQ1/5Pu1TGVeLOyTed0p66hibOe/o5znlvg1z2Z/v1Oduou3HvgXLZc9h3c+LUke0XUCzTgLwX6KqV6KqUSgCuAjwJvlrB36ch8dpRXs3zXoZZPDoGXvttGSoKZG07u6fJcv7x0NpVUsnZvBV9tKOG28b05vrADO8ubA35Tk+bJzzbSq2Mql43Kp0+nNAC2lFby1BebKMxO4fdnDwBgW1kVT32xiZSEOG4b36fFtvW3DCuN7N6BCYM6OzwXZzbRPSeV7WWeA/7sFc35kaYm3z5QK2vqeWV+86bz324ug/xRTsneCU7J3j9LsldEnD/TMt8Bvgf6K6WKlFI3aK0bgDuBz4ENwEyt9brQNLX9OmdIF1ISzLy3LPzJ2z0Hj/LRT/uYOrqQrJQEl+f7d05ja2kV/5q3jbTEOK4Z24MeHVM5UF1nmx//9cZStpRW8auz+hJnNtkC/qvzt7Nu3xHuOrMvvXPTMJsUs1fs5fN1Jdx8ai+yU13fz9nALhlcPKIbD3sY+umdm8oX60t44rONLs81Nmletgvc+3zMk0xfvJPDR+v53+1jASg9Yjd8Y0v2vmYkey96xZLsfVaSvSLifC6eprW+0sPxucDcoLVIuEhNjOOcIV34ZPU+/nLhYBLjzGF775fnb8OsFDfajY3b69c5ndqGJuas2c/Np/YiIymeHjnGsM/O8mqG5mfx6oLtdMtK5twhXQDokplEaoKZbzeV0TUziQuHdzV649kpLNhSTnpSHNe7+TbhTkKciacvH+7xeZPlQ+Bf87bRr3MaY3rmcOhoHfFmE5tLKtlRXs21J3Vn+ve72Fl+lPwOKRw+WkdGUjwmk+sHSG1DI28u2snp/XMZUdiB/p3TXfIVNolpMOxy46eqFNbOhjUzjZW9Xz4APU8xirkNugCSMn36e4UIhJRWiBHj+uRQXdfInoPhm61TWlnDzGVF/GxkN/Iyk9ye0z/PGFKJMymuG9cDgB4dUwAjcbu66DBLdhzkunE9iDMb/9yUUvS29PKvGdt8vFeuceyKEwpISwxOIdcbT2n+4Lh7xk9MW7yTc59byMR/zOeNhTvo2TGVW07rDcCO8ip2Hahm+F++tM0mcjZ3zX4OVNfZPpB6dExxyVcAHK1rcDxgSfY6rOw9vMcx2btxDjTUBeXvFsIdCfgxojDbCKJ7DnroTYbAf3/YTUNjE7ec2tvjOX06pRFvVlwwvCtdMo0Ea/fs5h7+9MW7SEuM4/ITChxeNyAvnZQEM1eeUOhwzKTgmpN6BO1vGNk9mwfOG2R7PGNp8yziFbsPM3V0IV0yk0iON7Oj/Kht2Oz7bQdobNLUNjiucp6+eBe9clMZ17sjAD1yUtlaWsXBaiNQ7zl4lAc+XMugBz5nU7GHVcjWlb13rbQke681kr3vTm3es1eSvSIEpB5+jCjoYAn4h8IX8Oeu2c+Ynjn0sJuZ4ywlIY6Zt5xkG5cHSE4w0yUziQ37jzB/cxnnD+tKelK8w+t+N2kA15/ck8yU5uM3ndqLiYM7U2D5cAsW6zcOMBaGWcWbFRcd3w2lFD06prK1rIpNxcb0zM/WFXPVaz9y6Ggdn/36VMBYL7Bqz2EePH+QbbgnI9lo//EPf8klI/Mdah/938xVzLnrFM8NU8pI9uaPgkl/he3zjBr+q2fC8mmQWQBDLjXWAHQaGKS7IdozCfgxIjc9kcQ4E7s9jRcH2dbSKraUVnHVid1bPHdEYQeXYz1yjGRpY5PmwhGua/Fy0xNdNnfJTI5naH5Wq9vsybD8LIblZ5IQZ2LpzkPkZSRRfKSGswZ2pmOa0YbC7GS+3lBKg91Mne+3H0ApqG9sIt5s4qOf9mE2KaYMb/57zh3Shb99vgnApdCddQWwdfaPu5yAjTXZ23cC1FYZwztrZsKiZ2Dh05A3xBjvH3IJZHT1fB0hvJAhnRihlKIwO4XdYRrS+XxdMQCTBue16vU9OqbS2KTpmpnE6B7ZwWya33LSEvnwzpNtf8v95w3CpOBquw+zrlnJNDRpUhLM/OeGMbbjWhs1erTWfPzTPk7u09Fh9pD3bz9Gcv3W/yznxMe+9r3B1mTvVe8be/ZOfgJM8c1lnGVlr2glCfgxpCA7hT2HwpO0/WxtMSMKszwma1vSI8cYRjl/eFfvPdswunJ0IbNvH8u5Q7uw8v6JjO3T0facdYHXhEGdOblvRzpnNH/72F9Rw4rdh9l7+BgXDHPtXX/6q+Zhm7yMJH56cCK/mdCPo3WNVNc28MX6Ekora/lw1V7W7avgPz/sYsVuH9dUWJO9N38Ldy6XZK8IiAzpxJDC7BSW7DiI1tpruYFAFR06ypq9FbbFUK0xorADCWYTlxyfH8SWBSY1MY7jLcNP9rkDMFY0A5w/1Ajol4zM52B1He8s2cOlL33PgLx0EuJMTBjsuLgLjLUAPz0wkTizIjnejMmkbB+Uc+yqgv7q3VUAmBRcMKyrrS0+69jHSPaOvw/2Lrfs2Tvbbs/ei4zx/oITZc9e4ZYE/BhSkJ1CVW0Dh4/W08GHRUmt9dlaYzhn8nGtG84BGN0zm9UPTSQpPnxrBgJxev9cXr92FGcMMKp4/27SAKpqG3hniTGrZ2NxJaf2yyXDKfls5fwB0tXyjeGeWatdzm3SUFoZQK0d52Tvtm+N8f7VM2D5m5BZaIz1S7JXOJGAH0MKOhhBZPfBo34H/Mc+3cCYntmcMcC1h+rs83XFDOySYev1tlasBHswyjCcOdDx3jivBTitn+/F/LrYDYUN6ZbJsfpGhwqhAQV8e+Z46DfR+JFkr2iBfO+LIYWWcXF/E7eLt5Xz8nfb+d/Klpfzl1bWsGzXISa3MlnblvkT8LvaFX0b1CWDL+8+lbvOaK4NVHKkxt3LAuNLsnflfyTZ245JwI8hrZmLr7XmH19uBqCkouUg892mMrSGiW7GqtujH/9wJpmWufa9c33/xpMUb+wKFmdS3HRqT5RStumpA/LSqaxp4EBVCEsouyR77zGSvR/eYUn2XivJ3nZIhnRiSGpiHDmpCX6ttl209QBLdx4iJcFMsQ+9yo3FlSTGmWybm7R3nTOS+O534zlW3+h3ovy8oV05b2jzMEqv3DQ++/UpvLesiI3FlYx85Ct2PHZOSBPwgCXZ+wcY/3unZO8HkuxtZ+T/3RhTkJ3icz0drTX/+GozXTKTuGxUAcWW+eTebCmtok8no3KlMGSlJNjKRgRqQF6GQ5kJ64YvYWFN9p7zN6OM89T3jIVeP71rKeM8TMo4t3ES8GNMgR+LrxZsKWf5rkPcfnofCrNTqGto4tDReq+v2VJSKb37ELO/v4eqHf//KD1Sw+Kt5aFvhDXZ+7PX4HdbjTLOHfsayd4XT4SXpIxzWyQBP8YUZiez9/AxGlrYmFtrzdNfbqZbVjKXjcq3zQsv9jKOf6Smnv0VNQ51cURo/G5SfwAOHnUcQz/3+YVMfe3H8O5uZk32Xj1bkr1tnAT8GFPQIYXGJs3+FhKw8zaXsWrPYe44vQ+JcWY6ZxgB39vsEOu0Qenhh97Y3jkA/Pa9nxjz16/YsP8Ii7eVU2aZrhnWoR57kuxt0yRpG2PsyyR7qiqpteYZS+/+kpHGSldbD99LwN9i2Yy8X2fp4YeatR6P9UP27Gcd99Q9fLTepcJo2Nkne4uWGfP7177vlOy9HArGSLI3RkjAjzHWIO9tauYP2w/yU1EFj188xLapd6f0RJTyPqSzpaSKxDgT+R2CW55YuGpp4VzFsXoKvJ4RRkpBwQnGj/3K3p/edVrZezl0an05DhF68rEcY7pkJmE2Ka+JW2thrnOGdrEdizebyElN9Dqks1lm6IRNegs7er2xcAfLdx3iua+3hKlFPvKa7B1jJHsXPy/J3iglPfwYE2c20S0rmd1epmZu2H+EblnJLnVf8jITvQ7pbC2pZHTPyJYybi88zb3vmJZAeVUds1fuZfbKvQDccXqf6PwQtt+zt7IE1s02Nm/54k/wxf3Q81Rjfv/A82XP3ighPfwYVJid4nXx1cbiSgZ2yXA5npeR7HFIp7Kmnn0VNfSVhG1E/XZif5djgx74LAIt8VN6ZzjxNqdk724j2ftUP0n2RgkJ+DGoIDvZY8CvqW9ke1kVg7q4Bm5vPfwtMkMnYrY+ejZ3n9UPgHF2NfqtahuauNdN1U1v9lcc46XvtoV3eqeVNdl710q44Ss4/hrYucDYs/fv/eDjX8Ou72XP3giQgB+DCrJTOFBdR7WbqXtbSqpo0jDAbQ8/icNH66mpb3R5bmuJEfD7yhz8sMlIMkZU48wmfnlGH1Y9MIGC7BSW/PFMl3NnLNvjcsybG6cv4/FPN7L3cHg2zHHLmuw952/G/P6p70HvMy0reycbK3u//guUboxcG9sZGcOPQfZF1AbkOQb2DfuNfVTdDenYz8V3Ln28ucSooRPsDcSFZwvuOYMGSy/XZFJkpRgzdzqlt26XMavahkbbfrqvLdjBr87sG9L9E3ziUMa50hjeWT0TFv4DFvwd8oYa4/3HXQIZXVq+nmgV6eHHIOtcfHcbmq/ff4TkeLPtHHveVttuKa2id67M0AmnzJR4ctIS3T738tUjATyWqd5RXs3ROveLs+7/YK3t92mLd3Lhi4vocd8c3lmyO8AWB0liOgy7wljZ+38bYfLjYIozkr1PD4TpF8jK3hCRgB+DbIuv3Oxvu7H4CP3z0t0G7rwMz4uvjBo6MpwTLSYNzmPn4+fyt0uHkp4Uh1LYxuMbGps4/al5/PLtlbbzl+w4SI/75jDy4S+ZuazI4Vq7LB2D389e4/a99h0+xoylEfowcJvs3eWU7J0ryd4gkYAfg7JS4klLjHNJ3Gqt2bDf/QwdgM6Z7ssryAyd6JWeFM9t43ujtZG8Bfh6Y6nD/wJc9vL3AByo9j8wXvX6j9z7/hqO1HgvrBdytmTvKiPZO+JqS7L3SiPZ+8ndkuwNkIzhxyCllKVMsmPA319RQ8Wxega6maEDxmKf1AQzxRWOG29Yl/dLwjY6WbdarKptICnezC3/Xh7U628vqwZgweZyzh0aBePn9it7Jz9mrOxdPQNWvQPL3jBW9g691Ni6UVb2+kUCfowq6JDMjvJqh2Mbiz0nbMH4oOicmUTxEcehoC0lMiUzmqUkGP+ZHq1tBKfP5M/WFpPfwfda/ZuKK1m7t4KKY/Vcf3JPh2mbd7y9gnOHnhuUNgeNJHuDSgJ+jCrMTmH+ljK01rZVmxv2G8XP+ud5Dtx5GUkuSduNxZUkxcsMnWiVlmhsBl9d1+AypfbW/3ju7X/zm9PomJ5IemIcv56xig9X7WPSM/Ntz3fNSnLZ2KW61ngPT8nkiLIme4ddYbeyd4bTyt7LLSt73Xd62jsJ+DGqMCeFmvomyqpqbdP4Nuw/Qn4H15IK9vIykvhxx0GHYz8VHea4rpkyQydKWXv4Zz+7gIuP7+bTa644oYBeuc1fB+ZvLnM559b/rHA5NvjBzwHY9MhkEuPMrWlueFiTvSfeBuVbYM17RvD/8HaY83/Qb7IR/PucBXERnpIaRSRpG6Nsc/HtxvE37D/icTjHqnNmEiVHamhqMr7K1zc2sXZvBcMLskLWVhGYVLtCa7NX7PV43nNXjuCFqSPITk3g0YuGODw3vn8nv96zsiZC9fhbo2NfSfb6SHr4Mco6/LL74FFGds+mpr6RHeXVnDvE+zhmXkYSDU2aA9V15KYnsnF/JbUNTQwvzApDq0VrJMf71tM+sWc2nTKSHDZOt/r7pcP430rPHxbOjtW5rsaOei7J3m+M8X5J9tpIDz9GWRN11g3NN5dU0qQ9J2ytnHe+WrXHKKUsPfzolZzgGvCfv3KEy//X2V5W05r8HK6L2I5bwWKOh36T4JLX4Xdb4KKXjW8CC/9hKeN8iqWM8/5ItzSsJODHqKR4M50zEm118a0lFdzV0LHXxWm17co9h+mYlkC3LN9neojwSop3/c/0/GFdyUx2/IIeZ27df85/mTLY5ZjzKt45q/e7rcEUE9yu7DU7rez9L9QciXRLQ04CfgyzL5O8YX8lKQlmurcw08ZaXmG/rYd/mOEFWR7rs4vIS3JKnna1/H+4psi/0gPOK6kzk+NZeO/pXHNSD+bcdbLDc2//2Fys7YftB7jj7RU8/mlki5w1NWmXDyKtNUVedn9zYVvZOw/uXAan/s6ysvd2eKovvPeLNr2yV8bwY1hBhxR+2H4AMHr4/fPSW/zq3jEtEbNJUVJRQ8XReraXVXPxCN9mfojI6JCawAd3jKO44hi3/mcFKZYkbrXdOPu8345v8TpPXzac855fCBglme2/EQzu6rhByfsrinh/RRFpiXG24Z3SSs+b54RDrz/MBeCr/zuVPp2MqcfXvLGEBVvKefvGMYx1U1raq4594Yw/GgnfoqXGeP+62bDuf5Dcwdizd8hlbWrP3rbxV7RTBdkp7D9SQ21DIxv2H3GpnOmO2aTITTPq4v9UdBiA4QUdQtxSEajhBVlkJhtj9KluxvR7dEx1OeasU3rz3Hp3wz//vXGMyzH7sfxomab53nKjVtCuA9Us2FIOwNTXfmz9BZWCgtFw7lOWMs4zjTLOq94xyjg/13bKOEvAj2GF2SloDct3HuJITYPbTU/csU7NXLXnMErB0ALZfi4WWFfFukvi+sIasK15HGfj+nRkyR9ca/E3vz5y4aKuoXlK5ZBuxr/X0/42L/hv5C7Zm9PHKdn7QswmeyXgxzDr1Mwv1pcALSdsrfIyEimuMAJ+79w0rwu1RPQ4ahnCSU1wHIl11zN3JzMlng/vGMc3vxnv8ZxOGZ5r8c/bVMaCLa4LuLz5ac9hetw3h8Vby/16nbPZK5orgN759krqG13n1D/9xSZq6htZty9IZZVtyd7/GcneSY9Zkr1/jNlkrwT8GGYtk/zFumIABngpqWCvS6axt+1PloStiA0n9+3IlOFd+cuFxwHGTB1wvy2iJ8MKslr8hvDclSPcHi8+UsPVry/x+b0ApvxzEWAMuSxxWuHtj/ucSjt/urbY5ZznvtnKve+v5tznFrK/Isg7faV3hpNuj/lkrwT8GNYpPZGEOBP7KmooyE4m3ceeeueMJCprGzhQXccwCfgxIynezLNXjLBNoX36smGsfmhi0N/n/KFdOKWvnwlQH/zz262tep11Vbi9u95p3gvg9P65tt8/XLUPgHv83APYL9Zk712r4IYvjZW9279zXNm7+weIxH7CLZCAH8NMJmVbgOVLwtYqL7M5eTdCAn7MijebQjIcp5RiyvDAZm4t2lrOnNWO49wNrSxtMPW1H7w+/9q1J7gcsyZzQ8o+2fvbzZZk7xlGsveNSfDsUCPZW7Yp9G3xkUzLjHGF2SlsL6tucYWtPetq28Q4k9fKmqL90gH2Tn/uZtbM6X7W87Gy38pzXJ8cFm09YHu88/EoKedsTfb2m2RXxnmGUxnny+G4n0W0jLP08GOctYjaQD8Ct3WrwyHdMolv5epM0bY12A2j9HKa8rm9rKpV12zt4r5fntnX9vvDU47z+XURKw/RUrL3rSkRS/aG9b92pVSqUmqZUuq8cL5vW9Y9xwj4g7r6M6SThEnBCCmYJjxosMyCyUlN4J2bT3R4rrWBtLWlGdbsNWbdXDIyn64eSoC4SzQfZyn1HFHukr2HdkYs2etTwFdKvaGUKlVKrXU6PlkptUkptVUpdZ8Pl7oXmNmahgr3LjuhgJevHkn3nJYX3lilJMTx+i9O4NbTeoewZSKWZaYYi7yuP7mnbQjQKqGF+fjuhoOUgtpWBvy3fzQ2WP/zBYNJsqsceuaA5iGiMT2zbb8/fGHztwBPs3W01rYPtWmLdtDjvjnsdNpBLuiiINnr6xj+NOAF4C3rAaWUGfgnMAEoApYqpT4CzMBjTq+/HhgGrAc8T/QVfstIimfS4Dy/X9fa8VTRPpw/tAtaa7fltluKR3Vu5sgDVNUGVnwtyalM9MtXj7T9bj/V9BS7aaonPfaN23H+1xfu4JE5G7jqxEL+84PxgTL+qXmtygkUV9SwtbSK0T2zW/wwBJqTvQWj7co42+3Zm1UIQy6FUddDZr7f7fHGp4CvtZ6vlOrhdHg0sFVrvd34G9S7wBSt9WOAy5CNUmo8kAoMAo4ppeZqrV3+ZSilbgZuBigsLPT5DxFCBI/zTJ2pYwptPe2GRu8R/63FuxweJ8SZqGto4o1FO7htfG9OePQrh3o43izfZczdTzCbbDuyuQvKaQlxnN4/l6ljutuGOb15ZM4GAFuwt7LfMtQX28qqOPPv39ke+/2B4Zzs3fAJrLHs2TvgvKAH/EDG8LsBe+weF1mOuaW1/qPW+tfA28Cr7oK95bxXtNajtNajcnNz3Z0ihAizv140hDd/YUx/PP+FhV7P/WpDicNj+6HD05+aB8BNb3nei9fez/71PQC9cr0PWZpMijevG82EQZ1RSnFRKwsCWj8IfHWN00K0RjdrBnyWmA7DrzSSvb/ZBF3dL4ALRNinaGitp2mtPwn3+wohAmM/o2vXAc/j3Uftqng+fdkw7j6rLwPy0pk4qLMt4etc5rglzsM5Lfm/Cf38Ot/q9YU7/EpK7z3smCM4dDRIyde0TsbQT5AFEvD3AgV2j/Mtx4QQbZB9heDDR+s9ntc5o3lhX99O6SiliDebHOrfmHwIZvYrbHcf9KPmPY4F4uaucVwA9sI3W7y+9rgHP/f7A8nqcLACfogEEvCXAn2VUj2VUgnAFcBHwWmWECLa7CxvDrpmL/suHDlmBMtfn9WX47oZ04X3V9SweFvzgqn9FS3X1h9vGf4BOFjtXyC1L//8wIfG5MKHP1nP2Me+5qkvNrf4+pZ6+VW1DVxgN7R186m9ADjr6fl+tTPcfJ2W+Q7wPdBfKVWklLpBa90A3Al8DmwAZmqt14WuqUKISNI097jX7vVckXLJTiPR+qsz+9oSoOVVtdQ2OKbtbpy+1Ov7+dur96S8yviweH3hDvZ5+KC58/Q+Do9r6ryXgTjuwc9ZbbfjWFcPJaejjU8BX2t9pda6i9Y6Xmudr7V+3XJ8rta6n9a6t9b60dA2VQgRSfZlmZ2rV7rT0myXrzaUBtwmX7krwAZw2ah8nr1iOL+d1N/huL+Ly64c0zyj8KnPo6d2jjNZVy+E8MkFlnLMseiHHQdcjt02vjdPXjLMNv3UfoOXc55bwMvfbfPp2jseO8dhN7AXWlkVNBwk4AshfGIyKW45rVeL52WlxLvsqvXLM/p4ONs3fzxnYECv31bmOqvo2pN6ODy+/uSeDo8f83HTdnffZGrqGwMuQBcKEvCFED47w26FtnWYpLahkdqG5qmYh4/WuyRlEwIo0rfz8XO56dSWP2iczb3rFNvv93+w1uX5sspah8c93ewL3OO+Oexzmnr5057DLb73gPs/o+fv57a6flCoSMAXQvjMfvbLDstc/KEPfUH/P33m9XXxfu6H++r87f43zklLBQVz0hIcHp831H3Z4unf73R4fMu/3S8a+8XYHi7HyqtqXU+MIAn4Qgif2ffU40yKuoYm2+ybq177kWN17nu0Zg8JXE8rUx+d69+K19bomJbo8DglIc5taYRlOw/Zft9eVkXxkeZvL+/depLtd3eLwzaXVHKwui5qAr8EfCGEz/YedpyLX2M3lLNwaznvLNnt7mX84yv3c9/dbUYeTLeNd18R9vkrR/hW6AxYvssI+N9uKuUMu7o50647gRN6NFfpvP101/fqlJ7E8Q9/yahHvvKn2SEjAV8I4TP7sgk19U0u0x3/8sn6Fl9nL9QB376Esr3W7Nn7sWW/XKvxThVn3W03GbFNWDyQgC+E8Nlwuz2Q/zVvG/UtVM5syZoizwu4gmGUXQ/cXlZKgtvjAAvuOd3t8dkr/a8cc8Ur3vfjDTcJ+EIIn9kHytqGRo8bkztXt7zFbpbNoxc1b1BiHS6xF+npjAXZKSz701mteu2r14zy+Fyk/y6QgC+E8EO82TH5Wt/gPogNy89yeNzfbs9l+8JpjW6CoP1+ug+dP6g1zXTw8JTBDo8/uGNci69xTug++Zlvc/KdZ/7Y87QxTDhJwBdC+CzdbpxaKUW9hx5+epLj3krOcf3eyQMAeOarLS55APtx/eGFHQJpLgCd7LZofPD8QQ7DUt68f9tY2+8vzvNt1a11O8hxfXJcnqtrkIAvhIgxd53ZF4C0RLPDbk/2OjiNkfd0GuIZlp9p+73J6dPA/ltDnJeqnL6yv4Lz6lpvRnb3/GGz6ZHJbo93y0pm0X1n8Nb1Y1yecy4eFwkS8IUQfpk4qDMA7yzZ43DcvlfrXD75+MIOvH3jGMb3z+W8oV0cnm9w7uHbfWuIMwce8O2vbvLzA+T+81yHlJ66dJhD7Rxn3bKSMZsU6/8yyeF4NKy6lYAvhPCLp1r49ouoLj+hwOX5sX06Mu260aQnxTsE8gH3f8am4kp63DeHt3/c7TCkE4w8p6dKmb5IdDNX/5KRvu0zm5LgOKwlPXwhRMzxtFuV/XB+5wzv9eHNJsfQ89oCo5TCawu3Owzp/Ljdtcqlv9wlhn1l3bg9GKSHL4SIOZ5GRawbn/jCeWze+hmicBzSGdwtk0A1BLBWIKBNyXHcW7e2oSniUzMl4Ash/NLSxia+cB4WUpbUqlLKIUA7l1lujR5uqmD6KjPFcfXsm9ed4Nfr7zqzL/+90UjgXvziYnr+fm6r2xIMcS2fIoQQzYIwcYbKGseSA9YRHpNynJaZ3yEl4PcaXpDFuzefyPGtmOJpXy7hNxP6cXp/96UavPl+W+DDUsEiPXwhhF+C0cNvcFqEtH5/pXFtlMusnWA4sVeOz8XS7NkXX9te7rqJii/s9wqAyK64lYAvhPCL87z51nD+0LBuKrKppNL2YfCvnx8f8PsEKiWhefrl/1pRSwfgbrtxfAg8LxAICfhCCL+466H6u6OVt2Eha0E25/HzSAjGwq9kpzr5ofgG4ysJ+EIIv7iLV+7mq3szoIvn3aisBdniA9gWMVj8XajljvO3mc/WFrO/4piHs0Mr8ndUCBFT3I3oZKX61xtPTfC8UnXvISMYBqN3HSj7nbo8LTjz169nrGLC0/ODci1/ScAXQvjF3Rj+iAL/ZsB4S/wesyxQClaADYT93zrKS20df0VqYxQJ+EIIv6S46Z37G5x9OVv5dFZo2Q9fndovN3INCRIJ+EIIv3TPcV3IdKC6zq9reJvZaS3dEITZn0HQHPFzUj3Xuo8VEvCFEAGbv7nMr/ODMZc/HHJSmzdCiYYkcqBi/y8QQkSVjl52fbI3trfrJiEQLT17Qwe7Xv24Pv5vfG5lX/8/kiTgCyGC6rkrR/h0nrtcADSP70dL4LfOFjIFEC1n3z6O6dePDlKLWk8CvhAiqDKTfZui2dKC3WhI2oJdTiGA9phNitOiIOkrAV8IEVS+1n33FO8jW0DYlfWbRjhmiTY2aTaXVIbs+hLwhRBB5ltk9FREzLpDVbQM6TQH/NA36MnPNzLxH/PZXlYVkutLwBdCBJWvK2R/Pqa72+PWj4FoCfjWQB+OgP/52mIA9hwKTekFCfhCiIAMdZqB4uvG42cN6szGhye7HP/zx+uD0q5gCdWK36JDR12O7TxgHLtx+tKQvKcEfCFEQF6+eqTD42AFyGhJ2lrn3zc0BXcT8pOf+Nbjc/UBbMvojQR8IURA0pPi6WBXytifoQ9vp0bLkM7143oAkJoY+AaBqx+aaPvdeVpqj/vmBHz9lkjAF0IEROG4KMmfDn44xsUDdecZfdn5+LkkxXuu8Okr+y0Tj9b5NpspmCTgCyECEmdWzNtkX1rBjx5+K59ra5bvOhSW95GAL4QISGJc63u+3mrqxEDnP2h+9q/FLsdCsfetBHwhhN+6ZCY5PLYPTv4E6nYU0/32warW7aHrjQR8IYTfgjX27v0y7fvj4ECVfyWnfSEBXwjhN6+za/y6TvPZCU774ranIZ1wCXyekRCi3XEOxvaBu2dH1w1SfOE8u6c9xPub31rG+P6d3D4Xij0DJOALIfzmPKTjOIbfukAVC1M0g+2L9SV8sb7E7XOhuBthG9JRSpmUUo8qpZ5XSl0brvcVQgRfKIKz87z0WNkVy1+Xjyrw6bxeua37puSNTwFfKfWGUqpUKbXW6fhkpdQmpdRWpdR9LVxmCpAP1ANFrWuuECIaeBvSCdp7BP2K0eGvFw/x6bw8p5lQweDrkM404AXgLesBpZQZ+CcwASOAL1VKfQSYgcecXn890B9YrLV+WSk1C/g6sKYLISLFuYffVoNzKPhaa8gcqTF8rfV8pVQPp8Ojga1a6+0ASql3gSla68eA85yvoZQqAqzzjMK/plgIETQuoSgEEb+Njuj4rLyqjr6dg3vNQMbwuwF77B4XWY55MhuYpJR6Hpjv6SSl1M1KqWVKqWVlZWWeThNCRFA4evjRUi0zUtbsPRz0a4Ztlo7W+ihwgw/nvQK8AjBq1Kho2+1MCIH0vsPh8NH6oF8zkB7+XsA+3ZxvOSaEEAFr7x8qwajO6SyQgL8U6KuU6qmUSgCuAD4KTrOEENHMeVZOMGbpdMtKDvgabYnzTmLB4Ou0zHeA74H+SqkipdQNWusG4E7gc2ADMFNrvS7oLRRCRB3n8J4UH/iSnrsn9Av4Gm1Jbnpi0K/p6yydKz0cnwvMDWqLhBAx568XDeGG6csCuka82flbQ0CXizkmBU12WUvr1opBfY+gX1EI0eY5B+Ps1IQgXDP4w0Sx5KyBjnMwQ7GaWQK+EMJvzrEoFKV821e4d72nQdoL3vGawb+kEKKtc54jv/vg0SBcs31zvqe+rsj1hwR8IYTfXGvpBH7NlbsPB/2ascTkFI1lSEcI0Wa9sWiHw+P2ttLWOWdhkh6+ECIauCRYI9SOtiQ90XHSpIzhCyGiQjgCfPsb0lEsvPd0OqTEG49lSEcIEQ2kHn7wHayqI79Dii1ZKwFfCBEV2lswDofaBqNqvHW3yFAM6cietkKIgIVk+KWdfqq8fdOJzF5RFJTFbM4k4AshAib18ANnrarQPy+d358zMCTvIUM6QggRBZLigl8O2ZkEfCFEVGpvs3QykkM/4CIBXwjht3BsRdfO4n1YSMAXQvityTnih2JaZjvr4ocjZyEBXwgROB14n39cn5wgNER4IwFfCBEVDlY7btrdvvr34SEBXwjhvyD06J05LzRqZyM6YSEBXwjht1Akbdt7gA/H3y8BXwgRFZyTlu1t4VWjSyY8+CTgCyEClpIQ+Bxylx5u+4r3LNt1KOTvIQFfCBGwE3sHPsMmBGkB4UQCvhAiKoRi20ThSAK+ECIqtPddtMLx90rAF0L4LRTDL+0twLuQWTpCiGikQzAxMxy7aEUL6zaG9gqzU0L+vhLwhRB+c54yqUPQ5W+74d69rGTXD4Fgk4AvhPBbKDrfLrMy21vEDwMJ+EIIv4VkDF8ifMhJwBdC+C0UY/gutXTa8KCOu7sXjmUIEvCFEAELRo/fpbRC2433btU1NIX8PSTgCyGiQii+NcSST9cWh/w9JOALIUQ7IQFfCCHaCQn4Qgi/hWalbTsbtI8ACfhCiIAF5QNA4n3IScAXQvgtFD38OOd5mW3YtSf1iMj7SsAXQvgtFFMmu2UlB/+iUeruCf3Y+fi5YX9fCfhCCNFOSMAXQvjN05BOl8ykVl9TNkAJPQn4QoigiTNLlI5mEvCFEH5z7n0HY5WsTMsMPQn4Qgi/eRrSCSRo56Yntvq1wjcS8IUQfnOO94lxZgAKsls/02Z8/9wAWiR8EReuN1JKFQLPAQeBzVrrx8P13kKI0MrLTOKlq0ZyUq+coF1ThniCz6cevlLqDaVUqVJqrdPxyUqpTUqprUqp+1q4zBBgltb6emBEK9srhIhSk4/LI9PNXq0ievjaw58GvAC8ZT2glDID/wQmAEXAUqXUR4AZeMzp9dcDPwCzlFLXA/8OrNlCCCH85VPA11rPV0r1cDo8Gtiqtd4OoJR6F5iitX4MOM/5Gkqp3wIPWq41C3gzoJYLISImFIMtcWZJKYZaIGP43YA9do+LgDFezv8MeEgpNRXY6ekkpdTNwM2Wh1VKqU2W3zOBCqfTnY/ZP+4IlHtpTyDctSVYr/F2nqfnfLk37o5F8/3y9XXBul/ujsv98v5cJlChnvB4jl/3a8QTjo8Tn3B/no9tC9brWjonkP8mXR6rJ4L276u726Naa59+gB7AWrvHlwCv2T2+GnjB1+v5+wO80tIx+8fAsnC2JViv8Xaep+d8uTexdr98fV2w7ldL96c93y9Pz7X3++Xt+Wi9X4F8h9oLFNg9zrccC5WPfTjm7pxQaM37+Poab+d5es6Xe+PuWDTfL19fF6z75e643C/vz7X3++Xt+ai8X8ryadLyicYY/ida6+Msj+OAzcCZGIF+KTBVa70uNE31j1JqmdZ6VKTbESvkfvlH7pd/5H75J1T3y9dpme8A3wP9lVJFSqkbtNYNwJ3A58AGYGa0BHuLVyLdgBgj98s/cr/8I/fLPyG5Xz738IUQQsQ2mQclhBDthAR8IYRoJyTgCyFEO9FuAr5SKlUpNV0p9apS6ueRbk+0U0r1Ukq9blkVLVqglLrQ8m9rhlJqYqTbE+2UUgOVUi8ppWYppW6LdHtigSWGLVNKuVQy8FVMB3w/i7pdjFG87SbggrA3Ngr4c7+01tu11jdEpqXRwc/79YHl39atwOWRaG+k+Xm/NmitbwUuA8ZFor2R1oqilPcCMwN5z5gO+BhF3SbbH7Ar6nY2MAi4Uik1CGNhmLUURGMY2xhNpuH7/RKtu19/sjzfHk3Dj/ullLoAmAPMDW8zo8Y0fLxfSqkJwHqgNJA3jOmAr7Wej1Ff356tqJvWug54F5iCUesn33JOTP/dreXn/Wr3/LlfyvAE8KnWekW42xoN/P33pbX+SGt9NtAuh1j9vF/jgROBqcBNSqlWxbCwbYASRp6Kuj0HvKCUOpfwLfmOBW7vl1IqB3gUGKGU+r02qqAKz/++fgmcBWQqpfporV+KROOikKd/X+MxhlkTab89fHfc3i+t9Z0ASqlfAOVa66bWXLwtBny3tNbVwHWRbkes0FofwBiPFj7QWj+H0akQPtBazwPmRbgZMUdrPS2Q17fFoY1wF3WLdXK//CP3yz9yv/wT0vvVFgP+UqCvUqqnUioBuAL4KMJtimZyv/wj98s/cr/8E9L7FdMBP0aLukWM3C//yP3yj9wv/0TifknxNCGEaCdiuocvhBDCdxLwhRCinZCAL4QQ7YQEfCGEaCck4AshRDshAV8IIdoJCfhCCNFOSMAXQoh2QgK+EEK0E/8PfxFLENh4RckAAAAASUVORK5CYII=\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "num_samples_single = 10000\n", - "trip_id_to_check = 1767182865 #bp.index.values #[100:103]\n", - "\n", - "_, c_, _ = mode_choice_for_trip(choose_individual_max_utility=True,\n", - " trip_id_to_check=trip_id_to_check, num_samples=num_samples_single)\n", - "c_ = c_[['trip_mode']].reset_index(drop=True)\n", - "\n", - "temp = bp.loc[bp.index == trip_id_to_check]\n", - "temp.index = [\"exact\"]\n", - "temp = temp.T.reset_index().rename(columns={\"index\": \"trip_mode\"})\n", - "ms = (c_.value_counts() / c_.shape[0]).to_frame(\"drawn\").reset_index().merge(temp, on=\"trip_mode\", how=\"outer\").fillna(0)\n", - "ms = ms.loc[(ms.exact != 0)]\n", - "display(ms)\n", - "c_test = c_.copy()\n", - "c_test.index = np.arange(1, c_test.index.shape[0]+1)\n", - "check_mode = \"WALK_LRF\"\n", - "ms_exact = ms.loc[ms.trip_mode == check_mode].exact.values[0]\n", - "\n", - "c_test['trip_mode'] = np.where(c_test['trip_mode'] == check_mode, 1, 0)\n", - "c_test = c_test.cumsum()\n", - "c_test[\"share\"] = c_test.trip_mode / c_test.index\n", - "c_test[\"diff\"] = np.square(c_test[\"share\"] - ms_exact)\n", - "\n", - "plt.plot(c_test.index.values, c_test['diff'].values)\n", - "plt.plot(np.arange(1,num_samples_single), 1.0/np.arange(1,num_samples_single))\n", - "plt.yscale('log')\n", - "plt.xscale('log')\n", - "plt.ylim([1e-8,1])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "# Dev corner" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "## try to improve runtime of apply\n", - "\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 239, - "outputs": [ - { - "data": { - "text/plain": " DRIVEALONEFREE DRIVEALONEPAY SHARED2FREE SHARED2PAY \\\ntrip_id \n642446345 0.0 0.0 0.004883 0.0 \n\n SHARED3FREE SHARED3PAY WALK BIKE WALK_LOC WALK_LRF \\\ntrip_id \n642446345 4.701403e-14 0.0 0.044905 0.0 0.059962 0.889736 \n\n WALK_EXP WALK_HVY WALK_COM DRIVE_LOC DRIVE_LRF DRIVE_EXP \\\ntrip_id \n642446345 0.0 0.0 0.0 0.0 0.0 0.0 \n\n DRIVE_HVY DRIVE_COM TAXI TNC_SINGLE TNC_SHARED \ntrip_id \n642446345 0.0 0.0 0.000096 0.000131 0.000287 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DRIVEALONEFREEDRIVEALONEPAYSHARED2FREESHARED2PAYSHARED3FREESHARED3PAYWALKBIKEWALK_LOCWALK_LRFWALK_EXPWALK_HVYWALK_COMDRIVE_LOCDRIVE_LRFDRIVE_EXPDRIVE_HVYDRIVE_COMTAXITNC_SINGLETNC_SHARED
trip_id
6424463450.00.00.0048830.04.701403e-140.00.0449050.00.0599620.8897360.00.00.00.00.00.00.00.00.0000960.0001310.000287
\n
" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "nest_spec = ns[7] # 7 is social\n", - "nest_util = nu[7].loc[nu[7].index == trip_id_to_check]\n", - "\n", - "n = 10000\n", - "\n", - "nest_util = nest_util.loc[nest_util.index.repeat(n)]\n", - "nest_util.index += np.arange(n)\n", - "\n", - "display(base_probs[7].loc[base_probs[7].index == trip_id_to_check])" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 240, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 31.2 ms, sys: 0 ns, total: 31.2 ms\n", - "Wall time: 21.1 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "from numpy.random import default_rng\n", - "from activitysim.core import logit\n", - "\n", - "rng = default_rng(999)\n", - "\n", - "def random_for_df(df):\n", - " return rng.random(size=df.shape)\n", - "\n", - "def add_ev1_random(df):\n", - " uniform_rands = random_for_df(df)\n", - " df = df + logit.inverse_ev1_cdf(uniform_rands)\n", - " return df\n", - "\n", - "nest_utils_for_choice = add_ev1_random(nest_util)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 241, - "outputs": [], - "source": [ - "def group_nests_by_name(nest_spec):\n", - " # group nests by level, returns {level: [nest.name at that level]}\n", - " depth = np.max([x.level for x in logit.each_nest(nest_spec)])\n", - " nest_levels = {x: [] for x in range(1, depth+1)}\n", - " for n in logit.each_nest(nest_spec):\n", - " nest_levels[n.level].append(n.name)\n", - " return nest_levels\n", - "\n", - "def make_choices_apply(nested_utilities, all_alternatives, logit_nest_groups, nest_alternatives_by_name):\n", - " for level, nest_names in logit_nest_groups.items():\n", - " if level == 1:\n", - " next_level_alts = nest_alternatives_by_name[nest_names[0]]\n", - " continue\n", - " choice_this_level = nested_utilities[nested_utilities.index.isin(next_level_alts)].idxmax()\n", - " if choice_this_level in all_alternatives:\n", - " return choice_this_level\n", - " next_level_alts = nest_alternatives_by_name[choice_this_level]\n", - " raise ValueError(\"This should never happen - no alternative found\")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 242, - "outputs": [], - "source": [ - "%%prun\n", - "all_alternatives = set(nest.name for nest in logit.each_nest(nest_spec, type='leaf'))\n", - "logit_nest_groups = group_nests_by_name(nest_spec)\n", - "nest_alternatives_by_name = {n.name: n.alternatives for n in logit.each_nest(nest_spec)}\n", - "\n", - "choices_apply = nest_utils_for_choice.apply(\n", - " lambda x: make_choices_apply(x, all_alternatives, logit_nest_groups, nest_alternatives_by_name),\n", - " axis=1\n", - ")" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 243, - "outputs": [ - { - "data": { - "text/plain": "WALK_LRF 0.8383\nWALK_LOC 0.1158\nWALK 0.0405\nSHARED2FREE 0.0048\nTNC_SHARED 0.0003\nTAXI 0.0002\nTNC_SINGLE 0.0001\ndtype: float64" - }, - "execution_count": 243, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "choices_apply.value_counts() / n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "# stuff\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 172, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "register joint_tour_participants: no rows with household_id in [982875].\n" - ] - } - ], - "source": [ - "from activitysim.core import pipeline\n", - "\n", - "resume_after = \"trip_scheduling\"\n", - "model_name = \"trip_mode_choice\"\n", - "chunk_size = 0 # test_mtc means no chunking\n", - "\n", - "pipeline.open_pipeline(resume_after)\n", - "# preload any bulky injectables (e.g. skims) not in pipeline\n", - "inject.get_injectable('preload_injectables', None)\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "#step_name = model_name\n", - "args = {}\n", - "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "\n", - "finalise = True\n", - "if finalise:\n", - " inject.set_step_args(None)\n", - " #\n", - " pipeline._PIPELINE.rng().end_step(model_name)\n", - " pipeline.add_checkpoint(model_name)\n", - " if not pipeline.intermediate_checkpoint():\n", - " pipeline.add_checkpoint(pipeline.FINAL_CHECKPOINT_NAME)\n", - "\n", - " pipeline.close_pipeline()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 176, - "outputs": [], - "source": [ - "t_ = trips.to_frame()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 177, - "outputs": [ - { - "data": { - "text/plain": " person_id household_id primary_purpose trip_num outbound \\\ntrip_id \n2464104885 7512514 2821179 social 1 False \n\n trip_count destination origin tour_id purpose \\\ntrip_id \n2464104885 1 8 9 308013110 home \n\n destination_logsum depart \ntrip_id \n2464104885 NaN 16.0 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
person_idhousehold_idprimary_purposetrip_numoutboundtrip_countdestinationorigintour_idpurposedestination_logsumdepart
trip_id
246410488575125142821179social1False189308013110homeNaN16.0
\n
" - }, - "execution_count": 177, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t_.loc[t_.index == 2464104885]" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 52, - "outputs": [], - "source": [ - "from activitysim.core import config\n", - "resume_after = \"trip_scheduling\"\n", - "model_name = \"trip_mode_choice\"\n", - "\n", - "pipeline.open_pipeline(resume_after)\n", - "# preload any bulky injectables (e.g. skims) not in pipeline\n", - "inject.get_injectable('preload_injectables', None)\n", - "pipeline._PIPELINE.rng().begin_step(model_name)\n", - "#step_name = model_name\n", - "args = {}\n", - "#checkpoint = pipeline.intermediate_checkpoint(model_name)\n", - "inject.set_step_args(args)\n", - "\n", - "trips = inject.get_table('trips')\n", - "tours_merged = inject.get_table('tours_merged')\n", - "network_los = inject.get_injectable('network_los')\n", - "\n", - "trace_label = 'trip_mode_choice'\n", - "model_settings_file_name = 'trip_mode_choice.yaml'\n", - "model_settings = config.read_model_settings(model_settings_file_name)\n", - "\n", - "logsum_column_name = model_settings.get('MODE_CHOICE_LOGSUM_COLUMN_NAME')\n", - "mode_column_name = 'trip_mode'\n", - "\n", - "trips_df = trips.to_frame()\n", - "\n", - "tours_merged = tours_merged.to_frame()\n", - "tours_merged = tours_merged[model_settings['TOURS_MERGED_CHOOSER_COLUMNS']]\n", - "\n", - "# - trips_merged - merge trips and tours_merged\n", - "trips_merged = pd.merge(\n", - " trips_df,\n", - " tours_merged,\n", - " left_on='tour_id',\n", - " right_index=True,\n", - " how=\"left\")\n", - "assert trips_merged.index.equals(trips.index)\n", - "\n", - "# setup skim keys\n", - "assert ('trip_period' not in trips_merged)\n", - "trips_merged['trip_period'] = network_los.skim_time_period_label(trips_merged.depart)\n", - "pipeline.close_pipeline()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": 56, - "outputs": [ - { - "data": { - "text/plain": "Index(['person_id', 'household_id', 'primary_purpose', 'trip_num', 'outbound',\n 'trip_count', 'destination', 'origin', 'tour_id', 'purpose',\n 'destination_logsum', 'depart', 'hhsize', 'age', 'auto_ownership',\n 'number_of_participants', 'tour_category', 'parent_tour_id',\n 'tour_mode', 'duration', 'value_of_time', 'tour_type',\n 'free_parking_at_work', 'trip_period'],\n dtype='object')" - }, - "execution_count": 56, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trips_merged.columns" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file From e1e9d088d706019f9c09639aded311dc31dae8ac Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 2 Apr 2025 15:48:46 +1000 Subject: [PATCH 099/135] seeding corr test --- .../2025/correlation_variance_tests.ipynb | 706 ++++++++++++++++++ 1 file changed, 706 insertions(+) create mode 100644 notebooks/2025/correlation_variance_tests.ipynb diff --git a/notebooks/2025/correlation_variance_tests.ipynb b/notebooks/2025/correlation_variance_tests.ipynb new file mode 100644 index 0000000000..3b1ae83dd9 --- /dev/null +++ b/notebooks/2025/correlation_variance_tests.ipynb @@ -0,0 +1,706 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Which correlations do we want\n", + "\n", + "simulate destination mode choice model:\n", + " 1) MC\n", + " 2) EET fixed error per level\n", + " 3) EET only fixed when identical\n", + "\n", + "Does it make a difference in analysis?" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2022-05-01T05:31:19.564033Z", + "iopub.status.busy": "2022-05-01T05:31:19.563663Z", + "iopub.status.idle": "2022-05-01T05:31:19.824207Z", + "shell.execute_reply": "2022-05-01T05:31:19.823432Z", + "shell.execute_reply.started": "2022-05-01T05:31:19.563960Z" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "from numpy.random import default_rng\n", + "from copy import deepcopy" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "rng = default_rng(999)\n", + "\n", + "### Zones\n", + "num_zones = 10\n", + "destination = pd.DataFrame(\n", + " data=zip(np.arange(0, num_zones), rng.normal(1000, 10, size=num_zones), strict=True),\n", + " columns=['zone', 'attrs'],\n", + ")\n", + "\n", + "### People - assume each person makes one trip from random origin\n", + "num_ppl = 10000\n", + "vars=['id', 'income', 'orig']\n", + "ppl = pd.DataFrame(\n", + " data=zip(\n", + " np.arange(0, num_ppl),\n", + " rng.normal(50, 20, size=(num_ppl)).clip(2, 150),\n", + " rng.choice(np.arange(0, num_zones), num_ppl),\n", + " strict=True,\n", + " ),\n", + " columns=vars,\n", + ")\n", + "\n", + "\n", + "### Modes\n", + "num_modes = 3\n", + "mode_idx_to_str = {0: 'car', 1: 'pt', 2: 'walk'}\n", + "modes = np.array(['car', 'pt', 'walk'])\n", + "mode_asc = {'car': 2.5, 'pt': 0.0, 'walk': 10.0}\n", + "\n", + "### Cost and time by mode\n", + "skims = {}\n", + "skims['time'] = {}\n", + "skims['time']['car'] = rng.normal(20, 20, size=(num_zones, num_zones)).clip(4, 60)\n", + "skims['time']['pt'] = rng.normal(40, 10, size=(num_zones, num_zones)).clip(4, 80)\n", + "skims['time']['walk'] = 3.0 * skims['time']['car'].copy()\n", + "\n", + "skims['cost'] = {}\n", + "skims['cost']['car'] = skims['time']['car'].copy() / 4.0\n", + "skims['cost']['pt'] = 2.0 * np.ones_like(skims['time']['pt'])\n", + "skims['cost']['walk'] = np.zeros_like(skims['time']['walk'])\n", + "\n", + "\n", + "# scenario: decrease pt travel time to and from zone 0 by 70%\n", + "scenario_skims = deepcopy(skims)\n", + "scenario_skims['time']['pt'][0,:] *= 0.3\n", + "scenario_skims['time']['pt'] *= 0.3" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def logsum(utilities, nest_scale=1.0):\n", + " scaled_utils = utilities / nest_scale\n", + " max_util = np.max(scaled_utils, axis=1)\n", + " return max_util + np.log(np.sum(np.exp(scaled_utils - max_util.reshape((-1,1))), axis=1))\n", + "\n", + "def inverse_ev1_cdf(x, location=0.0, scale=1.0):\n", + " return location - scale * np.log(-np.log(x))\n", + "\n", + "### Utility function parameters\n", + "beta_time = {}\n", + "beta_time['car'] = -0.06\n", + "beta_time['pt'] = -0.03\n", + "beta_time['walk'] = -0.2\n", + "beta_cost = -0.01\n", + "\n", + "def mode_utility(mode, income, o, d, skims):\n", + " return (\n", + " beta_time[mode] * skims['time'][mode][o,d]\n", + " + beta_cost * skims['cost'][mode][o,d] / income\n", + " + mode_asc[mode]\n", + " )\n", + "\n", + "def mode_logsum(o: int, d: np.array, income: float, skims):\n", + " mode_utils = np.array([mode_utility(m_, income, o, d, skims) for m_ in modes]).T\n", + "\n", + " return logsum(mode_utils, 1.0)\n", + "\n", + "# mode_logsum(0, np.array([0, 1, 2]), 100)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#inverse_ev1_cdf(default_rng(seed=np.random.SeedSequence([777, 333])).random(100))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "beta_logsum = -0.1\n", + "\n", + "dest_altneratives = destination.zone.values\n", + "num_dests = dest_altneratives.shape[0]\n", + "dest_attrs = np.log(destination['attrs'].values)\n", + "ppl['dest_utils'] = ppl.apply(\n", + " lambda x: (\n", + " beta_logsum * mode_logsum(int(x['orig']), dest_altneratives, x['income'], skims)\n", + " + dest_attrs\n", + " ),\n", + " axis=1,\n", + ")\n", + "ppl['dest_utils_scen'] = ppl.apply(\n", + " lambda x: (\n", + " beta_logsum * mode_logsum(int(x['orig']), dest_altneratives, x['income'], scenario_skims)\n", + " + dest_attrs\n", + " ),\n", + " axis=1,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "### EET\n", + "def destination_choice_model(ppl, seed=777, util_column_name=\"dest_utils\"):\n", + " dest_choice = ppl.apply(\n", + " lambda x: np.argmax(\n", + " x[util_column_name] + inverse_ev1_cdf(\n", + " default_rng(seed=np.random.SeedSequence([seed, int(x['id']), 100])).random(size=num_dests)\n", + " )\n", + " )\n", + " , axis=1\n", + " )\n", + " return dest_choice\n", + "\n", + "def mode_choice_model(ppl, skims, dest_col_name='dest', seed=777, od_seed_contrib=False):\n", + " \"\"\"\"od_seed_contrib: if True, seed depedns on destination\"\"\"\n", + " def seed_o_d(d):\n", + " if od_seed_contrib:\n", + " return d\n", + " return 0\n", + "\n", + " mode_choice = ppl.apply(\n", + " lambda x: np.argmax(\n", + " [mode_utility(m_, x['income'], int(x['orig']), int(x[dest_col_name]), skims) for m_ in modes]\n", + " + inverse_ev1_cdf(default_rng(seed=np.random.SeedSequence(\n", + " [seed, int(x['id']), 200, seed_o_d(int(x[dest_col_name]))]\n", + " )).random(size=num_modes))\n", + " ),\n", + " axis=1,\n", + " )\n", + " #mode_choice = mode_utils.apply(np.argmax)\n", + " return mode_choice.map(mode_idx_to_str)\n", + "\n", + "### MC\n", + "def destination_choice_model_mc(ppl, seed=777, util_column_name=\"dest_utils\"):\n", + " dest_choice = ppl.apply(\n", + " lambda x: default_rng(\n", + " seed=np.random.SeedSequence([seed, int(x['id']), 100])\n", + " ).choice(dest_altneratives, p=np.exp(x[util_column_name]) / np.sum(np.exp(x[util_column_name])), size=1)[0],\n", + " axis=1,\n", + " )\n", + " return dest_choice\n", + "\n", + "def mode_choice_model_mc(ppl, skims, dest_col_name='dest', seed=777, od_seed_contrib=False):\n", + " def seed_o_d(d):\n", + " if od_seed_contrib:\n", + " return d\n", + " return 0\n", + "\n", + " ppl['mode_exp_utils'] = ppl.apply(\n", + " lambda x: np.exp(\n", + " [mode_utility(m_, x['income'], int(x['orig']), int(x[dest_col_name]), skims) for m_ in modes]\n", + " ),\n", + " axis=1,\n", + " )\n", + " mode_choice = ppl.apply(\n", + " lambda x: default_rng(\n", + " seed=np.random.SeedSequence([seed, int(x['id']), 200, seed_o_d(int(x[dest_col_name]))])\n", + " ).choice(modes, p=x.mode_exp_utils / np.sum(x.mode_exp_utils), size=1)[0],\n", + " axis=1,\n", + " )\n", + " ppl.drop(columns=['mode_exp_utils'], inplace=True)\n", + " return mode_choice" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "#for i in range(0, 10):\n", + "# ppl.dest_utils.apply(lambda x: (np.exp(x) / np.sum(np.exp(x)))[i]).hist(figsize=(3,2))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# 10k ppl: per seed about 3.5s total [w/o sedd, with 12.4]\n", + "for s_ in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:\n", + " ppl[f'dest_{s_}'] = destination_choice_model(ppl, util_column_name='dest_utils', seed=s_)\n", + " ppl[f'dest_scen_{s_}'] = destination_choice_model(ppl, util_column_name='dest_utils_scen', seed=s_)\n", + " ppl[f'dest_mc_{s_}'] = destination_choice_model_mc(ppl, util_column_name='dest_utils', seed=s_)\n", + " ppl[f'dest_mc_scen_{s_}'] = destination_choice_model_mc(ppl, util_column_name='dest_utils_scen', seed=s_)\n", + " \n", + " \n", + " ppl[f'mode_choice_{s_}'] = mode_choice_model(ppl, skims, dest_col_name=f'dest_{s_}', seed=s_)\n", + " ppl[f'mode_choice_scen_{s_}'] = mode_choice_model(ppl, scenario_skims, dest_col_name=f'dest_scen_{s_}', seed=s_)\n", + " #ppl[f'mode_choice_seedd_{s_}'] = mode_choice_model(ppl, skims, dest_col_name=f'dest_{s_}', seed=s_, od_seed_contrib=True)\n", + " #ppl[f'mode_choice_seedd_scen_{s_}'] = mode_choice_model(ppl, scenario_skims, dest_col_name=f'dest_scen_{s_}', seed=s_, od_seed_contrib=True)\n", + "\n", + " ppl[f'mode_choice_mc_{s_}'] = mode_choice_model_mc(ppl, skims, dest_col_name=f'dest_mc_{s_}', seed=s_)\n", + " ppl[f'mode_choice_mc_scen_{s_}'] = mode_choice_model_mc(ppl, scenario_skims, dest_col_name=f'dest_mc_scen_{s_}', seed=s_)\n", + " #ppl[f'mode_choice_mc_seedd_{s_}'] = mode_choice_model_mc(ppl, skims, dest_col_name=f'dest_mc_{s_}', seed=s_, od_seed_contrib=True)\n", + " #ppl[f'mode_choice_mc_seedd_scen_{s_}'] = mode_choice_model_mc(ppl, scenario_skims, dest_col_name=f'dest_mc_scen_{s_}', seed=s_, od_seed_contrib=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def display_metric(metric, ppl, seed):\n", + " return ( \n", + " ppl[f'{metric}_{seed}'].value_counts().to_frame(f'eet_base_{seed}').join(\n", + " ppl[f'{metric}_mc_{seed}'].value_counts().to_frame(f'mc_base_{seed}'), how='outer').join(\n", + " ppl[f'{metric}_scen_{seed}'].value_counts().to_frame(f'eet_scen_{seed}'), how='outer').join(\n", + " ppl[f'{metric}_mc_scen_{seed}'].value_counts().to_frame(f'mc_scen_{seed}'), how='outer').fillna(0).astype(int)\n", + " )\n", + " # ppl[f'mode_choice_seedd_{seed}'].value_counts().to_frame(f'eet_base_seedd_{seed}'), how='outer').join(\n", + " # ppl[f'mode_choice_mc_seedd_{seed}'].value_counts().to_frame(f'mc_base_seedd_{seed}'), how='outer').join(\n", + " # ppl[f'mode_choice_seedd_scen_{seed}'].value_counts().to_frame(f'eet_scen_seedd_{seed}'), how='outer').join(\n", + " # ppl[f'mode_choice_mc_seedd_scen_{seed}'].value_counts().to_frame(f'mc_scen_seedd_{seed}'), how='outer')\\" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eet_base_0mc_base_0eet_scen_0mc_scen_0eet_base_1mc_base_1eet_scen_1mc_scen_1eet_base_2mc_base_2eet_scen_2mc_scen_2
car619162155223518861706117517251836163620151745246
pt149114732445248914691493246024291448142624252360
walk231823122332232323612390236823882389237324012394
\n", + "
" + ], + "text/plain": [ + " eet_base_0 mc_base_0 eet_scen_0 mc_scen_0 eet_base_1 mc_base_1 \\\n", + "car 6191 6215 5223 5188 6170 6117 \n", + "pt 1491 1473 2445 2489 1469 1493 \n", + "walk 2318 2312 2332 2323 2361 2390 \n", + "\n", + " eet_scen_1 mc_scen_1 eet_base_2 mc_base_2 eet_scen_2 mc_scen_2 \n", + "car 5172 5183 6163 6201 5174 5246 \n", + "pt 2460 2429 1448 1426 2425 2360 \n", + "walk 2368 2388 2389 2373 2401 2394 " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.concat([display_metric('mode_choice', ppl, i) for i in range(0, 3)], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eet_base_0mc_base_0eet_scen_0mc_scen_0eet_base_1mc_base_1eet_scen_1mc_scen_1eet_base_2mc_base_2eet_scen_2mc_scen_2
0117011041159108911051131108511171138110411231097
1106110381065104410891079109810751056103710621035
2791857801865864906869915882807891814
3103411041034109710781029107010311046106810411062
4951873955881919896930908923920934924
5103210371023103210211053102010421007104610041046
610311028102910239719899729891048105010451046
7871856868862902897899895876932879934
810271026103110339701035975104110159801014985
91032107710351074108198510829871009105610071057
\n", + "
" + ], + "text/plain": [ + " eet_base_0 mc_base_0 eet_scen_0 mc_scen_0 eet_base_1 mc_base_1 \\\n", + "0 1170 1104 1159 1089 1105 1131 \n", + "1 1061 1038 1065 1044 1089 1079 \n", + "2 791 857 801 865 864 906 \n", + "3 1034 1104 1034 1097 1078 1029 \n", + "4 951 873 955 881 919 896 \n", + "5 1032 1037 1023 1032 1021 1053 \n", + "6 1031 1028 1029 1023 971 989 \n", + "7 871 856 868 862 902 897 \n", + "8 1027 1026 1031 1033 970 1035 \n", + "9 1032 1077 1035 1074 1081 985 \n", + "\n", + " eet_scen_1 mc_scen_1 eet_base_2 mc_base_2 eet_scen_2 mc_scen_2 \n", + "0 1085 1117 1138 1104 1123 1097 \n", + "1 1098 1075 1056 1037 1062 1035 \n", + "2 869 915 882 807 891 814 \n", + "3 1070 1031 1046 1068 1041 1062 \n", + "4 930 908 923 920 934 924 \n", + "5 1020 1042 1007 1046 1004 1046 \n", + "6 972 989 1048 1050 1045 1046 \n", + "7 899 895 876 932 879 934 \n", + "8 975 1041 1015 980 1014 985 \n", + "9 1082 987 1009 1056 1007 1057 " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.concat([display_metric('dest', ppl, i) for i in range(0, 3)], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 14441a6281faa8ed684bf731ebf95f834ba6f43f Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 2 Apr 2025 16:26:28 +1000 Subject: [PATCH 100/135] corr --- .../2025/correlation_variance_tests.ipynb | 434 +++++++++--------- 1 file changed, 206 insertions(+), 228 deletions(-) diff --git a/notebooks/2025/correlation_variance_tests.ipynb b/notebooks/2025/correlation_variance_tests.ipynb index 3b1ae83dd9..7b7991c692 100644 --- a/notebooks/2025/correlation_variance_tests.ipynb +++ b/notebooks/2025/correlation_variance_tests.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 32, "metadata": { "execution": { "iopub.execute_input": "2022-05-01T05:31:19.564033Z", @@ -37,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -83,15 +83,15 @@ "skims['cost']['walk'] = np.zeros_like(skims['time']['walk'])\n", "\n", "\n", - "# scenario: decrease pt travel time to and from zone 0 by 70%\n", + "# scenario: decrease pt travel time to (and within zone) 0 by 70%\n", "scenario_skims = deepcopy(skims)\n", - "scenario_skims['time']['pt'][0,:] *= 0.3\n", - "scenario_skims['time']['pt'] *= 0.3" + "#scenario_skims['time']['pt'][0,:] *= 0.3\n", + "scenario_skims['time']['pt'][:,0] *= 0.3" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -127,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -136,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -163,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -232,7 +232,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -242,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -267,7 +267,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -286,7 +286,16 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "#pd.concat([display_metric('dest', ppl, i) for i in range(0, 3)], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, "metadata": {}, "outputs": [ { @@ -329,46 +338,46 @@ " car\n", " 6191\n", " 6215\n", - " 5223\n", - " 5188\n", + " 6048\n", + " 6060\n", " 6170\n", " 6117\n", - " 5172\n", - " 5183\n", + " 6013\n", + " 5962\n", " 6163\n", " 6201\n", - " 5174\n", - " 5246\n", - " \n", - " \n", - " pt\n", - " 1491\n", - " 1473\n", - " 2445\n", - " 2489\n", - " 1469\n", - " 1493\n", - " 2460\n", - " 2429\n", - " 1448\n", - " 1426\n", - " 2425\n", - " 2360\n", + " 6000\n", + " 6065\n", " \n", " \n", " walk\n", " 2318\n", " 2312\n", - " 2332\n", - " 2323\n", + " 2322\n", + " 2324\n", " 2361\n", " 2390\n", - " 2368\n", - " 2388\n", + " 2367\n", + " 2405\n", " 2389\n", " 2373\n", - " 2401\n", - " 2394\n", + " 2395\n", + " 2380\n", + " \n", + " \n", + " pt\n", + " 1491\n", + " 1473\n", + " 1630\n", + " 1616\n", + " 1469\n", + " 1493\n", + " 1620\n", + " 1633\n", + " 1448\n", + " 1426\n", + " 1605\n", + " 1555\n", " \n", " \n", "\n", @@ -376,17 +385,17 @@ ], "text/plain": [ " eet_base_0 mc_base_0 eet_scen_0 mc_scen_0 eet_base_1 mc_base_1 \\\n", - "car 6191 6215 5223 5188 6170 6117 \n", - "pt 1491 1473 2445 2489 1469 1493 \n", - "walk 2318 2312 2332 2323 2361 2390 \n", + "car 6191 6215 6048 6060 6170 6117 \n", + "walk 2318 2312 2322 2324 2361 2390 \n", + "pt 1491 1473 1630 1616 1469 1493 \n", "\n", " eet_scen_1 mc_scen_1 eet_base_2 mc_base_2 eet_scen_2 mc_scen_2 \n", - "car 5172 5183 6163 6201 5174 5246 \n", - "pt 2460 2429 1448 1426 2425 2360 \n", - "walk 2368 2388 2389 2373 2401 2394 " + "car 6013 5962 6163 6201 6000 6065 \n", + "walk 2367 2405 2389 2373 2395 2380 \n", + "pt 1620 1633 1448 1426 1605 1555 " ] }, - "execution_count": 18, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -397,7 +406,32 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "metric = \"mode_choice\"\n", + "\n", + "dfs_ = []\n", + "\n", + "for seed in range(0, 10):\n", + " for seed_2 in range(0, 10):\n", + " df = ppl[f'{metric}_{seed}'].value_counts().to_frame(f'eet_base_{seed}').join(\n", + " ppl[f'{metric}_scen_{seed_2}'].value_counts().to_frame(f'eet_scen_{seed_2}'), how='outer').join(\n", + " ppl[f'{metric}_mc_{seed}'].value_counts().to_frame(f'mc_base_{seed}'), how='outer').join(\n", + " ppl[f'{metric}_mc_scen_{seed_2}'].value_counts().to_frame(f'mc_scen_{seed_2}'), how='outer'\n", + " ).fillna(0).astype(int)\n", + "\n", + " df[f'diff_eet_{seed}_{seed_2}'] = df[f'eet_scen_{seed_2}'] - df[f'eet_base_{seed}']\n", + " df[f'diff_mc_{seed}_{seed_2}'] = df[f'mc_scen_{seed_2}'] - df[f'mc_base_{seed}']\n", + " dfs_.append(df[[f'diff_eet_{seed}_{seed_2}', f'diff_mc_{seed}_{seed_2}']])\n", + "\n", + "diff_mc = pd.concat(dfs_, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -421,208 +455,140 @@ " \n", " \n", " \n", - " eet_base_0\n", - " mc_base_0\n", - " eet_scen_0\n", - " mc_scen_0\n", - " eet_base_1\n", - " mc_base_1\n", - " eet_scen_1\n", - " mc_scen_1\n", - " eet_base_2\n", - " mc_base_2\n", - " eet_scen_2\n", - " mc_scen_2\n", + " diff_eet_0_0\n", + " diff_mc_0_0\n", + " diff_eet_0_1\n", + " diff_mc_0_1\n", + " diff_eet_0_2\n", + " diff_mc_0_2\n", + " diff_eet_0_3\n", + " diff_mc_0_3\n", + " diff_eet_0_4\n", + " diff_mc_0_4\n", + " ...\n", + " diff_eet_9_5\n", + " diff_mc_9_5\n", + " diff_eet_9_6\n", + " diff_mc_9_6\n", + " diff_eet_9_7\n", + " diff_mc_9_7\n", + " diff_eet_9_8\n", + " diff_mc_9_8\n", + " diff_eet_9_9\n", + " diff_mc_9_9\n", " \n", " \n", " \n", " \n", - " 0\n", - " 1170\n", - " 1104\n", - " 1159\n", - " 1089\n", - " 1105\n", - " 1131\n", - " 1085\n", - " 1117\n", - " 1138\n", - " 1104\n", - " 1123\n", - " 1097\n", - " \n", - " \n", - " 1\n", - " 1061\n", - " 1038\n", - " 1065\n", - " 1044\n", - " 1089\n", - " 1079\n", - " 1098\n", - " 1075\n", - " 1056\n", - " 1037\n", - " 1062\n", - " 1035\n", - " \n", - " \n", - " 2\n", - " 791\n", - " 857\n", - " 801\n", - " 865\n", - " 864\n", - " 906\n", - " 869\n", - " 915\n", - " 882\n", - " 807\n", - " 891\n", - " 814\n", - " \n", - " \n", - " 3\n", - " 1034\n", - " 1104\n", - " 1034\n", - " 1097\n", - " 1078\n", - " 1029\n", - " 1070\n", - " 1031\n", - " 1046\n", - " 1068\n", - " 1041\n", - " 1062\n", - " \n", - " \n", - " 4\n", - " 951\n", - " 873\n", - " 955\n", - " 881\n", - " 919\n", - " 896\n", - " 930\n", - " 908\n", - " 923\n", - " 920\n", - " 934\n", - " 924\n", - " \n", - " \n", - " 5\n", - " 1032\n", - " 1037\n", - " 1023\n", - " 1032\n", - " 1021\n", - " 1053\n", - " 1020\n", - " 1042\n", - " 1007\n", - " 1046\n", - " 1004\n", - " 1046\n", - " \n", - " \n", - " 6\n", - " 1031\n", - " 1028\n", - " 1029\n", - " 1023\n", - " 971\n", - " 989\n", - " 972\n", - " 989\n", - " 1048\n", - " 1050\n", - " 1045\n", - " 1046\n", - " \n", - " \n", - " 7\n", - " 871\n", - " 856\n", - " 868\n", - " 862\n", - " 902\n", - " 897\n", - " 899\n", - " 895\n", - " 876\n", - " 932\n", - " 879\n", - " 934\n", + " car\n", + " -143\n", + " -155\n", + " -178\n", + " -253\n", + " -191\n", + " -150\n", + " -164\n", + " -264\n", + " -181\n", + " -179\n", + " ...\n", + " -230\n", + " -112\n", + " -30\n", + " -174\n", + " -160\n", + " -153\n", + " -156\n", + " -205\n", + " -166\n", + " -136\n", " \n", " \n", - " 8\n", - " 1027\n", - " 1026\n", - " 1031\n", - " 1033\n", - " 970\n", - " 1035\n", - " 975\n", - " 1041\n", - " 1015\n", - " 980\n", - " 1014\n", - " 985\n", + " walk\n", + " 4\n", + " 12\n", + " 49\n", + " 93\n", + " 77\n", + " 68\n", + " 21\n", + " 50\n", + " 55\n", + " 106\n", + " ...\n", + " 64\n", + " -85\n", + " -32\n", + " -44\n", + " -5\n", + " -29\n", + " -18\n", + " 18\n", + " 13\n", + " -16\n", " \n", " \n", - " 9\n", - " 1032\n", - " 1077\n", - " 1035\n", - " 1074\n", - " 1081\n", - " 985\n", - " 1082\n", - " 987\n", - " 1009\n", - " 1056\n", - " 1007\n", - " 1057\n", + " pt\n", + " 139\n", + " 143\n", + " 129\n", + " 160\n", + " 114\n", + " 82\n", + " 143\n", + " 214\n", + " 126\n", + " 73\n", + " ...\n", + " 166\n", + " 197\n", + " 62\n", + " 218\n", + " 165\n", + " 182\n", + " 174\n", + " 187\n", + " 153\n", + " 152\n", " \n", " \n", "\n", + "

3 rows × 200 columns

\n", "" ], "text/plain": [ - " eet_base_0 mc_base_0 eet_scen_0 mc_scen_0 eet_base_1 mc_base_1 \\\n", - "0 1170 1104 1159 1089 1105 1131 \n", - "1 1061 1038 1065 1044 1089 1079 \n", - "2 791 857 801 865 864 906 \n", - "3 1034 1104 1034 1097 1078 1029 \n", - "4 951 873 955 881 919 896 \n", - "5 1032 1037 1023 1032 1021 1053 \n", - "6 1031 1028 1029 1023 971 989 \n", - "7 871 856 868 862 902 897 \n", - "8 1027 1026 1031 1033 970 1035 \n", - "9 1032 1077 1035 1074 1081 985 \n", + " diff_eet_0_0 diff_mc_0_0 diff_eet_0_1 diff_mc_0_1 diff_eet_0_2 \\\n", + "car -143 -155 -178 -253 -191 \n", + "walk 4 12 49 93 77 \n", + "pt 139 143 129 160 114 \n", "\n", - " eet_scen_1 mc_scen_1 eet_base_2 mc_base_2 eet_scen_2 mc_scen_2 \n", - "0 1085 1117 1138 1104 1123 1097 \n", - "1 1098 1075 1056 1037 1062 1035 \n", - "2 869 915 882 807 891 814 \n", - "3 1070 1031 1046 1068 1041 1062 \n", - "4 930 908 923 920 934 924 \n", - "5 1020 1042 1007 1046 1004 1046 \n", - "6 972 989 1048 1050 1045 1046 \n", - "7 899 895 876 932 879 934 \n", - "8 975 1041 1015 980 1014 985 \n", - "9 1082 987 1009 1056 1007 1057 " + " diff_mc_0_2 diff_eet_0_3 diff_mc_0_3 diff_eet_0_4 diff_mc_0_4 ... \\\n", + "car -150 -164 -264 -181 -179 ... \n", + "walk 68 21 50 55 106 ... \n", + "pt 82 143 214 126 73 ... \n", + "\n", + " diff_eet_9_5 diff_mc_9_5 diff_eet_9_6 diff_mc_9_6 diff_eet_9_7 \\\n", + "car -230 -112 -30 -174 -160 \n", + "walk 64 -85 -32 -44 -5 \n", + "pt 166 197 62 218 165 \n", + "\n", + " diff_mc_9_7 diff_eet_9_8 diff_mc_9_8 diff_eet_9_9 diff_mc_9_9 \n", + "car -153 -156 -205 -166 -136 \n", + "walk -29 -18 18 13 -16 \n", + "pt 182 174 187 153 152 \n", + "\n", + "[3 rows x 200 columns]" ] }, - "execution_count": 16, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "pd.concat([display_metric('dest', ppl, i) for i in range(0, 3)], axis=1)" + "# check eet \n", + "\n", + "diff_mc" ] }, { @@ -653,6 +619,18 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# increase in pt trips to/from zone 0\n", + "\n", + "\n", + "# increase/decrease in other areas -> and why. NEST SWITCHING? should not switch away from 0, right?" + ] + }, { "cell_type": "code", "execution_count": null, From 73b8c32471485f2c2d7d88fd03046cef309ad993 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 2 Apr 2025 20:14:44 +1000 Subject: [PATCH 101/135] sign on logsum --- .../2025/correlation_variance_tests.ipynb | 289 +++++++++--------- 1 file changed, 147 insertions(+), 142 deletions(-) diff --git a/notebooks/2025/correlation_variance_tests.ipynb b/notebooks/2025/correlation_variance_tests.ipynb index 7b7991c692..33d4a844f3 100644 --- a/notebooks/2025/correlation_variance_tests.ipynb +++ b/notebooks/2025/correlation_variance_tests.ipynb @@ -37,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -136,11 +136,11 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ - "beta_logsum = -0.1\n", + "beta_logsum = 0.5 # applied to expected maximum utility, so positive sign\n", "\n", "dest_altneratives = destination.zone.values\n", "num_dests = dest_altneratives.shape[0]\n", @@ -163,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -232,7 +232,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 100, "metadata": {}, "outputs": [], "source": [ @@ -242,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -267,7 +267,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ @@ -286,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -295,7 +295,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 104, "metadata": {}, "outputs": [ { @@ -335,49 +335,49 @@ " \n", " \n", " \n", - " car\n", - " 6191\n", - " 6215\n", - " 6048\n", - " 6060\n", - " 6170\n", - " 6117\n", - " 6013\n", - " 5962\n", - " 6163\n", - " 6201\n", - " 6000\n", - " 6065\n", + " walk\n", + " 8817\n", + " 8843\n", + " 8806\n", + " 8831\n", + " 8750\n", + " 8794\n", + " 8733\n", + " 8785\n", + " 8824\n", + " 8846\n", + " 8815\n", + " 8839\n", " \n", " \n", - " walk\n", - " 2318\n", - " 2312\n", - " 2322\n", - " 2324\n", - " 2361\n", - " 2390\n", - " 2367\n", - " 2405\n", - " 2389\n", - " 2373\n", - " 2395\n", - " 2380\n", + " car\n", + " 1027\n", + " 1016\n", + " 1027\n", + " 1017\n", + " 1072\n", + " 1033\n", + " 1072\n", + " 1021\n", + " 1000\n", + " 995\n", + " 988\n", + " 984\n", " \n", " \n", " pt\n", - " 1491\n", - " 1473\n", - " 1630\n", - " 1616\n", - " 1469\n", - " 1493\n", - " 1620\n", - " 1633\n", - " 1448\n", - " 1426\n", - " 1605\n", - " 1555\n", + " 156\n", + " 141\n", + " 167\n", + " 152\n", + " 178\n", + " 173\n", + " 195\n", + " 194\n", + " 176\n", + " 159\n", + " 197\n", + " 177\n", " \n", " \n", "\n", @@ -385,17 +385,17 @@ ], "text/plain": [ " eet_base_0 mc_base_0 eet_scen_0 mc_scen_0 eet_base_1 mc_base_1 \\\n", - "car 6191 6215 6048 6060 6170 6117 \n", - "walk 2318 2312 2322 2324 2361 2390 \n", - "pt 1491 1473 1630 1616 1469 1493 \n", + "walk 8817 8843 8806 8831 8750 8794 \n", + "car 1027 1016 1027 1017 1072 1033 \n", + "pt 156 141 167 152 178 173 \n", "\n", " eet_scen_1 mc_scen_1 eet_base_2 mc_base_2 eet_scen_2 mc_scen_2 \n", - "car 6013 5962 6163 6201 6000 6065 \n", - "walk 2367 2405 2389 2373 2395 2380 \n", - "pt 1620 1633 1448 1426 1605 1555 " + "walk 8733 8785 8824 8846 8815 8839 \n", + "car 1072 1021 1000 995 988 984 \n", + "pt 195 194 176 159 197 177 " ] }, - "execution_count": 42, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } @@ -406,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 105, "metadata": {}, "outputs": [], "source": [ @@ -431,7 +431,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 106, "metadata": {}, "outputs": [ { @@ -480,76 +480,76 @@ " \n", " \n", " \n", - " car\n", - " -143\n", - " -155\n", - " -178\n", - " -253\n", - " -191\n", - " -150\n", - " -164\n", - " -264\n", - " -181\n", - " -179\n", + " walk\n", + " -11\n", + " -12\n", + " -84\n", + " -58\n", + " -2\n", + " -4\n", + " -57\n", + " -66\n", + " 15\n", + " -66\n", " ...\n", - " -230\n", - " -112\n", - " -30\n", - " -174\n", - " -160\n", - " -153\n", - " -156\n", - " -205\n", - " -166\n", - " -136\n", + " -7\n", + " -60\n", + " 7\n", + " -21\n", + " 10\n", + " -22\n", + " 81\n", + " -45\n", + " -6\n", + " -14\n", " \n", " \n", - " walk\n", - " 4\n", - " 12\n", - " 49\n", - " 93\n", - " 77\n", - " 68\n", - " 21\n", - " 50\n", - " 55\n", - " 106\n", - " ...\n", - " 64\n", - " -85\n", + " car\n", + " 0\n", + " 1\n", + " 45\n", + " 5\n", + " -39\n", " -32\n", + " 13\n", + " 9\n", " -44\n", - " -5\n", - " -29\n", + " 28\n", + " ...\n", " -18\n", - " 18\n", - " 13\n", - " -16\n", + " 32\n", + " -8\n", + " -10\n", + " -47\n", + " -5\n", + " -107\n", + " 12\n", + " -14\n", + " 1\n", " \n", " \n", " pt\n", - " 139\n", - " 143\n", - " 129\n", - " 160\n", - " 114\n", - " 82\n", - " 143\n", - " 214\n", - " 126\n", - " 73\n", + " 11\n", + " 11\n", + " 39\n", + " 53\n", + " 41\n", + " 36\n", + " 44\n", + " 57\n", + " 29\n", + " 38\n", " ...\n", - " 166\n", - " 197\n", - " 62\n", - " 218\n", - " 165\n", - " 182\n", - " 174\n", - " 187\n", - " 153\n", - " 152\n", + " 25\n", + " 28\n", + " 1\n", + " 31\n", + " 37\n", + " 27\n", + " 26\n", + " 33\n", + " 20\n", + " 13\n", " \n", " \n", "\n", @@ -558,52 +558,57 @@ ], "text/plain": [ " diff_eet_0_0 diff_mc_0_0 diff_eet_0_1 diff_mc_0_1 diff_eet_0_2 \\\n", - "car -143 -155 -178 -253 -191 \n", - "walk 4 12 49 93 77 \n", - "pt 139 143 129 160 114 \n", + "walk -11 -12 -84 -58 -2 \n", + "car 0 1 45 5 -39 \n", + "pt 11 11 39 53 41 \n", "\n", " diff_mc_0_2 diff_eet_0_3 diff_mc_0_3 diff_eet_0_4 diff_mc_0_4 ... \\\n", - "car -150 -164 -264 -181 -179 ... \n", - "walk 68 21 50 55 106 ... \n", - "pt 82 143 214 126 73 ... \n", + "walk -4 -57 -66 15 -66 ... \n", + "car -32 13 9 -44 28 ... \n", + "pt 36 44 57 29 38 ... \n", "\n", " diff_eet_9_5 diff_mc_9_5 diff_eet_9_6 diff_mc_9_6 diff_eet_9_7 \\\n", - "car -230 -112 -30 -174 -160 \n", - "walk 64 -85 -32 -44 -5 \n", - "pt 166 197 62 218 165 \n", + "walk -7 -60 7 -21 10 \n", + "car -18 32 -8 -10 -47 \n", + "pt 25 28 1 31 37 \n", "\n", " diff_mc_9_7 diff_eet_9_8 diff_mc_9_8 diff_eet_9_9 diff_mc_9_9 \n", - "car -153 -156 -205 -166 -136 \n", - "walk -29 -18 18 13 -16 \n", - "pt 182 174 187 153 152 \n", + "walk -22 81 -45 -6 -14 \n", + "car -5 -107 12 -14 1 \n", + "pt 27 26 33 20 13 \n", "\n", "[3 rows x 200 columns]" ] }, - "execution_count": 61, + "execution_count": 106, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# check eet \n", - "\n", + "# check eet\n", "diff_mc" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 111, "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "(6, 0)" + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ppl.loc[(ppl.mode_choice_mc_0 != \"walk\") & (ppl.mode_choice_mc_scen_0 == \"walk\")].shape[0], ppl.loc[(ppl.mode_choice_0 != \"walk\") & (ppl.mode_choice_scen_0 == \"walk\")].shape[0]" + ] }, { "cell_type": "code", From f5f365fdd8d24be137c255632794c9180bb0797d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Wed, 2 Apr 2025 20:23:57 +1000 Subject: [PATCH 102/135] removes notebook folder --- .../2025/correlation_variance_tests.ipynb | 689 ------------------ 1 file changed, 689 deletions(-) delete mode 100644 notebooks/2025/correlation_variance_tests.ipynb diff --git a/notebooks/2025/correlation_variance_tests.ipynb b/notebooks/2025/correlation_variance_tests.ipynb deleted file mode 100644 index 33d4a844f3..0000000000 --- a/notebooks/2025/correlation_variance_tests.ipynb +++ /dev/null @@ -1,689 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Which correlations do we want\n", - "\n", - "simulate destination mode choice model:\n", - " 1) MC\n", - " 2) EET fixed error per level\n", - " 3) EET only fixed when identical\n", - "\n", - "Does it make a difference in analysis?" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "execution": { - "iopub.execute_input": "2022-05-01T05:31:19.564033Z", - "iopub.status.busy": "2022-05-01T05:31:19.563663Z", - "iopub.status.idle": "2022-05-01T05:31:19.824207Z", - "shell.execute_reply": "2022-05-01T05:31:19.823432Z", - "shell.execute_reply.started": "2022-05-01T05:31:19.563960Z" - } - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "\n", - "from numpy.random import default_rng\n", - "from copy import deepcopy" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [], - "source": [ - "rng = default_rng(999)\n", - "\n", - "### Zones\n", - "num_zones = 10\n", - "destination = pd.DataFrame(\n", - " data=zip(np.arange(0, num_zones), rng.normal(1000, 10, size=num_zones), strict=True),\n", - " columns=['zone', 'attrs'],\n", - ")\n", - "\n", - "### People - assume each person makes one trip from random origin\n", - "num_ppl = 10000\n", - "vars=['id', 'income', 'orig']\n", - "ppl = pd.DataFrame(\n", - " data=zip(\n", - " np.arange(0, num_ppl),\n", - " rng.normal(50, 20, size=(num_ppl)).clip(2, 150),\n", - " rng.choice(np.arange(0, num_zones), num_ppl),\n", - " strict=True,\n", - " ),\n", - " columns=vars,\n", - ")\n", - "\n", - "\n", - "### Modes\n", - "num_modes = 3\n", - "mode_idx_to_str = {0: 'car', 1: 'pt', 2: 'walk'}\n", - "modes = np.array(['car', 'pt', 'walk'])\n", - "mode_asc = {'car': 2.5, 'pt': 0.0, 'walk': 10.0}\n", - "\n", - "### Cost and time by mode\n", - "skims = {}\n", - "skims['time'] = {}\n", - "skims['time']['car'] = rng.normal(20, 20, size=(num_zones, num_zones)).clip(4, 60)\n", - "skims['time']['pt'] = rng.normal(40, 10, size=(num_zones, num_zones)).clip(4, 80)\n", - "skims['time']['walk'] = 3.0 * skims['time']['car'].copy()\n", - "\n", - "skims['cost'] = {}\n", - "skims['cost']['car'] = skims['time']['car'].copy() / 4.0\n", - "skims['cost']['pt'] = 2.0 * np.ones_like(skims['time']['pt'])\n", - "skims['cost']['walk'] = np.zeros_like(skims['time']['walk'])\n", - "\n", - "\n", - "# scenario: decrease pt travel time to (and within zone) 0 by 70%\n", - "scenario_skims = deepcopy(skims)\n", - "#scenario_skims['time']['pt'][0,:] *= 0.3\n", - "scenario_skims['time']['pt'][:,0] *= 0.3" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [], - "source": [ - "def logsum(utilities, nest_scale=1.0):\n", - " scaled_utils = utilities / nest_scale\n", - " max_util = np.max(scaled_utils, axis=1)\n", - " return max_util + np.log(np.sum(np.exp(scaled_utils - max_util.reshape((-1,1))), axis=1))\n", - "\n", - "def inverse_ev1_cdf(x, location=0.0, scale=1.0):\n", - " return location - scale * np.log(-np.log(x))\n", - "\n", - "### Utility function parameters\n", - "beta_time = {}\n", - "beta_time['car'] = -0.06\n", - "beta_time['pt'] = -0.03\n", - "beta_time['walk'] = -0.2\n", - "beta_cost = -0.01\n", - "\n", - "def mode_utility(mode, income, o, d, skims):\n", - " return (\n", - " beta_time[mode] * skims['time'][mode][o,d]\n", - " + beta_cost * skims['cost'][mode][o,d] / income\n", - " + mode_asc[mode]\n", - " )\n", - "\n", - "def mode_logsum(o: int, d: np.array, income: float, skims):\n", - " mode_utils = np.array([mode_utility(m_, income, o, d, skims) for m_ in modes]).T\n", - "\n", - " return logsum(mode_utils, 1.0)\n", - "\n", - "# mode_logsum(0, np.array([0, 1, 2]), 100)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "#inverse_ev1_cdf(default_rng(seed=np.random.SeedSequence([777, 333])).random(100))" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [], - "source": [ - "beta_logsum = 0.5 # applied to expected maximum utility, so positive sign\n", - "\n", - "dest_altneratives = destination.zone.values\n", - "num_dests = dest_altneratives.shape[0]\n", - "dest_attrs = np.log(destination['attrs'].values)\n", - "ppl['dest_utils'] = ppl.apply(\n", - " lambda x: (\n", - " beta_logsum * mode_logsum(int(x['orig']), dest_altneratives, x['income'], skims)\n", - " + dest_attrs\n", - " ),\n", - " axis=1,\n", - ")\n", - "ppl['dest_utils_scen'] = ppl.apply(\n", - " lambda x: (\n", - " beta_logsum * mode_logsum(int(x['orig']), dest_altneratives, x['income'], scenario_skims)\n", - " + dest_attrs\n", - " ),\n", - " axis=1,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [], - "source": [ - "### EET\n", - "def destination_choice_model(ppl, seed=777, util_column_name=\"dest_utils\"):\n", - " dest_choice = ppl.apply(\n", - " lambda x: np.argmax(\n", - " x[util_column_name] + inverse_ev1_cdf(\n", - " default_rng(seed=np.random.SeedSequence([seed, int(x['id']), 100])).random(size=num_dests)\n", - " )\n", - " )\n", - " , axis=1\n", - " )\n", - " return dest_choice\n", - "\n", - "def mode_choice_model(ppl, skims, dest_col_name='dest', seed=777, od_seed_contrib=False):\n", - " \"\"\"\"od_seed_contrib: if True, seed depedns on destination\"\"\"\n", - " def seed_o_d(d):\n", - " if od_seed_contrib:\n", - " return d\n", - " return 0\n", - "\n", - " mode_choice = ppl.apply(\n", - " lambda x: np.argmax(\n", - " [mode_utility(m_, x['income'], int(x['orig']), int(x[dest_col_name]), skims) for m_ in modes]\n", - " + inverse_ev1_cdf(default_rng(seed=np.random.SeedSequence(\n", - " [seed, int(x['id']), 200, seed_o_d(int(x[dest_col_name]))]\n", - " )).random(size=num_modes))\n", - " ),\n", - " axis=1,\n", - " )\n", - " #mode_choice = mode_utils.apply(np.argmax)\n", - " return mode_choice.map(mode_idx_to_str)\n", - "\n", - "### MC\n", - "def destination_choice_model_mc(ppl, seed=777, util_column_name=\"dest_utils\"):\n", - " dest_choice = ppl.apply(\n", - " lambda x: default_rng(\n", - " seed=np.random.SeedSequence([seed, int(x['id']), 100])\n", - " ).choice(dest_altneratives, p=np.exp(x[util_column_name]) / np.sum(np.exp(x[util_column_name])), size=1)[0],\n", - " axis=1,\n", - " )\n", - " return dest_choice\n", - "\n", - "def mode_choice_model_mc(ppl, skims, dest_col_name='dest', seed=777, od_seed_contrib=False):\n", - " def seed_o_d(d):\n", - " if od_seed_contrib:\n", - " return d\n", - " return 0\n", - "\n", - " ppl['mode_exp_utils'] = ppl.apply(\n", - " lambda x: np.exp(\n", - " [mode_utility(m_, x['income'], int(x['orig']), int(x[dest_col_name]), skims) for m_ in modes]\n", - " ),\n", - " axis=1,\n", - " )\n", - " mode_choice = ppl.apply(\n", - " lambda x: default_rng(\n", - " seed=np.random.SeedSequence([seed, int(x['id']), 200, seed_o_d(int(x[dest_col_name]))])\n", - " ).choice(modes, p=x.mode_exp_utils / np.sum(x.mode_exp_utils), size=1)[0],\n", - " axis=1,\n", - " )\n", - " ppl.drop(columns=['mode_exp_utils'], inplace=True)\n", - " return mode_choice" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [], - "source": [ - "#for i in range(0, 10):\n", - "# ppl.dest_utils.apply(lambda x: (np.exp(x) / np.sum(np.exp(x)))[i]).hist(figsize=(3,2))" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "# 10k ppl: per seed about 3.5s total [w/o sedd, with 12.4]\n", - "for s_ in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:\n", - " ppl[f'dest_{s_}'] = destination_choice_model(ppl, util_column_name='dest_utils', seed=s_)\n", - " ppl[f'dest_scen_{s_}'] = destination_choice_model(ppl, util_column_name='dest_utils_scen', seed=s_)\n", - " ppl[f'dest_mc_{s_}'] = destination_choice_model_mc(ppl, util_column_name='dest_utils', seed=s_)\n", - " ppl[f'dest_mc_scen_{s_}'] = destination_choice_model_mc(ppl, util_column_name='dest_utils_scen', seed=s_)\n", - " \n", - " \n", - " ppl[f'mode_choice_{s_}'] = mode_choice_model(ppl, skims, dest_col_name=f'dest_{s_}', seed=s_)\n", - " ppl[f'mode_choice_scen_{s_}'] = mode_choice_model(ppl, scenario_skims, dest_col_name=f'dest_scen_{s_}', seed=s_)\n", - " #ppl[f'mode_choice_seedd_{s_}'] = mode_choice_model(ppl, skims, dest_col_name=f'dest_{s_}', seed=s_, od_seed_contrib=True)\n", - " #ppl[f'mode_choice_seedd_scen_{s_}'] = mode_choice_model(ppl, scenario_skims, dest_col_name=f'dest_scen_{s_}', seed=s_, od_seed_contrib=True)\n", - "\n", - " ppl[f'mode_choice_mc_{s_}'] = mode_choice_model_mc(ppl, skims, dest_col_name=f'dest_mc_{s_}', seed=s_)\n", - " ppl[f'mode_choice_mc_scen_{s_}'] = mode_choice_model_mc(ppl, scenario_skims, dest_col_name=f'dest_mc_scen_{s_}', seed=s_)\n", - " #ppl[f'mode_choice_mc_seedd_{s_}'] = mode_choice_model_mc(ppl, skims, dest_col_name=f'dest_mc_{s_}', seed=s_, od_seed_contrib=True)\n", - " #ppl[f'mode_choice_mc_seedd_scen_{s_}'] = mode_choice_model_mc(ppl, scenario_skims, dest_col_name=f'dest_mc_scen_{s_}', seed=s_, od_seed_contrib=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "def display_metric(metric, ppl, seed):\n", - " return ( \n", - " ppl[f'{metric}_{seed}'].value_counts().to_frame(f'eet_base_{seed}').join(\n", - " ppl[f'{metric}_mc_{seed}'].value_counts().to_frame(f'mc_base_{seed}'), how='outer').join(\n", - " ppl[f'{metric}_scen_{seed}'].value_counts().to_frame(f'eet_scen_{seed}'), how='outer').join(\n", - " ppl[f'{metric}_mc_scen_{seed}'].value_counts().to_frame(f'mc_scen_{seed}'), how='outer').fillna(0).astype(int)\n", - " )\n", - " # ppl[f'mode_choice_seedd_{seed}'].value_counts().to_frame(f'eet_base_seedd_{seed}'), how='outer').join(\n", - " # ppl[f'mode_choice_mc_seedd_{seed}'].value_counts().to_frame(f'mc_base_seedd_{seed}'), how='outer').join(\n", - " # ppl[f'mode_choice_seedd_scen_{seed}'].value_counts().to_frame(f'eet_scen_seedd_{seed}'), how='outer').join(\n", - " # ppl[f'mode_choice_mc_seedd_scen_{seed}'].value_counts().to_frame(f'mc_scen_seedd_{seed}'), how='outer')\\" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "#pd.concat([display_metric('dest', ppl, i) for i in range(0, 3)], axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
eet_base_0mc_base_0eet_scen_0mc_scen_0eet_base_1mc_base_1eet_scen_1mc_scen_1eet_base_2mc_base_2eet_scen_2mc_scen_2
walk881788438806883187508794873387858824884688158839
car102710161027101710721033107210211000995988984
pt156141167152178173195194176159197177
\n", - "
" - ], - "text/plain": [ - " eet_base_0 mc_base_0 eet_scen_0 mc_scen_0 eet_base_1 mc_base_1 \\\n", - "walk 8817 8843 8806 8831 8750 8794 \n", - "car 1027 1016 1027 1017 1072 1033 \n", - "pt 156 141 167 152 178 173 \n", - "\n", - " eet_scen_1 mc_scen_1 eet_base_2 mc_base_2 eet_scen_2 mc_scen_2 \n", - "walk 8733 8785 8824 8846 8815 8839 \n", - "car 1072 1021 1000 995 988 984 \n", - "pt 195 194 176 159 197 177 " - ] - }, - "execution_count": 104, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pd.concat([display_metric('mode_choice', ppl, i) for i in range(0, 3)], axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "metric = \"mode_choice\"\n", - "\n", - "dfs_ = []\n", - "\n", - "for seed in range(0, 10):\n", - " for seed_2 in range(0, 10):\n", - " df = ppl[f'{metric}_{seed}'].value_counts().to_frame(f'eet_base_{seed}').join(\n", - " ppl[f'{metric}_scen_{seed_2}'].value_counts().to_frame(f'eet_scen_{seed_2}'), how='outer').join(\n", - " ppl[f'{metric}_mc_{seed}'].value_counts().to_frame(f'mc_base_{seed}'), how='outer').join(\n", - " ppl[f'{metric}_mc_scen_{seed_2}'].value_counts().to_frame(f'mc_scen_{seed_2}'), how='outer'\n", - " ).fillna(0).astype(int)\n", - "\n", - " df[f'diff_eet_{seed}_{seed_2}'] = df[f'eet_scen_{seed_2}'] - df[f'eet_base_{seed}']\n", - " df[f'diff_mc_{seed}_{seed_2}'] = df[f'mc_scen_{seed_2}'] - df[f'mc_base_{seed}']\n", - " dfs_.append(df[[f'diff_eet_{seed}_{seed_2}', f'diff_mc_{seed}_{seed_2}']])\n", - "\n", - "diff_mc = pd.concat(dfs_, axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
diff_eet_0_0diff_mc_0_0diff_eet_0_1diff_mc_0_1diff_eet_0_2diff_mc_0_2diff_eet_0_3diff_mc_0_3diff_eet_0_4diff_mc_0_4...diff_eet_9_5diff_mc_9_5diff_eet_9_6diff_mc_9_6diff_eet_9_7diff_mc_9_7diff_eet_9_8diff_mc_9_8diff_eet_9_9diff_mc_9_9
walk-11-12-84-58-2-4-57-6615-66...-7-607-2110-2281-45-6-14
car01455-39-32139-4428...-1832-8-10-47-5-10712-141
pt11113953413644572938...2528131372726332013
\n", - "

3 rows × 200 columns

\n", - "
" - ], - "text/plain": [ - " diff_eet_0_0 diff_mc_0_0 diff_eet_0_1 diff_mc_0_1 diff_eet_0_2 \\\n", - "walk -11 -12 -84 -58 -2 \n", - "car 0 1 45 5 -39 \n", - "pt 11 11 39 53 41 \n", - "\n", - " diff_mc_0_2 diff_eet_0_3 diff_mc_0_3 diff_eet_0_4 diff_mc_0_4 ... \\\n", - "walk -4 -57 -66 15 -66 ... \n", - "car -32 13 9 -44 28 ... \n", - "pt 36 44 57 29 38 ... \n", - "\n", - " diff_eet_9_5 diff_mc_9_5 diff_eet_9_6 diff_mc_9_6 diff_eet_9_7 \\\n", - "walk -7 -60 7 -21 10 \n", - "car -18 32 -8 -10 -47 \n", - "pt 25 28 1 31 37 \n", - "\n", - " diff_mc_9_7 diff_eet_9_8 diff_mc_9_8 diff_eet_9_9 diff_mc_9_9 \n", - "walk -22 81 -45 -6 -14 \n", - "car -5 -107 12 -14 1 \n", - "pt 27 26 33 20 13 \n", - "\n", - "[3 rows x 200 columns]" - ] - }, - "execution_count": 106, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# check eet\n", - "diff_mc" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(6, 0)" - ] - }, - "execution_count": 111, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ppl.loc[(ppl.mode_choice_mc_0 != \"walk\") & (ppl.mode_choice_mc_scen_0 == \"walk\")].shape[0], ppl.loc[(ppl.mode_choice_0 != \"walk\") & (ppl.mode_choice_scen_0 == \"walk\")].shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# increase in pt trips to/from zone 0\n", - "\n", - "\n", - "# increase/decrease in other areas -> and why. NEST SWITCHING? should not switch away from 0, right?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} From 0f54cec22ee73887efd07e2a7c6f17ba5eb64732 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 6 Apr 2025 20:07:50 +1000 Subject: [PATCH 103/135] compute_setting overrides for eet, currently only for interaction_sample to test loc sampling with MC --- activitysim/core/configuration/base.py | 5 +++++ activitysim/core/interaction_sample.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/activitysim/core/configuration/base.py b/activitysim/core/configuration/base.py index ec27fee994..374ef8e71b 100644 --- a/activitysim/core/configuration/base.py +++ b/activitysim/core/configuration/base.py @@ -135,6 +135,10 @@ class ComputeSettings(PydanticBase): Sharrow settings for a component. """ + # Make this more general compute settings and use for explicit error term overrides + # TODO: Default None work for sub-components + use_explicit_error_terms: None | bool | dict[str, bool] = None + sharrow_skip: bool | dict[str, bool] = False """Skip sharrow when evaluating this component. @@ -244,6 +248,7 @@ def subcomponent_settings(self, subcomponent: str) -> ComputeSettings: use_numba=self.use_numba, drop_unused_columns=self.drop_unused_columns, protect_columns=self.protect_columns, + use_explicit_error_terms=self.use_explicit_error_terms, ) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 8c3645da9f..5f8f34b98f 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -524,8 +524,14 @@ def _interaction_sample( state.tracing.dump_df(DUMP, utilities, trace_label, "utilities") + if compute_settings and (compute_settings.use_explicit_error_terms is not None): + use_eet = compute_settings.use_explicit_error_terms + logger.info(f"Interaction sample model-specific EET overrides for {trace_label}: eet = {use_eet}") + else: + use_eet = state.settings.use_explicit_error_terms + # sample_size == 0 is for estimation mode, see below - if state.settings.use_explicit_error_terms and (sample_size != 0): + if (sample_size != 0) and use_eet: choices_df = make_sample_choices_utility_based( state, choosers, From 11d86a1f53488587c4d7ce9a4fb37ac871fa4b3c Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 7 Apr 2025 02:34:05 +1000 Subject: [PATCH 104/135] Adds subcomponent-specific eet setting --- activitysim/core/configuration/base.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/activitysim/core/configuration/base.py b/activitysim/core/configuration/base.py index 374ef8e71b..b11643cdc3 100644 --- a/activitysim/core/configuration/base.py +++ b/activitysim/core/configuration/base.py @@ -136,7 +136,7 @@ class ComputeSettings(PydanticBase): """ # Make this more general compute settings and use for explicit error term overrides - # TODO: Default None work for sub-components + # Default None work for sub-components defined in getter below (eet_subcomponent) use_explicit_error_terms: None | bool | dict[str, bool] = None sharrow_skip: bool | dict[str, bool] = False @@ -222,6 +222,13 @@ def should_skip(self, subcomponent: str) -> bool: else: return bool(self.sharrow_skip) + def eet_subcomponent(self, subcomponent: str) -> bool: + """Check for EET overrides for a particular subcomponent.""" + if isinstance(self.use_explicit_error_terms, dict): + return self.use_explicit_error_terms.get(subcomponent, None) + else: + return bool(self.use_explicit_error_terms) + @contextmanager def pandas_option_context(self): """Context manager to set pandas options for compute settings.""" @@ -248,7 +255,7 @@ def subcomponent_settings(self, subcomponent: str) -> ComputeSettings: use_numba=self.use_numba, drop_unused_columns=self.drop_unused_columns, protect_columns=self.protect_columns, - use_explicit_error_terms=self.use_explicit_error_terms, + use_explicit_error_terms=self.eet_subcomponent(subcomponent), ) From 30ddb3e1970d6648d6b9f05e44a11ace2f02a14b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 7 Apr 2025 02:44:54 +1000 Subject: [PATCH 105/135] default compute settings set before, no need to check if none --- activitysim/core/interaction_sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 5f8f34b98f..72be64787a 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -524,7 +524,7 @@ def _interaction_sample( state.tracing.dump_df(DUMP, utilities, trace_label, "utilities") - if compute_settings and (compute_settings.use_explicit_error_terms is not None): + if compute_settings.use_explicit_error_terms is not None: use_eet = compute_settings.use_explicit_error_terms logger.info(f"Interaction sample model-specific EET overrides for {trace_label}: eet = {use_eet}") else: From c6d60c6cbf577fc80f0ac6cc73a62d3fb9ff1073 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 12 Apr 2025 05:23:30 +1000 Subject: [PATCH 106/135] comment clean up --- activitysim/abm/models/joint_tour_participation.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 65471d32a9..cda3535e12 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -424,10 +424,6 @@ def joint_tour_participation( if i not in model_settings.compute_settings.protect_columns: model_settings.compute_settings.protect_columns.append(i) - # TODO: loads of code duplication, could make this implicit and get rid of it - # if state.settings.use_explicit_error_terms: - # custom_chooser = participants_chooser_utility_based - # else: custom_chooser = participants_chooser choices = simulate.simple_simulate_by_chunk_id( From 6a0987d96590f8502353f43c710527b419d716ae Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 12 Apr 2025 05:31:56 +1000 Subject: [PATCH 107/135] identifies eet todos --- activitysim/abm/models/joint_tour_participation.py | 2 +- activitysim/abm/models/util/cdap.py | 2 +- activitysim/core/logit.py | 14 +++++++------- activitysim/core/simulate.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index cda3535e12..9c3dc9e770 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -207,7 +207,7 @@ def participants_chooser( ) print(unsatisfied_candidates.head(20)) - # TODO: explicit error term impl of force participation + # TODO-EET: explicit error term impl of force participation if model_settings.FORCE_PARTICIPATION and not state.settings.use_explicit_error_terms: logger.warning( f"Forcing joint tour participation for {num_tours_remaining} tours." diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index 5d4d90fc84..7602cfdd40 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -1033,7 +1033,7 @@ def household_activity_choices( "%s.hhsize%d_activity_choices" % (trace_label, hhsize), column_labels=["expression", "household"], ) - # TODO: check if different num rands matters here + # TODO-EET: check if different num rands matters here state.tracing.trace_df( rands, "%s.hhsize%d_rands" % (trace_label, hhsize), columns=[None, "rand"] ) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 8d75487e46..ba1bb79010 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -270,7 +270,7 @@ def utils_to_probs( return probs -# todo: check state, add type annotations, check new-world tracing, etc. +# TODO-EET: check state, add type annotations, check new-world tracing, etc. def add_ev1_random(state: workflow.State, df: pd.DataFrame): nest_utils_for_choice = df.copy() nest_utils_for_choice += state.get_rn_generator().gumbel_for_df(nest_utils_for_choice, n=df.shape[1]) @@ -289,7 +289,7 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter raise ValueError("This should never happen - no alternative found") -# TODO [janzill Jun2022]: make everything in nested and here numpy from beginning to make choices consistent with +# TODO-EET [janzill Jun2022]: make everything in nested and here numpy from beginning to make choices consistent with # previous impl (want column index and not alternative name) # Note: this is relatively slow due to the apply. # It could *maybe* be sped up by using the fact that the nesting structure is the same for all rows: Add ev1(0,1) to @@ -322,7 +322,7 @@ def make_choices_explicit_error_term_nl(state, nested_utilities, alt_order_array def make_choices_explicit_error_term_mnl(state, utilities): utilities_incl_unobs = add_ev1_random(state, utilities) choices = np.argmax(utilities_incl_unobs.to_numpy(), axis=1) - # TODO: reporting like for zero probs + # TODO-EET: reporting like for zero probs assert not np.isnan(choices).any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=utilities_incl_unobs.index) return choices @@ -337,7 +337,7 @@ def make_choices_explicit_error_term(state, utilities, alt_order_array, nest_spe return choices -# TODO: memory usage +# TODO-EET: memory usage def make_choices_utility_based( state: workflow.State, utilities: pd.DataFrame, @@ -351,10 +351,10 @@ def make_choices_utility_based( ) -> tuple[pd.Series, pd.Series]: trace_label = tracing.extend_trace_label(trace_label, 'make_choices_utility_based') - # TODO: index of choices for nested utilities is different than unnested - this needs to be consistent for + # TODO-EET: index of choices for nested utilities is different than unnested - this needs to be consistent for # turning indexes into alternative names to keep code changes to minimum for now choices = make_choices_explicit_error_term(state, utilities, name_mapping, nest_spec, trace_label) - # TODO: rands - log all zeros for now + # TODO-EET: rands - log all zeros for now rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) return choices, rands @@ -663,7 +663,7 @@ def each_nest(nest_spec: dict | LogitNestSpec, type=None, post_order=False): yield nest -# TODO: do I need to implement this for LogitNestSpec? +# TODO-EET: do I need to implement this for LogitNestSpec? def count_nests(nest_spec): """ count the nests in nest_spec, return 0 if nest_spec is none diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index a395236624..3e9b600c0d 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1436,12 +1436,12 @@ def eval_nl( ) if state.settings.use_explicit_error_terms: - # TODO [janzill Jun2022]: combine with nested_exp_utilities? + # TODO-EET [janzill Jun2022]: combine with nested_exp_utilities? # utilities of leaves and nests nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) chunk_sizer.log_df(trace_label, "nested_utilities", nested_utilities) - # TODO [janzill Jun2022]: this can be done from utils directly, but use existing methodology for prototype + # TODO-EET [janzill Jun2022]: this can be done from utils directly, but use existing methodology for prototype if want_logsums: # logsum of nest root # exponentiated utilities of leaves and nests @@ -1450,7 +1450,7 @@ def eval_nl( logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) chunk_sizer.log_df(trace_label, "logsums", logsums) - # TODO: index of choices for nested utilities is different than unnested - this needs to be consistent for + # TODO-EET: index of choices for nested utilities is different than unnested - this needs to be consistent for # turning indexes into alternative names to keep code changes to minimum for now name_mapping = raw_utilities.columns.values From 5637fb0d84ec9c13887ed40534750e300914f096 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 12 Apr 2025 05:41:00 +1000 Subject: [PATCH 108/135] test SOA MC for disagg access --- activitysim/abm/models/disaggregate_accessibility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index fd79a188dc..2a9d983e8b 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -75,7 +75,8 @@ class DisaggregateAccessibilityAnnotateSettings(PydanticReadable, extra="forbid" annotate: PreprocessorSettings -class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"): +# TODO-EET: add eet override for SOA MC sampling +class DisaggregateAccessibilitySettings(PydanticReadable): #, extra="forbid") suffixes: DisaggregateAccessibilitySuffixes = DisaggregateAccessibilitySuffixes() ORIGIN_SAMPLE_SIZE: float | int = 0 """ From 6f8958be3a0b869156cb64ab673911585d637d42 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sat, 12 Apr 2025 05:54:06 +1000 Subject: [PATCH 109/135] lint --- .../abm/models/disaggregate_accessibility.py | 2 +- .../abm/models/joint_tour_participation.py | 5 +- .../abm/models/trip_departure_choice.py | 2 +- activitysim/abm/models/util/cdap.py | 4 +- activitysim/core/interaction_sample.py | 67 +++++++++------- .../core/interaction_sample_simulate.py | 36 ++++++--- activitysim/core/interaction_simulate.py | 2 +- activitysim/core/logit.py | 58 +++++++++----- activitysim/core/random.py | 4 +- activitysim/core/simulate.py | 80 +++++++++++-------- 10 files changed, 159 insertions(+), 101 deletions(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index 2a9d983e8b..3882384a73 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -76,7 +76,7 @@ class DisaggregateAccessibilityAnnotateSettings(PydanticReadable, extra="forbid" # TODO-EET: add eet override for SOA MC sampling -class DisaggregateAccessibilitySettings(PydanticReadable): #, extra="forbid") +class DisaggregateAccessibilitySettings(PydanticReadable): # , extra="forbid") suffixes: DisaggregateAccessibilitySuffixes = DisaggregateAccessibilitySuffixes() ORIGIN_SAMPLE_SIZE: float | int = 0 """ diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 9c3dc9e770..c74076fc17 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -208,7 +208,10 @@ def participants_chooser( print(unsatisfied_candidates.head(20)) # TODO-EET: explicit error term impl of force participation - if model_settings.FORCE_PARTICIPATION and not state.settings.use_explicit_error_terms: + if ( + model_settings.FORCE_PARTICIPATION + and not state.settings.use_explicit_error_terms + ): logger.warning( f"Forcing joint tour participation for {num_tours_remaining} tours." ) diff --git a/activitysim/abm/models/trip_departure_choice.py b/activitysim/abm/models/trip_departure_choice.py index 3886f3400a..577c65d17a 100644 --- a/activitysim/abm/models/trip_departure_choice.py +++ b/activitysim/abm/models/trip_departure_choice.py @@ -347,7 +347,7 @@ def choose_tour_leg_pattern( ) del utilities_df - chunk_sizer.log_df(trace_label, 'utilities_df', None) + chunk_sizer.log_df(trace_label, "utilities_df", None) else: # convert to probabilities (utilities exponentiated and normalized to probs) # probs is same shape as utilities, one row per chooser and one column for alternative diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index 7602cfdd40..e2f5598207 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -996,7 +996,9 @@ def household_activity_choices( utils = utils.add(joint_tour_utils) if state.settings.use_explicit_error_terms: - idx_choices, rands = logit.make_choices_utility_based(state, utils, trace_label=trace_label) + idx_choices, rands = logit.make_choices_utility_based( + state, utils, trace_label=trace_label + ) else: probs = logit.utils_to_probs(state, utils, trace_label=trace_label) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index e06a490068..5ba8254e25 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -45,61 +45,69 @@ def make_sample_choices_utility_based( # Note [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension # (len(choosers), alternative_count, sample_size) can get very large - #choices = np.zeros_like(utilities, dtype=np.uint32) - #zero_dim_index = np.arange(utilities.shape[0]) + # choices = np.zeros_like(utilities, dtype=np.uint32) + # zero_dim_index = np.arange(utilities.shape[0]) utils_array = utilities.to_numpy() - chunk_sizer.log_df(trace_label, 'utils_array', utils_array) + chunk_sizer.log_df(trace_label, "utils_array", utils_array) chosen_destinations = [] rands = state.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) - chunk_sizer.log_df(trace_label, 'rands', rands) + chunk_sizer.log_df(trace_label, "rands", rands) for i in range(sample_size): - #rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) - #choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 - #choices[ + # rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) + # choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 + # choices[ # zero_dim_index, # np.argmax(pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + utils_array, axis=1) - #] += 1 + # ] += 1 # created this once for memory logging if i > 0: - rands = state.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + rands = state.get_rn_generator().gumbel_for_df( + utilities, n=alternative_count + ) chosen_destinations.append(np.argmax(utils_array + rands, axis=1)) chosen_destinations = np.concatenate(chosen_destinations, axis=0) - chunk_sizer.log_df(trace_label, 'chosen_destinations', chosen_destinations) + chunk_sizer.log_df(trace_label, "chosen_destinations", chosen_destinations) del utils_array - chunk_sizer.log_df(trace_label, 'utils_array', None) + chunk_sizer.log_df(trace_label, "utils_array", None) del rands - chunk_sizer.log_df(trace_label, 'rands', None) + chunk_sizer.log_df(trace_label, "rands", None) chooser_idx = np.tile(np.arange(utilities.shape[0]), sample_size) - #chunk.log_df(trace_label, 'choices_array', choices_array) + # chunk.log_df(trace_label, 'choices_array', choices_array) # choices array has same dim as utilities, with values indicating number of counts per chooser and alternative # let's turn the nonzero values into a dataframe - #i, j = np.nonzero(choices_array) - chunk_sizer.log_df(trace_label, 'chooser_idx', chooser_idx) + # i, j = np.nonzero(choices_array) + chunk_sizer.log_df(trace_label, "chooser_idx", chooser_idx) probs = logit.utils_to_probs( - state, utilities, allow_zero_probs=allow_zero_probs, trace_label=trace_label, trace_choosers=choosers + state, + utilities, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label, + trace_choosers=choosers, + ) + chunk_sizer.log_df(trace_label, "probs", probs) + + choices_df = pd.DataFrame( + { + alt_col_name: alternatives.index.values[chosen_destinations], + # "pick_count": choices_array[i, j], + "prob": probs.to_numpy()[chooser_idx, chosen_destinations], + choosers.index.name: choosers.index.values[chooser_idx], + } ) - chunk_sizer.log_df(trace_label, 'probs', probs) - - choices_df = pd.DataFrame({ - alt_col_name: alternatives.index.values[chosen_destinations], - #"pick_count": choices_array[i, j], - "prob": probs.to_numpy()[chooser_idx, chosen_destinations], - choosers.index.name: choosers.index.values[chooser_idx] - }) chunk_sizer.log_df(trace_label, "choices_df", choices_df) del chooser_idx - chunk_sizer.log_df(trace_label, 'chooser_idx', None) + chunk_sizer.log_df(trace_label, "chooser_idx", None) del chosen_destinations - chunk_sizer.log_df(trace_label, 'chosen_destinations', None) + chunk_sizer.log_df(trace_label, "chosen_destinations", None) del probs - chunk_sizer.log_df(trace_label, 'probs', None) + chunk_sizer.log_df(trace_label, "probs", None) # handing this off to caller chunk_sizer.log_df(trace_label, "choices_df", None) @@ -525,7 +533,9 @@ def _interaction_sample( if compute_settings.use_explicit_error_terms is not None: use_eet = compute_settings.use_explicit_error_terms - logger.info(f"Interaction sample model-specific EET overrides for {trace_label}: eet = {use_eet}") + logger.info( + f"Interaction sample model-specific EET overrides for {trace_label}: eet = {use_eet}" + ) else: use_eet = state.settings.use_explicit_error_terms @@ -747,7 +757,6 @@ def interaction_sample( sample_size = min(sample_size, len(alternatives.index)) logger.info(f" --- interaction_sample sample size = {sample_size}") - result_list = [] for ( i, diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 5f37a3e83c..1ca92481e2 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -98,12 +98,16 @@ def _interaction_sample_simulate( # this is the more general check (not requiring is_monotonic_increasing) last_repeat = alternatives.index != np.roll(alternatives.index, -1) - assert (choosers.shape[0] == 1) or choosers.index.equals(alternatives.index[last_repeat]) + assert (choosers.shape[0] == 1) or choosers.index.equals( + alternatives.index[last_repeat] + ) have_trace_targets = state.tracing.has_trace_targets(choosers) if have_trace_targets: - state.tracing.trace_df(choosers, tracing.extend_trace_label(trace_label, "choosers")) + state.tracing.trace_df( + choosers, tracing.extend_trace_label(trace_label, "choosers") + ) state.tracing.trace_df( alternatives, tracing.extend_trace_label(trace_label, "alternatives"), @@ -129,7 +133,9 @@ def _interaction_sample_simulate( # assert alternatives.index.name == choosers.index.name # asserting the index names are the same tells us nothing about the underlying data so why? - logger.info(f"{trace_label} start merging choosers and alternatives to create interaction_df") + logger.info( + f"{trace_label} start merging choosers and alternatives to create interaction_df" + ) # drop variables before the interaction dataframe is created sharrow_enabled = state.settings.sharrow @@ -163,17 +169,23 @@ def _interaction_sample_simulate( ) interaction_df = alternatives.join(choosers, how="left", rsuffix="_chooser") - logger.info(f"{trace_label} end merging choosers and alternatives to create interaction_df") + logger.info( + f"{trace_label} end merging choosers and alternatives to create interaction_df" + ) if log_alt_losers: # logit.interaction_dataset adds ALT_CHOOSER_ID column if log_alt_losers is True # to enable detection of zero_prob-driving utils (e.g. -999 for all alts in a chooser) - interaction_df[interaction_simulate.ALT_CHOOSER_ID] = interaction_df.index.values + interaction_df[ + interaction_simulate.ALT_CHOOSER_ID + ] = interaction_df.index.values chunk_sizer.log_df(trace_label, "interaction_df", interaction_df) if have_trace_targets: - trace_rows, trace_ids = state.tracing.interaction_trace_rows(interaction_df, choosers) + trace_rows, trace_ids = state.tracing.interaction_trace_rows( + interaction_df, choosers + ) state.tracing.trace_df( interaction_df, @@ -229,7 +241,9 @@ def _interaction_sample_simulate( # so we need to pad with dummy utilities so low that they are never chosen # number of samples per chooser - sample_counts = interaction_utilities.groupby(interaction_utilities.index).size().values + sample_counts = ( + interaction_utilities.groupby(interaction_utilities.index).size().values + ) chunk_sizer.log_df(trace_label, "sample_counts", sample_counts) # max number of alternatvies for any chooser @@ -274,7 +288,9 @@ def _interaction_sample_simulate( if state.settings.use_explicit_error_terms: if want_logsums: - logsums = logit.utils_to_logsums(utilities_df, allow_zero_probs=allow_zero_probs) + logsums = logit.utils_to_logsums( + utilities_df, allow_zero_probs=allow_zero_probs + ) chunk_sizer.log_df(trace_label, "logsums", logsums) if skip_choice: @@ -334,7 +350,9 @@ def _interaction_sample_simulate( # make choices # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample - positions, rands = logit.make_choices(state, probs, trace_label=trace_label, trace_choosers=choosers) + positions, rands = logit.make_choices( + state, probs, trace_label=trace_label, trace_choosers=choosers + ) del probs chunk_sizer.log_df(trace_label, "probs", None) diff --git a/activitysim/core/interaction_simulate.py b/activitysim/core/interaction_simulate.py index 6dec23d0b2..7f70c83218 100644 --- a/activitysim/core/interaction_simulate.py +++ b/activitysim/core/interaction_simulate.py @@ -887,7 +887,7 @@ def _interaction_simulate( ) del utilities - chunk_sizer.log_df(trace_label, 'utilities', None) + chunk_sizer.log_df(trace_label, "utilities", None) else: # convert to probabilities (utilities exponentiated and normalized to probs) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index ba1bb79010..4351681ade 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -273,11 +273,15 @@ def utils_to_probs( # TODO-EET: check state, add type annotations, check new-world tracing, etc. def add_ev1_random(state: workflow.State, df: pd.DataFrame): nest_utils_for_choice = df.copy() - nest_utils_for_choice += state.get_rn_generator().gumbel_for_df(nest_utils_for_choice, n=df.shape[1]) + nest_utils_for_choice += state.get_rn_generator().gumbel_for_df( + nest_utils_for_choice, n=df.shape[1] + ) return nest_utils_for_choice -def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alternatives_by_name): +def choose_from_tree( + nest_utils, all_alternatives, logit_nest_groups, nest_alternatives_by_name +): for level, nest_names in logit_nest_groups.items(): if level == 1: next_level_alts = nest_alternatives_by_name[nest_names[0]] @@ -297,17 +301,21 @@ def choose_from_tree(nest_utils, all_alternatives, logit_nest_groups, nest_alter # alternatives and set the corresponding entry to 1 for each row, set all other alternatives at this level to zero. # Once the tree is walked (all alternatives have been processed), take the product of the alternatives in each # leaf's alternative list. Then pick the only alternative with entry 1, all others must be 0. -def make_choices_explicit_error_term_nl(state, nested_utilities, alt_order_array, nest_spec): - """ walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" +def make_choices_explicit_error_term_nl( + state, nested_utilities, alt_order_array, nest_spec +): + """walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" nest_utils_for_choice = add_ev1_random(state, nested_utilities) - all_alternatives = set(nest.name for nest in each_nest(nest_spec, type='leaf')) + all_alternatives = set(nest.name for nest in each_nest(nest_spec, type="leaf")) logit_nest_groups = group_nest_names_by_level(nest_spec) nest_alternatives_by_name = {n.name: n.alternatives for n in each_nest(nest_spec)} choices = nest_utils_for_choice.apply( - lambda x: choose_from_tree(x, all_alternatives, logit_nest_groups, nest_alternatives_by_name), - axis=1 + lambda x: choose_from_tree( + x, all_alternatives, logit_nest_groups, nest_alternatives_by_name + ), + axis=1, ) assert not choices.isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=nest_utils_for_choice.index) @@ -328,32 +336,38 @@ def make_choices_explicit_error_term_mnl(state, utilities): return choices -def make_choices_explicit_error_term(state, utilities, alt_order_array, nest_spec=None, trace_label=None): - trace_label = tracing.extend_trace_label(trace_label, 'make_choices_ru_frozen') +def make_choices_explicit_error_term( + state, utilities, alt_order_array, nest_spec=None, trace_label=None +): + trace_label = tracing.extend_trace_label(trace_label, "make_choices_ru_frozen") if nest_spec is None: choices = make_choices_explicit_error_term_mnl(state, utilities) else: - choices = make_choices_explicit_error_term_nl(state, utilities, alt_order_array, nest_spec) + choices = make_choices_explicit_error_term_nl( + state, utilities, alt_order_array, nest_spec + ) return choices # TODO-EET: memory usage def make_choices_utility_based( - state: workflow.State, - utilities: pd.DataFrame, - # for nested: need mapping of index to alternative name to "fake" indexes if I want to keep with current - # structure, OR need to make returning names optional. sharrow impl will make our life so much easier - name_mapping=None, - nest_spec=None, - trace_label: str = None, - trace_choosers=None, - allow_bad_probs=False, + state: workflow.State, + utilities: pd.DataFrame, + # for nested: need mapping of index to alternative name to "fake" indexes if I want to keep with current + # structure, OR need to make returning names optional. sharrow impl will make our life so much easier + name_mapping=None, + nest_spec=None, + trace_label: str = None, + trace_choosers=None, + allow_bad_probs=False, ) -> tuple[pd.Series, pd.Series]: - trace_label = tracing.extend_trace_label(trace_label, 'make_choices_utility_based') + trace_label = tracing.extend_trace_label(trace_label, "make_choices_utility_based") # TODO-EET: index of choices for nested utilities is different than unnested - this needs to be consistent for # turning indexes into alternative names to keep code changes to minimum for now - choices = make_choices_explicit_error_term(state, utilities, name_mapping, nest_spec, trace_label) + choices = make_choices_explicit_error_term( + state, utilities, name_mapping, nest_spec, trace_label + ) # TODO-EET: rands - log all zeros for now rands = pd.Series(np.zeros_like(utilities.index.values), index=utilities.index) return choices, rands @@ -686,7 +700,7 @@ def count_each_nest(spec, count): def group_nest_names_by_level(nest_spec): # group nests by level, returns {level: [nest.name at that level]} depth = np.max([x.level for x in each_nest(nest_spec)]) - nest_levels = {x: [] for x in range(1, depth+1)} + nest_levels = {x: [] for x in range(1, depth + 1)} for n in each_nest(nest_spec): nest_levels[n.level].append(n.name) return nest_levels diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 0ad75fdf70..dcef272ff6 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -282,12 +282,12 @@ def gumbel_for_df(self, df, step_name, n=1): # - reminder: prng must be called when yielded as generated sequence, not serialized generators = self._generators_for_df(df) - #rands = np.asanyarray([prng.gumbel(size=n) for prng in generators]) + # rands = np.asanyarray([prng.gumbel(size=n) for prng in generators]) # this is about 20% faster for large arrays, like for destination choice rands = np.asanyarray([-np.log(-np.log(prng.rand(n))) for prng in generators]) # update offset for rows we handled - self.row_states.loc[df.index, 'offset'] += n + self.row_states.loc[df.index, "offset"] += n return rands def normal_for_df(self, df, step_name, mu, sigma, lognormal=False, size=None): diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 3e9b600c0d..75ff479926 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1035,41 +1035,45 @@ def set_skim_wrapper_targets(df, skims): def compute_nested_utilities(raw_utilities, nest_spec): """ - compute nest utilities based on nesting coefficients - - For nest nodes this is the logsum of alternatives adjusted by nesting coefficient - - leaf <- raw_utility / nest_coefficient - nest <- ln(sum of exponentiated raw_utility of leaves) * nest_coefficient) - - Parameters - ---------- - raw_utilities : pandas.DataFrame - dataframe with the raw alternative utilities of all leaves - (what in non-nested logit would be the utilities of all the alternatives) - nest_spec : dict - Nest tree dict from the model spec yaml file - - Returns - ------- - nested_utilities : pandas.DataFrame - Will have the index of `raw_utilities` and columns for leaf and node utilities - """ + compute nest utilities based on nesting coefficients + + For nest nodes this is the logsum of alternatives adjusted by nesting coefficient + + leaf <- raw_utility / nest_coefficient + nest <- ln(sum of exponentiated raw_utility of leaves) * nest_coefficient) + + Parameters + ---------- + raw_utilities : pandas.DataFrame + dataframe with the raw alternative utilities of all leaves + (what in non-nested logit would be the utilities of all the alternatives) + nest_spec : dict + Nest tree dict from the model spec yaml file + + Returns + ------- + nested_utilities : pandas.DataFrame + Will have the index of `raw_utilities` and columns for leaf and node utilities + """ nested_utilities = pd.DataFrame(index=raw_utilities.index) for nest in logit.each_nest(nest_spec, post_order=True): name = nest.name if nest.is_leaf: # do not scale here, do afterwards so recursive structure works - nested_utilities[name] = raw_utilities[name].astype(float) / nest.product_of_coefficients + nested_utilities[name] = ( + raw_utilities[name].astype(float) / nest.product_of_coefficients + ) else: # the alternative nested_utilities will already have been computed due to post_order - with np.errstate(divide='ignore'): - nested_utilities[name] = \ - nest.coefficient * np.log( - np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) + with np.errstate(divide="ignore"): + nested_utilities[name] = nest.coefficient * np.log( + np.exp(nested_utilities[nest.alternatives]).sum(axis=1) + ) return nested_utilities + + # So it looks like TM1 was estimated such that the nest coefficient is the ratio of lower level and upper level # nest in larch. This means the values can all be between 0 and 1, unlike for larch where they need to be decreasing # going down the tree. In that world, the above would read @@ -1304,12 +1308,16 @@ def eval_mnl( if state.settings.use_explicit_error_terms: if custom_chooser: - choices, rands = custom_chooser(state, utilities, choosers, spec, trace_label) + choices, rands = custom_chooser( + state, utilities, choosers, spec, trace_label + ) else: - choices, rands = logit.make_choices_utility_based(state, utilities, trace_label=trace_label) + choices, rands = logit.make_choices_utility_based( + state, utilities, trace_label=trace_label + ) del utilities - chunk_sizer.log_df(trace_label, 'utilities', None) + chunk_sizer.log_df(trace_label, "utilities", None) else: probs = logit.utils_to_probs( @@ -1445,8 +1453,12 @@ def eval_nl( if want_logsums: # logsum of nest root # exponentiated utilities of leaves and nests - nested_exp_utilities = compute_nested_exp_utilities(raw_utilities, nest_spec) - chunk_sizer.log_df(trace_label, "nested_exp_utilities", nested_exp_utilities) + nested_exp_utilities = compute_nested_exp_utilities( + raw_utilities, nest_spec + ) + chunk_sizer.log_df( + trace_label, "nested_exp_utilities", nested_exp_utilities + ) logsums = pd.Series(np.log(nested_exp_utilities.root), index=choosers.index) chunk_sizer.log_df(trace_label, "logsums", logsums) @@ -1455,7 +1467,7 @@ def eval_nl( name_mapping = raw_utilities.columns.values del raw_utilities - chunk_sizer.log_df(trace_label, 'raw_utilities', None) + chunk_sizer.log_df(trace_label, "raw_utilities", None) if custom_chooser: choices, rands = custom_chooser( @@ -1465,7 +1477,7 @@ def eval_nl( choosers=choosers, spec=spec, nest_spec=nest_spec, - trace_label=trace_label + trace_label=trace_label, ) else: choices, rands = logit.make_choices_utility_based( @@ -1473,10 +1485,10 @@ def eval_nl( nested_utilities, name_mapping=name_mapping, nest_spec=nest_spec, - trace_label=trace_label + trace_label=trace_label, ) - del(nested_utilities) + del nested_utilities chunk_sizer.log_df(trace_label, "nested_utilities", None) else: From 9d12b08fd3dbcbb740d5317d6d91a385c160c437 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 25 May 2025 07:55:04 +1000 Subject: [PATCH 110/135] comments --- activitysim/core/interaction_sample.py | 3 ++- activitysim/core/simulate.py | 17 ----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 5ba8254e25..355290454d 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -43,7 +43,7 @@ def make_sample_choices_utility_based( assert isinstance(alternatives, pd.DataFrame) assert len(alternatives) == alternative_count - # Note [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension + # TODO-EET [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension # (len(choosers), alternative_count, sample_size) can get very large # choices = np.zeros_like(utilities, dtype=np.uint32) # zero_dim_index = np.arange(utilities.shape[0]) @@ -540,6 +540,7 @@ def _interaction_sample( use_eet = state.settings.use_explicit_error_terms # sample_size == 0 is for estimation mode, see below + # TODO-EET: add sample_size == 0 to EET, this needs to be an option not just for estimation mode if (sample_size != 0) and use_eet: choices_df = make_sample_choices_utility_based( state, diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 75ff479926..8acdc6f19c 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1074,23 +1074,6 @@ def compute_nested_utilities(raw_utilities, nest_spec): return nested_utilities -# So it looks like TM1 was estimated such that the nest coefficient is the ratio of lower level and upper level -# nest in larch. This means the values can all be between 0 and 1, unlike for larch where they need to be decreasing -# going down the tree. In that world, the above would read -# for nest in logit.each_nest(nest_spec, post_order=True): -# name = nest.name -# if nest.is_leaf: -# # do not scale here, do afterwards so recursive structure works -# nested_utilities[name] = raw_utilities[name].astype(float) -# else: -# # the alternative nested_utilities will already have been computed due to post_order -# with np.errstate(divide='ignore'): -# nested_utilities[name] = \ -# nest.coefficient * np.log(np.exp(nested_utilities[nest.alternatives]).sum(axis=1)) -# nested_utilities[name] /= parent_nest_scale # parent_nest_scale would need to be defined as part of nest -# # and would be = coeffiecient for leaves and for nests it would be that of the parent nest - - def compute_nested_exp_utilities(raw_utilities, nest_spec): """ compute exponentiated nest utilities based on nesting coefficients From 9a0fb663dc11e68b288d067aaa5916930f541035 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 25 May 2025 08:32:05 +1000 Subject: [PATCH 111/135] more comments --- activitysim/core/logit.py | 6 ++---- activitysim/core/simulate.py | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 4351681ade..91a9d2e973 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -317,6 +317,7 @@ def make_choices_explicit_error_term_nl( ), axis=1, ) + # TODO-EET: reporting like for zero probs assert not choices.isnull().any(), "No choice for XXX - implement reporting" choices = pd.Series(choices, index=nest_utils_for_choice.index) @@ -349,12 +350,10 @@ def make_choices_explicit_error_term( return choices -# TODO-EET: memory usage +# TODO-EET: memory logging def make_choices_utility_based( state: workflow.State, utilities: pd.DataFrame, - # for nested: need mapping of index to alternative name to "fake" indexes if I want to keep with current - # structure, OR need to make returning names optional. sharrow impl will make our life so much easier name_mapping=None, nest_spec=None, trace_label: str = None, @@ -677,7 +676,6 @@ def each_nest(nest_spec: dict | LogitNestSpec, type=None, post_order=False): yield nest -# TODO-EET: do I need to implement this for LogitNestSpec? def count_nests(nest_spec): """ count the nests in nest_spec, return 0 if nest_spec is none diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 8acdc6f19c..e4ed3f3878 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1060,7 +1060,6 @@ def compute_nested_utilities(raw_utilities, nest_spec): for nest in logit.each_nest(nest_spec, post_order=True): name = nest.name if nest.is_leaf: - # do not scale here, do afterwards so recursive structure works nested_utilities[name] = ( raw_utilities[name].astype(float) / nest.product_of_coefficients ) @@ -1427,12 +1426,11 @@ def eval_nl( ) if state.settings.use_explicit_error_terms: - # TODO-EET [janzill Jun2022]: combine with nested_exp_utilities? # utilities of leaves and nests nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) chunk_sizer.log_df(trace_label, "nested_utilities", nested_utilities) - # TODO-EET [janzill Jun2022]: this can be done from utils directly, but use existing methodology for prototype + # TODO-EET: use nested_utiltites directly to compute logsums? if want_logsums: # logsum of nest root # exponentiated utilities of leaves and nests From 0fa183ec26ad8e564b862054a0d3a38a0d285ad8 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 25 May 2025 11:15:07 +1000 Subject: [PATCH 112/135] jtp custom chooser for EET --- .../abm/models/joint_tour_participation.py | 59 +++++++++++-------- activitysim/core/logit.py | 13 ++-- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index c74076fc17..9819c87135 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -123,7 +123,7 @@ def get_tour_satisfaction(candidates, participate): def participants_chooser( state: workflow.State, - probs: pd.DataFrame, + probs_or_utils: pd.DataFrame, choosers: pd.DataFrame, spec: pd.DataFrame, trace_label: str, @@ -143,11 +143,11 @@ def participants_chooser( Parameters ---------- - probs : pandas.DataFrame + probs_or_utils : pandas.DataFrame Rows for choosers and columns for the alternatives from which they - are choosing. Values are expected to be valid probabilities across - each row, e.g. they should sum to 1, unless using explicit error terms, - in which case values are utilities. + are choosing. If running with explicit_error_terms, these are utilities. + Otherwise, values are expected to be valid probabilities across + each row, e.g. they should sum to 1. choosers : pandas.dataframe simple_simulate choosers df spec : pandas.DataFrame @@ -163,7 +163,7 @@ def participants_chooser( """ - assert probs.index.equals(choosers.index) + assert probs_or_utils.index.equals(choosers.index) # choice is boolean (participate or not) model_settings = JointTourParticipationSettings.read_settings_file( @@ -199,7 +199,7 @@ def participants_chooser( "%s max iterations exceeded (%s).", trace_label, MAX_ITERATIONS ) diagnostic_cols = ["tour_id", "household_id", "composition", "adult"] - unsatisfied_candidates = candidates[diagnostic_cols].join(probs) + unsatisfied_candidates = candidates[diagnostic_cols].join(probs_or_utils) state.tracing.write_csv( unsatisfied_candidates, file_name="%s.UNSATISFIED" % trace_label, @@ -207,18 +207,25 @@ def participants_chooser( ) print(unsatisfied_candidates.head(20)) - # TODO-EET: explicit error term impl of force participation - if ( - model_settings.FORCE_PARTICIPATION - and not state.settings.use_explicit_error_terms - ): + if model_settings.FORCE_PARTICIPATION: logger.warning( f"Forcing joint tour participation for {num_tours_remaining} tours." ) # anybody with probability > 0 is forced to join the joint tour - probs[choice_col] = np.where(probs[choice_col] > 0, 1, 0) - non_choice_col = [col for col in probs.columns if col != choice_col][0] - probs[non_choice_col] = 1 - probs[choice_col] + if state.settings.use_explicit_error_terms: + # need "is valid choice", which requires minimum probability. This depends + # on all other utilities, and we need to make sure that we set values such that we + # certainly choose those with non-zero values, and do not choose other ones. Let's + # use -999 as zero prob choice and 10 as definitive choice (based on diff and std gumbel). + probs_from_utils = logit.utils_to_probs(state, probs_or_utils) + probs_or_utils[choice_col] = np.where(probs_from_utils[choice_col] > 0, 10, -999) + non_choice_col = [col for col in probs_or_utils.columns if col != choice_col][0] + probs_or_utils[non_choice_col] = np.where(probs_or_utils[choice_col] == -999, 10, -999) + else: + probs_or_utils[choice_col] = np.where(probs_or_utils[choice_col] > 0, 1, 0) + non_choice_col = [col for col in probs_or_utils.columns if col != choice_col][0] + probs_or_utils[non_choice_col] = 1 - probs_or_utils[choice_col] + if iter > MAX_ITERATIONS + 1: raise RuntimeError( f"{num_tours_remaining} tours could not be satisfied even with forcing participation" @@ -228,14 +235,10 @@ def participants_chooser( f"{num_tours_remaining} tours could not be satisfied after {iter} iterations" ) - if state.settings.use_explicit_error_terms: - choices, rands = logit.make_choices_utility_based( - state, probs, trace_label=trace_label, trace_choosers=choosers - ) - else: - choices, rands = logit.make_choices( - state, probs, trace_label=trace_label, trace_choosers=choosers - ) + choice_function = ( + logit.make_choices_utility_based if state.settings.use_explicit_error_terms else logit.make_choices + ) + choices, rands = choice_function(state, probs_or_utils, trace_label=trace_label, trace_choosers=choosers) participate = choices == PARTICIPATE_CHOICE # satisfaction indexed by tour_id @@ -258,7 +261,7 @@ def participants_chooser( rands_list.append(rands[satisfied]) # remove candidates of satisfied tours - probs = probs[~satisfied] + probs_or_utils = probs_or_utils[~satisfied] candidates = candidates[~satisfied] logger.debug( @@ -427,6 +430,14 @@ def joint_tour_participation( if i not in model_settings.compute_settings.protect_columns: model_settings.compute_settings.protect_columns.append(i) + # TODO EET: this is related to the difference in nested logit and logit choice as per comment in + # make_choices_utility_based. As soon as alt_order_array is removed from arguments to + # make_choices_explicit_error_term_nl this guard can be removed + if state.settings.use_explicit_error_terms: + assert nest_spec is not None, ( + "Nested logit model custom chooser for EET requires name_mapping, currently not implemented in jtp" + ) + custom_chooser = participants_chooser choices = simulate.simple_simulate_by_chunk_id( diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 91a9d2e973..8bec1b910d 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -293,14 +293,6 @@ def choose_from_tree( raise ValueError("This should never happen - no alternative found") -# TODO-EET [janzill Jun2022]: make everything in nested and here numpy from beginning to make choices consistent with -# previous impl (want column index and not alternative name) -# Note: this is relatively slow due to the apply. -# It could *maybe* be sped up by using the fact that the nesting structure is the same for all rows: Add ev1(0,1) to -# all entries (as is currently being done). Then, at each level, pick the maximum of the available composite -# alternatives and set the corresponding entry to 1 for each row, set all other alternatives at this level to zero. -# Once the tree is walked (all alternatives have been processed), take the product of the alternatives in each -# leaf's alternative list. Then pick the only alternative with entry 1, all others must be 0. def make_choices_explicit_error_term_nl( state, nested_utilities, alt_order_array, nest_spec ): @@ -311,6 +303,11 @@ def make_choices_explicit_error_term_nl( logit_nest_groups = group_nest_names_by_level(nest_spec) nest_alternatives_by_name = {n.name: n.alternatives for n in each_nest(nest_spec)} + # Apply is slow. It could *maybe* be sped up by using the fact that the nesting structure is the same for all rows: + # Add ev1(0,1) to all entries (as is currently being done). Then, at each level, pick the maximum of the available + # composite alternatives and set the corresponding entry to 1 for each row, set all other alternatives at this level + # to zero. Once the tree is walked (all alternatives have been processed), take the product of the alternatives in + # each leaf's alternative list. Then pick the only alternative with entry 1, all others must be 0. choices = nest_utils_for_choice.apply( lambda x: choose_from_tree( x, all_alternatives, logit_nest_groups, nest_alternatives_by_name From 9900f48486282d9017109782308396dfe5487026 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Sun, 25 May 2025 19:23:41 +1000 Subject: [PATCH 113/135] lint --- .../abm/models/joint_tour_participation.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 9819c87135..80458ce27b 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -218,12 +218,22 @@ def participants_chooser( # certainly choose those with non-zero values, and do not choose other ones. Let's # use -999 as zero prob choice and 10 as definitive choice (based on diff and std gumbel). probs_from_utils = logit.utils_to_probs(state, probs_or_utils) - probs_or_utils[choice_col] = np.where(probs_from_utils[choice_col] > 0, 10, -999) - non_choice_col = [col for col in probs_or_utils.columns if col != choice_col][0] - probs_or_utils[non_choice_col] = np.where(probs_or_utils[choice_col] == -999, 10, -999) + probs_or_utils[choice_col] = np.where( + probs_from_utils[choice_col] > 0, 10, -999 + ) + non_choice_col = [ + col for col in probs_or_utils.columns if col != choice_col + ][0] + probs_or_utils[non_choice_col] = np.where( + probs_or_utils[choice_col] == -999, 10, -999 + ) else: - probs_or_utils[choice_col] = np.where(probs_or_utils[choice_col] > 0, 1, 0) - non_choice_col = [col for col in probs_or_utils.columns if col != choice_col][0] + probs_or_utils[choice_col] = np.where( + probs_or_utils[choice_col] > 0, 1, 0 + ) + non_choice_col = [ + col for col in probs_or_utils.columns if col != choice_col + ][0] probs_or_utils[non_choice_col] = 1 - probs_or_utils[choice_col] if iter > MAX_ITERATIONS + 1: @@ -236,9 +246,13 @@ def participants_chooser( ) choice_function = ( - logit.make_choices_utility_based if state.settings.use_explicit_error_terms else logit.make_choices + logit.make_choices_utility_based + if state.settings.use_explicit_error_terms + else logit.make_choices + ) + choices, rands = choice_function( + state, probs_or_utils, trace_label=trace_label, trace_choosers=choosers ) - choices, rands = choice_function(state, probs_or_utils, trace_label=trace_label, trace_choosers=choosers) participate = choices == PARTICIPATE_CHOICE # satisfaction indexed by tour_id @@ -431,12 +445,12 @@ def joint_tour_participation( model_settings.compute_settings.protect_columns.append(i) # TODO EET: this is related to the difference in nested logit and logit choice as per comment in - # make_choices_utility_based. As soon as alt_order_array is removed from arguments to + # make_choices_utility_based. As soon as alt_order_array is removed from arguments to # make_choices_explicit_error_term_nl this guard can be removed if state.settings.use_explicit_error_terms: - assert nest_spec is not None, ( - "Nested logit model custom chooser for EET requires name_mapping, currently not implemented in jtp" - ) + assert ( + nest_spec is not None + ), "Nested logit model custom chooser for EET requires name_mapping, currently not implemented in jtp" custom_chooser = participants_chooser From 892c3fbe8d2b2f29f1549804f14c5b8c70608496 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 06:39:37 +1000 Subject: [PATCH 114/135] move interaction sample without sampling out of MC loop to enable for both MC and eet --- activitysim/core/interaction_sample.py | 85 +++++++++++++++----------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 355290454d..7469618555 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -539,6 +539,45 @@ def _interaction_sample( else: use_eet = state.settings.use_explicit_error_terms + if sample_size == 0: + # FIXME return full alternative set rather than sample + logger.info( + "Estimation mode for %s using unsampled alternatives" % (trace_label,) + ) + probs = logit.utils_to_probs( + state, + utilities, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label, + trace_choosers=choosers, + overflow_protection=not allow_zero_probs, + ) + chunk_sizer.log_df(trace_label, "probs", probs) + + del utilities + chunk_sizer.log_df(trace_label, "utilities", None) + + index_name = probs.index.name + choices_df = ( + pd.melt(probs.reset_index(), id_vars=[index_name]) + .sort_values(by=index_name, kind="mergesort") + .set_index(index_name) + .rename(columns={"value": "prob"}) + .drop(columns="variable") + ) + chunk_sizer.log_df(trace_label, "choices_df", choices_df) + + del probs + chunk_sizer.log_df(trace_label, "probs", None) + + choices_df["pick_count"] = 1 + choices_df.insert( + 0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index)) + ) + chunk_sizer.log_df(trace_label, "choices_df", choices_df) + + return choices_df + # sample_size == 0 is for estimation mode, see below # TODO-EET: add sample_size == 0 to EET, this needs to be an option not just for estimation mode if (sample_size != 0) and use_eet: @@ -579,40 +618,18 @@ def _interaction_sample( column_labels=["alternative", "probability"], ) - if sample_size == 0: - # FIXME return full alternative set rather than sample - logger.info( - "Estimation mode for %s using unsampled alternatives" % (trace_label,) - ) - - index_name = probs.index.name - choices_df = ( - pd.melt(probs.reset_index(), id_vars=[index_name]) - .sort_values(by=index_name, kind="mergesort") - .set_index(index_name) - .rename(columns={"value": "prob"}) - .drop(columns="variable") - ) - - choices_df["pick_count"] = 1 - choices_df.insert( - 0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index)) - ) - - return choices_df - else: - choices_df = make_sample_choices( - state, - choosers, - probs, - alternatives, - sample_size, - alternative_count, - alt_col_name, - allow_zero_probs=allow_zero_probs, - trace_label=trace_label, - chunk_sizer=chunk_sizer, - ) + choices_df = make_sample_choices( + state, + choosers, + probs, + alternatives, + sample_size, + alternative_count, + alt_col_name, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label, + chunk_sizer=chunk_sizer, + ) del probs chunk_sizer.log_df(trace_label, "probs", None) From d6ecb0d9a1b9670fe2122f2a05db3953edd39f3a Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 06:51:17 +1000 Subject: [PATCH 115/135] update comments --- activitysim/core/interaction_sample.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 7469618555..9788573e15 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -578,9 +578,7 @@ def _interaction_sample( return choices_df - # sample_size == 0 is for estimation mode, see below - # TODO-EET: add sample_size == 0 to EET, this needs to be an option not just for estimation mode - if (sample_size != 0) and use_eet: + if use_eet: choices_df = make_sample_choices_utility_based( state, choosers, From 3793ec24442b9dbf759d8f900b5bb4a684f0a283 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 07:36:30 +1000 Subject: [PATCH 116/135] fix no sampling in interaction_sample --- activitysim/core/interaction_sample.py | 38 ++++++++------------------ 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 9788573e15..c9a9f81b38 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -540,41 +540,25 @@ def _interaction_sample( use_eet = state.settings.use_explicit_error_terms if sample_size == 0: - # FIXME return full alternative set rather than sample - logger.info( - "Estimation mode for %s using unsampled alternatives" % (trace_label,) - ) - probs = logit.utils_to_probs( - state, - utilities, - allow_zero_probs=allow_zero_probs, - trace_label=trace_label, - trace_choosers=choosers, - overflow_protection=not allow_zero_probs, - ) - chunk_sizer.log_df(trace_label, "probs", probs) + # Return full alternative set rather than sample + logger.info("Using unsampled alternatives for %s" % (trace_label,)) - del utilities - chunk_sizer.log_df(trace_label, "utilities", None) - - index_name = probs.index.name + index_name = utilities.index.name choices_df = ( - pd.melt(probs.reset_index(), id_vars=[index_name]) + pd.melt(utilities.reset_index(), id_vars=[index_name], value_name='prob', var_name=alt_col_name) .sort_values(by=index_name, kind="mergesort") .set_index(index_name) - .rename(columns={"value": "prob"}) - .drop(columns="variable") + .assign(prob = 1) + .assign(pick_count = 1) ) chunk_sizer.log_df(trace_label, "choices_df", choices_df) - del probs - chunk_sizer.log_df(trace_label, "probs", None) + # utilities are numbered 0..n-1 so we need to map back to alt ids + alternative_map = pd.Series(alternatives.index).to_dict() + choices_df[alt_col_name] = choices_df[alt_col_name].map(alternative_map) - choices_df["pick_count"] = 1 - choices_df.insert( - 0, alt_col_name, np.tile(alternatives.index.values, len(choosers.index)) - ) - chunk_sizer.log_df(trace_label, "choices_df", choices_df) + del utilities + chunk_sizer.log_df(trace_label, "utilities", None) return choices_df From fc686978108f018a433fd9d491cae6d88a443506 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 07:40:31 +1000 Subject: [PATCH 117/135] fix _interaction_sample return doc --- activitysim/core/interaction_sample.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index c9a9f81b38..a4efe30a8f 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -279,12 +279,12 @@ def _interaction_sample( choices_df : pandas.DataFrame A DataFrame where index should match the index of the choosers DataFrame - and columns alt_col_name, prob, rand, pick_count + and columns alt_col_name, prob, pick_count + alt_col_name: int + the identifier of the alternatives prob: float the probability of the chosen alternative - rand: float - the rand that did the choosing pick_count : int number of duplicate picks for chooser, alt """ From 28142cdf2cbf63a70ec38b16454c14273659bcb6 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 07:59:04 +1000 Subject: [PATCH 118/135] fix bool conversion of None for eet compute settings in sub-components --- activitysim/core/configuration/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/configuration/base.py b/activitysim/core/configuration/base.py index b11643cdc3..320278bf1a 100644 --- a/activitysim/core/configuration/base.py +++ b/activitysim/core/configuration/base.py @@ -227,7 +227,7 @@ def eet_subcomponent(self, subcomponent: str) -> bool: if isinstance(self.use_explicit_error_terms, dict): return self.use_explicit_error_terms.get(subcomponent, None) else: - return bool(self.use_explicit_error_terms) + return self.use_explicit_error_terms @contextmanager def pandas_option_context(self): From 5de1d23903677e5c744c95517e2804b768df4717 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 08:29:10 +1000 Subject: [PATCH 119/135] compute settings in disaggreagte accessibility w/o hack --- activitysim/abm/models/disaggregate_accessibility.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index 3882384a73..82c6d4249f 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -15,7 +15,7 @@ from activitysim.abm.models.util import tour_destination from activitysim.abm.tables import shadow_pricing from activitysim.core import estimation, los, tracing, util, workflow -from activitysim.core.configuration.base import PreprocessorSettings, PydanticReadable +from activitysim.core.configuration.base import PreprocessorSettings, PydanticReadable, ComputeSettings from activitysim.core.configuration.logit import TourLocationComponentSettings from activitysim.core.expressions import assign_columns @@ -75,8 +75,7 @@ class DisaggregateAccessibilityAnnotateSettings(PydanticReadable, extra="forbid" annotate: PreprocessorSettings -# TODO-EET: add eet override for SOA MC sampling -class DisaggregateAccessibilitySettings(PydanticReadable): # , extra="forbid") +class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"): suffixes: DisaggregateAccessibilitySuffixes = DisaggregateAccessibilitySuffixes() ORIGIN_SAMPLE_SIZE: float | int = 0 """ @@ -185,6 +184,8 @@ class DisaggregateAccessibilitySettings(PydanticReadable): # , extra="forbid") If not supplied or None, will default to the chunk size in the location choice model settings. """ + compute_settings: ComputeSettings | None = None + def read_disaggregate_accessibility_yaml( state: workflow.State, file_name @@ -770,6 +771,11 @@ def get_disaggregate_logsums( if disagg_model_settings.explicit_chunk is not None: model_settings.explicit_chunk = disagg_model_settings.explicit_chunk + # Can set compute settings for disaggregate accessibility + # Otherwise this will be set to whatever is in the location model settings + if disagg_model_settings.compute_settings is not None: + model_settings.compute_settings = disagg_model_settings.compute_settings + # Include the suffix tags to pass onto downstream logsum models (e.g., tour mode choice) if model_settings.LOGSUM_SETTINGS: suffixes = util.concat_suffix_dict(disagg_model_settings.suffixes) From 2a563ff1ea2b3e869c87022f17e5bf9455db35bb Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 08:39:50 +1000 Subject: [PATCH 120/135] lint --- activitysim/abm/models/disaggregate_accessibility.py | 6 +++++- activitysim/core/interaction_sample.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index 82c6d4249f..87bdaffca3 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -15,7 +15,11 @@ from activitysim.abm.models.util import tour_destination from activitysim.abm.tables import shadow_pricing from activitysim.core import estimation, los, tracing, util, workflow -from activitysim.core.configuration.base import PreprocessorSettings, PydanticReadable, ComputeSettings +from activitysim.core.configuration.base import ( + PreprocessorSettings, + PydanticReadable, + ComputeSettings, +) from activitysim.core.configuration.logit import TourLocationComponentSettings from activitysim.core.expressions import assign_columns diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index a4efe30a8f..89804730fc 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -545,11 +545,16 @@ def _interaction_sample( index_name = utilities.index.name choices_df = ( - pd.melt(utilities.reset_index(), id_vars=[index_name], value_name='prob', var_name=alt_col_name) + pd.melt( + utilities.reset_index(), + id_vars=[index_name], + value_name="prob", + var_name=alt_col_name, + ) .sort_values(by=index_name, kind="mergesort") .set_index(index_name) - .assign(prob = 1) - .assign(pick_count = 1) + .assign(prob=1) + .assign(pick_count=1) ) chunk_sizer.log_df(trace_label, "choices_df", choices_df) From ef0be214a901418e6a58c1eb946a5328bcaa7239 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 10:06:06 +1000 Subject: [PATCH 121/135] fix nest_spec is None bug --- activitysim/abm/models/joint_tour_participation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 80458ce27b..f8a0c3b755 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -449,7 +449,7 @@ def joint_tour_participation( # make_choices_explicit_error_term_nl this guard can be removed if state.settings.use_explicit_error_terms: assert ( - nest_spec is not None + nest_spec is None ), "Nested logit model custom chooser for EET requires name_mapping, currently not implemented in jtp" custom_chooser = participants_chooser From 1f815e82a78b042da88e85a0cb8daf81896f511f Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 10:24:21 +1000 Subject: [PATCH 122/135] remove comment --- activitysim/abm/models/util/cdap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index e2f5598207..848200097c 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -1035,7 +1035,7 @@ def household_activity_choices( "%s.hhsize%d_activity_choices" % (trace_label, hhsize), column_labels=["expression", "household"], ) - # TODO-EET: check if different num rands matters here + state.tracing.trace_df( rands, "%s.hhsize%d_rands" % (trace_label, hhsize), columns=[None, "rand"] ) From 41d49ffd3d7eff90420708abd6927937e20f456c Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 10:39:59 +1000 Subject: [PATCH 123/135] clean up comments --- activitysim/core/interaction_sample.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 89804730fc..c84fd2c4d1 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -43,10 +43,6 @@ def make_sample_choices_utility_based( assert isinstance(alternatives, pd.DataFrame) assert len(alternatives) == alternative_count - # TODO-EET [janzill Jun2022]: this needs for loop for memory like previous method, an array of dimension - # (len(choosers), alternative_count, sample_size) can get very large - # choices = np.zeros_like(utilities, dtype=np.uint32) - # zero_dim_index = np.arange(utilities.shape[0]) utils_array = utilities.to_numpy() chunk_sizer.log_df(trace_label, "utils_array", utils_array) chosen_destinations = [] @@ -54,13 +50,10 @@ def make_sample_choices_utility_based( rands = state.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) chunk_sizer.log_df(trace_label, "rands", rands) + # TODO-EET [janzill Jun2022]: using for-loop to keep memory usage low, an array of dimension + # (len(choosers), alternative_count, sample_size) can get very large. Probably better to + # use chunking for this. for i in range(sample_size): - # rands = pipeline.get_rn_generator().random_for_df(utilities, n=alternative_count) - # choices[zero_dim_index, np.argmax(inverse_ev1_cdf(rands) + utils_array, axis=1)] += 1 - # choices[ - # zero_dim_index, - # np.argmax(pipeline.get_rn_generator().gumbel_for_df(utilities, n=alternative_count) + utils_array, axis=1) - # ] += 1 # created this once for memory logging if i > 0: rands = state.get_rn_generator().gumbel_for_df( @@ -77,10 +70,6 @@ def make_sample_choices_utility_based( chunk_sizer.log_df(trace_label, "rands", None) chooser_idx = np.tile(np.arange(utilities.shape[0]), sample_size) - # chunk.log_df(trace_label, 'choices_array', choices_array) - # choices array has same dim as utilities, with values indicating number of counts per chooser and alternative - # let's turn the nonzero values into a dataframe - # i, j = np.nonzero(choices_array) chunk_sizer.log_df(trace_label, "chooser_idx", chooser_idx) probs = logit.utils_to_probs( @@ -95,7 +84,6 @@ def make_sample_choices_utility_based( choices_df = pd.DataFrame( { alt_col_name: alternatives.index.values[chosen_destinations], - # "pick_count": choices_array[i, j], "prob": probs.to_numpy()[chooser_idx, chosen_destinations], choosers.index.name: choosers.index.values[chooser_idx], } From 582c998f516ab3e26fa888a6de7fd8282d945926 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Mon, 26 May 2025 10:52:14 +1000 Subject: [PATCH 124/135] comment clean up --- activitysim/core/logit.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 8bec1b910d..5202a8cf44 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -270,11 +270,11 @@ def utils_to_probs( return probs -# TODO-EET: check state, add type annotations, check new-world tracing, etc. +# TODO-EET: add doc string, tracing def add_ev1_random(state: workflow.State, df: pd.DataFrame): nest_utils_for_choice = df.copy() nest_utils_for_choice += state.get_rn_generator().gumbel_for_df( - nest_utils_for_choice, n=df.shape[1] + nest_utils_for_choice, n=nest_utils_for_choice.shape[1] ) return nest_utils_for_choice @@ -293,8 +293,9 @@ def choose_from_tree( raise ValueError("This should never happen - no alternative found") +# TODO-EET: add doc string, tracing def make_choices_explicit_error_term_nl( - state, nested_utilities, alt_order_array, nest_spec + state, nested_utilities, alt_order_array, nest_spec, trace_label ): """walk down the nesting tree and make choice at each level, which is the root of the next level choice.""" nest_utils_for_choice = add_ev1_random(state, nested_utilities) @@ -315,7 +316,7 @@ def make_choices_explicit_error_term_nl( axis=1, ) # TODO-EET: reporting like for zero probs - assert not choices.isnull().any(), "No choice for XXX - implement reporting" + assert not choices.isnull().any(), f"No choice for {trace_label}" choices = pd.Series(choices, index=nest_utils_for_choice.index) # In order for choice indexing to be consistent with MNL and cumsum MC choices, we need to index in the order @@ -325,11 +326,12 @@ def make_choices_explicit_error_term_nl( return choices -def make_choices_explicit_error_term_mnl(state, utilities): +# TODO-EET: add doc string, tracing +def make_choices_explicit_error_term_mnl(state, utilities, trace_label): utilities_incl_unobs = add_ev1_random(state, utilities) choices = np.argmax(utilities_incl_unobs.to_numpy(), axis=1) # TODO-EET: reporting like for zero probs - assert not np.isnan(choices).any(), "No choice for XXX - implement reporting" + assert not np.isnan(choices).any(), f"No choice for {trace_label}" choices = pd.Series(choices, index=utilities_incl_unobs.index) return choices @@ -337,17 +339,16 @@ def make_choices_explicit_error_term_mnl(state, utilities): def make_choices_explicit_error_term( state, utilities, alt_order_array, nest_spec=None, trace_label=None ): - trace_label = tracing.extend_trace_label(trace_label, "make_choices_ru_frozen") + trace_label = tracing.extend_trace_label(trace_label, "make_choices_eet") if nest_spec is None: - choices = make_choices_explicit_error_term_mnl(state, utilities) + choices = make_choices_explicit_error_term_mnl(state, utilities, trace_label) else: choices = make_choices_explicit_error_term_nl( - state, utilities, alt_order_array, nest_spec + state, utilities, alt_order_array, nest_spec, trace_label ) return choices -# TODO-EET: memory logging def make_choices_utility_based( state: workflow.State, utilities: pd.DataFrame, From c40b989f717176f227c4705d09d50412899b49dd Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 29 May 2025 20:55:47 +1000 Subject: [PATCH 125/135] avoid choosing alternatives corresponding to padded utilities in interaction_sample_simulate --- activitysim/core/interaction_sample_simulate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 1ca92481e2..f45a0c1a01 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -262,7 +262,10 @@ def _interaction_sample_simulate( chunk_sizer.log_df(trace_label, "sample_counts", None) # insert the zero-prob utilities to pad each alternative set to same size - padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999) + # -999 is not small enough for EET, there are edge cases where alternatives + # corresponding to padded utilities are chosen. TODO: proper zero-prob handling. + zero_probability_util = -999999 # min(interaction_utilities.utility.min().min() - 100, -999) + padded_utilities = np.insert(interaction_utilities.utility.values, inserts, zero_probability_util) chunk_sizer.log_df(trace_label, "padded_utilities", padded_utilities) del inserts From 5636cd9025a012c2b53891234309949cdb11885d Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 29 May 2025 21:09:25 +1000 Subject: [PATCH 126/135] lint --- activitysim/core/interaction_sample_simulate.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index f45a0c1a01..f506f43316 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -264,8 +264,12 @@ def _interaction_sample_simulate( # insert the zero-prob utilities to pad each alternative set to same size # -999 is not small enough for EET, there are edge cases where alternatives # corresponding to padded utilities are chosen. TODO: proper zero-prob handling. - zero_probability_util = -999999 # min(interaction_utilities.utility.min().min() - 100, -999) - padded_utilities = np.insert(interaction_utilities.utility.values, inserts, zero_probability_util) + zero_probability_util = ( + -999999 + ) # min(interaction_utilities.utility.min().min() - 100, -999) + padded_utilities = np.insert( + interaction_utilities.utility.values, inserts, zero_probability_util + ) chunk_sizer.log_df(trace_label, "padded_utilities", padded_utilities) del inserts From 94e629ace077e64fbfa9db74da89c8ca53e7ec03 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Thu, 29 May 2025 22:10:43 +1000 Subject: [PATCH 127/135] patch other instance of padded_utility for eet --- activitysim/abm/models/trip_departure_choice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/models/trip_departure_choice.py b/activitysim/abm/models/trip_departure_choice.py index 4a33d9635a..7dc3353320 100644 --- a/activitysim/abm/models/trip_departure_choice.py +++ b/activitysim/abm/models/trip_departure_choice.py @@ -314,7 +314,7 @@ def choose_tour_leg_pattern( chunk_sizer.log_df(trace_label, "sample_counts", None) # insert the zero-prob utilities to pad each alternative set to same size - padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999) + padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999999) del inserts del interaction_utilities From e102a9e54dbfb9e3dc663c4da06cde840fd42a86 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 30 May 2025 09:17:47 +1000 Subject: [PATCH 128/135] adds validate utility method to align with unavailable choices in MC simulation --- .../core/interaction_sample_simulate.py | 29 ++++++-- activitysim/core/logit.py | 69 +++++++++++++++++++ 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index f506f43316..06efa3fb1b 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -303,11 +303,32 @@ def _interaction_sample_simulate( if skip_choice: return choosers.join(logsums.to_frame("logsums")) + utilities_df = logit.validate_utils( + state, + utilities_df, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label, + trace_choosers=choosers, + ) + + if allow_zero_probs: + zero_probs = ( + utilities_df.sum(axis=1) + <= utilities_df.shape[1] * logit.UTIL_UNAVAILABLE + ) + if zero_probs.any(): + # copied from proabability below, fix when that gets fixed + # FIXME this is kind of gnarly, but we force choice of first alt + utilities_df.loc[ + zero_probs, 0 + ] = 3.0 # arbitrary value much larger than UTIL_UNAVAILABLE + # positions is series with the chosen alternative represented as a column index in utilities_df # which is an integer between zero and num alternatives in the alternative sample positions, rands = logit.make_choices_utility_based( state, utilities_df, trace_label=trace_label, trace_choosers=choosers ) + del utilities_df chunk_sizer.log_df(trace_label, "utilities_df", None) else: @@ -382,13 +403,7 @@ def _interaction_sample_simulate( chunk_sizer.log_df(trace_label, "choices", choices) - # order is important for short circuiting - no explicit error terms => no zero_probs - if ( - allow_zero_probs - and not state.settings.use_explicit_error_terms - and zero_probs.any() - and zero_prob_choice_val is not None - ): + if allow_zero_probs and zero_probs.any() and zero_prob_choice_val is not None: # FIXME this is kind of gnarly, patch choice for zero_probs choices.loc[zero_probs] = zero_prob_choice_val diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 5202a8cf44..0162f0b8c8 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -17,6 +17,11 @@ EXP_UTIL_MIN = 1e-300 EXP_UTIL_MAX = np.inf +# TODO-EET: Figure out what type we want UTIL_MIN to be, currently np.float64 +UTIL_MIN = np.log(EXP_UTIL_MIN, dtype=np.float64) +UTIL_UNAVAILABLE = 1000.0 * (UTIL_MIN - 1.0) + + PROB_MIN = 0.0 PROB_MAX = 1.0 @@ -123,6 +128,70 @@ def utils_to_logsums(utils, exponentiated=False, allow_zero_probs=False): return logsums +def validate_utils( + state: workflow.State, + utils, + trace_label=None, + allow_zero_probs=False, + trace_choosers=None, +): + """ + Validate utilities to ensure non-available choices are treated the same in EET and MC. + For EET decisions, no conversion to probabilities is required because choices + are made on the basis of comparing utilities (only differences matter). + However, large negative utility values are used in practice to make choices + unavailable based on probability calculations, which boils down to evaluating + exp(utility). We here use this to define a minimum utility that corresponds + to an unavailable choice. + + Parameters + ---------- + utils : pandas.DataFrame + Rows should be choosers and columns should be alternatives. + + trace_label : str, optional + label for tracing bad utility or probability values + + allow_zero_probs : bool + if True value rows in which all utility alts are UTIL_MIN will be set to + UTIL_UNAVAILABLE. + + trace_choosers : pandas.dataframe + the choosers df (for interaction_simulate) to facilitate the reporting of hh_id + by report_bad_choices because it can't deduce hh_id from the interaction_dataset + which is indexed on index values from alternatives df + + Returns + ------- + utils : pandas.DataFrame + utils with values that would lead to zero probability replaced by UTIL_UNAVAILABLE + + """ + trace_label = tracing.extend_trace_label(trace_label, "validate_utils") + + utils_arr = utils.values + + np.putmask(utils_arr, utils_arr <= UTIL_MIN, UTIL_UNAVAILABLE) + + arr_sum = utils_arr.sum(axis=1) + + if not allow_zero_probs: + zero_probs = arr_sum <= utils_arr.shape[1] * UTIL_UNAVAILABLE + if zero_probs.any(): + report_bad_choices( + state, + zero_probs, + utils, + trace_label=tracing.extend_trace_label(trace_label, "zero_prob_utils"), + msg="all probabilities are zero", + trace_choosers=trace_choosers, + ) + + utils = pd.DataFrame(utils_arr, columns=utils.columns, index=utils.index) + + return utils + + def utils_to_probs( state: workflow.State, utils, From 66cdac906395d70e52ab2970e3fb5af86b59dfcb Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 30 May 2025 10:23:42 +1000 Subject: [PATCH 129/135] consistently treat utilities that would lead to zero choices in MC simulation --- .../abm/models/joint_tour_participation.py | 11 ++---- .../abm/models/trip_departure_choice.py | 5 ++- activitysim/abm/models/util/cdap.py | 2 + activitysim/core/interaction_sample.py | 22 +++++++++++ .../core/interaction_sample_simulate.py | 9 +---- activitysim/core/interaction_simulate.py | 3 ++ activitysim/core/pathbuilder.py | 37 +++++++++++++------ activitysim/core/simulate.py | 7 ++++ activitysim/core/test/test_logit.py | 1 + 9 files changed, 69 insertions(+), 28 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index f8a0c3b755..f57fb176f3 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -213,19 +213,16 @@ def participants_chooser( ) # anybody with probability > 0 is forced to join the joint tour if state.settings.use_explicit_error_terms: - # need "is valid choice", which requires minimum probability. This depends - # on all other utilities, and we need to make sure that we set values such that we - # certainly choose those with non-zero values, and do not choose other ones. Let's - # use -999 as zero prob choice and 10 as definitive choice (based on diff and std gumbel). - probs_from_utils = logit.utils_to_probs(state, probs_or_utils) + # need "is valid choice" such that we certainly choose those with non-zero values, + # and do not choose others. Let's use 3.0 as large value here. probs_or_utils[choice_col] = np.where( - probs_from_utils[choice_col] > 0, 10, -999 + probs_or_utils[choice_col] > logit.UTIL_MIN, 3.0, logit.UTIL_UNAVAILABLE ) non_choice_col = [ col for col in probs_or_utils.columns if col != choice_col ][0] probs_or_utils[non_choice_col] = np.where( - probs_or_utils[choice_col] == -999, 10, -999 + probs_or_utils[choice_col] <= logit.UTIL_MIN, 3.0, logit.UTIL_UNAVAILABLE ) else: probs_or_utils[choice_col] = np.where( diff --git a/activitysim/abm/models/trip_departure_choice.py b/activitysim/abm/models/trip_departure_choice.py index 7dc3353320..65a4468a62 100644 --- a/activitysim/abm/models/trip_departure_choice.py +++ b/activitysim/abm/models/trip_departure_choice.py @@ -314,7 +314,7 @@ def choose_tour_leg_pattern( chunk_sizer.log_df(trace_label, "sample_counts", None) # insert the zero-prob utilities to pad each alternative set to same size - padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999999) + padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999) del inserts del interaction_utilities @@ -339,6 +339,9 @@ def choose_tour_leg_pattern( ) if state.settings.use_explicit_error_terms: + utilities_df = logit.validate_utils( + state, utilities_df, trace_label=trace_label, trace_choosers=trip_segment + ) # make choices # positions is series with the chosen alternative represented as a column index in probs # which is an integer between zero and num alternatives in the alternative sample diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index 848200097c..71aca621de 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -996,6 +996,8 @@ def household_activity_choices( utils = utils.add(joint_tour_utils) if state.settings.use_explicit_error_terms: + utils = logit.validate_utils(state, utils, trace_label=trace_label) + idx_choices, rands = logit.make_choices_utility_based( state, utils, trace_label=trace_label ) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index c84fd2c4d1..60dfa64b01 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -43,6 +43,20 @@ def make_sample_choices_utility_based( assert isinstance(alternatives, pd.DataFrame) assert len(alternatives) == alternative_count + if allow_zero_probs: + zero_probs = ( + utilities.sum(axis=1) + <= utilities.shape[1] * logit.UTIL_UNAVAILABLE + ) + if zero_probs.all(): + return pd.DataFrame( + columns=[alt_col_name, "rand", "prob", choosers.index.name] + ) + if zero_probs.any(): + # remove from sample + utilities = utilities[~zero_probs] + choosers = choosers[~zero_probs] + utils_array = utilities.to_numpy() chunk_sizer.log_df(trace_label, "utils_array", utils_array) chosen_destinations = [] @@ -556,6 +570,14 @@ def _interaction_sample( return choices_df if use_eet: + utilities = logit.validate_utils( + state, + utilities, + allow_zero_probs=allow_zero_probs, + trace_label=trace_label, + trace_choosers=choosers, + ) + choices_df = make_sample_choices_utility_based( state, choosers, diff --git a/activitysim/core/interaction_sample_simulate.py b/activitysim/core/interaction_sample_simulate.py index 06efa3fb1b..0914334276 100644 --- a/activitysim/core/interaction_sample_simulate.py +++ b/activitysim/core/interaction_sample_simulate.py @@ -262,14 +262,7 @@ def _interaction_sample_simulate( chunk_sizer.log_df(trace_label, "sample_counts", None) # insert the zero-prob utilities to pad each alternative set to same size - # -999 is not small enough for EET, there are edge cases where alternatives - # corresponding to padded utilities are chosen. TODO: proper zero-prob handling. - zero_probability_util = ( - -999999 - ) # min(interaction_utilities.utility.min().min() - 100, -999) - padded_utilities = np.insert( - interaction_utilities.utility.values, inserts, zero_probability_util - ) + padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999) chunk_sizer.log_df(trace_label, "padded_utilities", padded_utilities) del inserts diff --git a/activitysim/core/interaction_simulate.py b/activitysim/core/interaction_simulate.py index 6867c2aebe..4aa3166d87 100644 --- a/activitysim/core/interaction_simulate.py +++ b/activitysim/core/interaction_simulate.py @@ -883,6 +883,9 @@ def _interaction_simulate( state.tracing.dump_df(DUMP, utilities, trace_label, "utilities") if state.settings.use_explicit_error_terms: + utilities = logit.validate_utils( + state, utilities, trace_label=trace_label, trace_choosers=choosers + ) positions, rands = logit.make_choices_utility_based( state, utilities, trace_label=trace_label, trace_choosers=choosers ) diff --git a/activitysim/core/pathbuilder.py b/activitysim/core/pathbuilder.py index bc0517df36..6ca6f7bb48 100644 --- a/activitysim/core/pathbuilder.py +++ b/activitysim/core/pathbuilder.py @@ -991,16 +991,16 @@ def build_virtual_path( utilities_df.index = orig.index with memo("#TVPB build_virtual_path make_choices"): - probs = logit.utils_to_probs( - self.network_los.state, - utilities_df, - allow_zero_probs=True, - trace_label=trace_label, - overflow_protection=False, - ) - chunk_sizer.log_df(trace_label, "probs", probs) - if trace: + probs = logit.utils_to_probs( + self.network_los.state, + utilities_df, + allow_zero_probs=True, + trace_label=trace_label, + overflow_protection=False, + ) + chunk_sizer.log_df(trace_label, "probs", probs) + choices = override_choices utilities_df["choices"] = choices @@ -1008,28 +1008,41 @@ def build_virtual_path( probs["choices"] = choices self.trace_df(probs, trace_label, "probs") + del probs + chunk_sizer.log_df(trace_label, "probs", None) else: if self.network_los.state.settings.use_explicit_error_terms: + utilities_df = logit.validate_utils( + self.network_los.state, utilities_df, allow_zero_probs=True, trace_label=trace_label + ) choices, rands = logit.make_choices_utility_based( self.network_los.state, utilities_df, - allow_bad_probs=True, trace_label=trace_label, ) else: + probs = logit.utils_to_probs( + self.network_los.state, + utilities_df, + allow_zero_probs=True, + trace_label=trace_label, + overflow_protection=False, + ) + chunk_sizer.log_df(trace_label, "probs", probs) + choices, rands = logit.make_choices( self.network_los.state, probs, allow_bad_probs=True, trace_label=trace_label, ) + del probs + chunk_sizer.log_df(trace_label, "probs", None) chunk_sizer.log_df(trace_label, "rands", rands) del rands chunk_sizer.log_df(trace_label, "rands", None) - del probs - chunk_sizer.log_df(trace_label, "probs", None) # we need to get path_set, btap, atap from path_df row with same seq and path_num # drop seq join column, but keep path_num of choice to override_choices when tracing diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index afb74c09dd..5baacb25e8 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1290,6 +1290,8 @@ def eval_mnl( ) if state.settings.use_explicit_error_terms: + utilities = logit.validate_utils(state, utilities, trace_label=trace_label, trace_choosers=choosers) + if custom_chooser: choices, rands = custom_chooser( state, utilities, choosers, spec, trace_label @@ -1427,6 +1429,11 @@ def eval_nl( ) if state.settings.use_explicit_error_terms: + # TODO-EET: Nested utility zero choice probability + raw_utilities = logit.validate_utils( + state, raw_utilities, allow_zero_probs=True, trace_label=trace_label + ) + # utilities of leaves and nests nested_utilities = compute_nested_utilities(raw_utilities, nest_spec) chunk_sizer.log_df(trace_label, "nested_utilities", nested_utilities) diff --git a/activitysim/core/test/test_logit.py b/activitysim/core/test/test_logit.py index e249475de2..eb8f7587cf 100644 --- a/activitysim/core/test/test_logit.py +++ b/activitysim/core/test/test_logit.py @@ -69,6 +69,7 @@ def utilities(choosers, spec, test_data): columns=test_data["probabilities"].columns, ) +# TODO-EET: Add tests here! def test_utils_to_probs(utilities, test_data): state = workflow.State().default_settings() From 3b48a076636cd0afc6d1ae19c0f7de6fa4638661 Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 30 May 2025 10:26:11 +1000 Subject: [PATCH 130/135] lint --- activitysim/abm/models/joint_tour_participation.py | 8 ++++++-- activitysim/core/interaction_sample.py | 3 +-- activitysim/core/pathbuilder.py | 6 ++++-- activitysim/core/simulate.py | 4 +++- activitysim/core/test/test_logit.py | 2 ++ 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index f57fb176f3..3844a3f217 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -216,13 +216,17 @@ def participants_chooser( # need "is valid choice" such that we certainly choose those with non-zero values, # and do not choose others. Let's use 3.0 as large value here. probs_or_utils[choice_col] = np.where( - probs_or_utils[choice_col] > logit.UTIL_MIN, 3.0, logit.UTIL_UNAVAILABLE + probs_or_utils[choice_col] > logit.UTIL_MIN, + 3.0, + logit.UTIL_UNAVAILABLE, ) non_choice_col = [ col for col in probs_or_utils.columns if col != choice_col ][0] probs_or_utils[non_choice_col] = np.where( - probs_or_utils[choice_col] <= logit.UTIL_MIN, 3.0, logit.UTIL_UNAVAILABLE + probs_or_utils[choice_col] <= logit.UTIL_MIN, + 3.0, + logit.UTIL_UNAVAILABLE, ) else: probs_or_utils[choice_col] = np.where( diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 60dfa64b01..886b010e03 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -45,8 +45,7 @@ def make_sample_choices_utility_based( if allow_zero_probs: zero_probs = ( - utilities.sum(axis=1) - <= utilities.shape[1] * logit.UTIL_UNAVAILABLE + utilities.sum(axis=1) <= utilities.shape[1] * logit.UTIL_UNAVAILABLE ) if zero_probs.all(): return pd.DataFrame( diff --git a/activitysim/core/pathbuilder.py b/activitysim/core/pathbuilder.py index 6ca6f7bb48..da241addcd 100644 --- a/activitysim/core/pathbuilder.py +++ b/activitysim/core/pathbuilder.py @@ -1013,7 +1013,10 @@ def build_virtual_path( else: if self.network_los.state.settings.use_explicit_error_terms: utilities_df = logit.validate_utils( - self.network_los.state, utilities_df, allow_zero_probs=True, trace_label=trace_label + self.network_los.state, + utilities_df, + allow_zero_probs=True, + trace_label=trace_label, ) choices, rands = logit.make_choices_utility_based( self.network_los.state, @@ -1043,7 +1046,6 @@ def build_virtual_path( del rands chunk_sizer.log_df(trace_label, "rands", None) - # we need to get path_set, btap, atap from path_df row with same seq and path_num # drop seq join column, but keep path_num of choice to override_choices when tracing columns_to_cache = ["btap", "atap", "path_set", "path_num"] diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 5baacb25e8..651db117c5 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1290,7 +1290,9 @@ def eval_mnl( ) if state.settings.use_explicit_error_terms: - utilities = logit.validate_utils(state, utilities, trace_label=trace_label, trace_choosers=choosers) + utilities = logit.validate_utils( + state, utilities, trace_label=trace_label, trace_choosers=choosers + ) if custom_chooser: choices, rands = custom_chooser( diff --git a/activitysim/core/test/test_logit.py b/activitysim/core/test/test_logit.py index eb8f7587cf..c82606981f 100644 --- a/activitysim/core/test/test_logit.py +++ b/activitysim/core/test/test_logit.py @@ -69,8 +69,10 @@ def utilities(choosers, spec, test_data): columns=test_data["probabilities"].columns, ) + # TODO-EET: Add tests here! + def test_utils_to_probs(utilities, test_data): state = workflow.State().default_settings() probs = logit.utils_to_probs(state, utilities, trace_label=None) From 6f31c76dbb608468b0dfd86a0253827fca968c8b Mon Sep 17 00:00:00 2001 From: Jan Zill Date: Fri, 30 May 2025 13:11:31 +1000 Subject: [PATCH 131/135] fix allow_zero_probs for prob calculation in EET interaction_sample --- activitysim/core/interaction_sample.py | 1 + 1 file changed, 1 insertion(+) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 886b010e03..47548e442d 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -90,6 +90,7 @@ def make_sample_choices_utility_based( utilities, allow_zero_probs=allow_zero_probs, trace_label=trace_label, + overflow_protection=not allow_zero_probs, trace_choosers=choosers, ) chunk_sizer.log_df(trace_label, "probs", probs) From 14fe80d659a5d84d44c987df9ddd8e90a0229288 Mon Sep 17 00:00:00 2001 From: Matt Richards Date: Mon, 12 Jan 2026 21:56:27 +1100 Subject: [PATCH 132/135] refactor estimation mode into method --- activitysim/core/interaction_sample.py | 84 ++++++++++++++++---------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index b92305e1e6..17dca93135 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -21,6 +21,7 @@ from activitysim.core.exceptions import SegmentedSpecificationError from activitysim.core.skim_dataset import DatasetWrapper from activitysim.core.skim_dictionary import SkimWrapper +from activitysim.core.workflow import State logger = logging.getLogger(__name__) @@ -639,39 +640,15 @@ def _interaction_sample( chunk_sizer.log_df(trace_label, "choices_df", choices_df) if estimation.manager.enabled and sample_size > 0: - # we need to ensure chosen alternative is included in the sample - survey_choices = estimation.manager.get_survey_destination_choices( - state, choosers, trace_label + choices_df = _ensure_chosen_alts_in_sample( + alt_col_name, + alternatives, + choices_df, + choosers, + probs, + state, + trace_label, ) - if survey_choices is not None: - assert ( - survey_choices.index == choosers.index - ).all(), "survey_choices and choosers must have the same index" - survey_choices.name = alt_col_name - survey_choices = survey_choices.dropna().astype( - choices_df[alt_col_name].dtype - ) - - # merge all survey choices onto choices_df - probs_df = probs.reset_index().melt( - id_vars=[choosers.index.name], - var_name=alt_col_name, - value_name="prob", - ) - # probs are numbered 0..n-1 so we need to map back to alt ids - zone_map = pd.Series(alternatives.index).to_dict() - probs_df[alt_col_name] = probs_df[alt_col_name].map(zone_map) - - survey_choices = pd.merge( - survey_choices, - probs_df, - on=[choosers.index.name, alt_col_name], - how="left", - ) - survey_choices["rand"] = 0 - survey_choices["prob"].fillna(0, inplace=True) - choices_df = pd.concat([choices_df, survey_choices], ignore_index=True) - choices_df.sort_values(by=[choosers.index.name], inplace=True) del probs chunk_sizer.log_df(trace_label, "probs", None) @@ -722,6 +699,49 @@ def _interaction_sample( return choices_df +def _ensure_chosen_alts_in_sample( + alt_col_name, + alternatives: pd.DataFrame, + choices_df: pd.DataFrame, + choosers: pd.DataFrame, + probs: pd.DataFrame, + state: State, + trace_label, +) -> pd.DataFrame: + # we need to ensure chosen alternative is included in the sample + survey_choices = estimation.manager.get_survey_destination_choices( + state, choosers, trace_label + ) + if survey_choices is not None: + assert ( + survey_choices.index == choosers.index + ).all(), "survey_choices and choosers must have the same index" + survey_choices.name = alt_col_name + survey_choices = survey_choices.dropna().astype(choices_df[alt_col_name].dtype) + + # merge all survey choices onto choices_df + probs_df = probs.reset_index().melt( + id_vars=[choosers.index.name], + var_name=alt_col_name, + value_name="prob", + ) + # probs are numbered 0..n-1 so we need to map back to alt ids + zone_map = pd.Series(alternatives.index).to_dict() + probs_df[alt_col_name] = probs_df[alt_col_name].map(zone_map) + + survey_choices = pd.merge( + survey_choices, + probs_df, + on=[choosers.index.name, alt_col_name], + how="left", + ) + survey_choices["rand"] = 0 + survey_choices["prob"].fillna(0, inplace=True) + choices_df = pd.concat([choices_df, survey_choices], ignore_index=True) + choices_df.sort_values(by=[choosers.index.name], inplace=True) + return choices_df + + def interaction_sample( state: workflow.State, choosers: pd.DataFrame, From 2a8c8829fb305989dc15fdc0c444b828689449f7 Mon Sep 17 00:00:00 2001 From: Matt Richards Date: Mon, 12 Jan 2026 22:02:45 +1100 Subject: [PATCH 133/135] make estimation choice set preservation work for eet --- activitysim/core/interaction_sample.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 17dca93135..6f61c009db 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -111,13 +111,12 @@ def make_sample_choices_utility_based( chunk_sizer.log_df(trace_label, "chooser_idx", None) del chosen_destinations chunk_sizer.log_df(trace_label, "chosen_destinations", None) - del probs - chunk_sizer.log_df(trace_label, "probs", None) # handing this off to caller + chunk_sizer.log_df(trace_label, "probs", None) chunk_sizer.log_df(trace_label, "choices_df", None) - return choices_df + return choices_df, probs def make_sample_choices( @@ -581,7 +580,7 @@ def _interaction_sample( trace_choosers=choosers, ) - choices_df = make_sample_choices_utility_based( + choices_df, probs = make_sample_choices_utility_based( state, choosers, utilities, @@ -593,14 +592,25 @@ def _interaction_sample( trace_label=trace_label, chunk_sizer=chunk_sizer, ) + del utilities chunk_sizer.log_df(trace_label, "utilities", None) if estimation.manager.enabled and sample_size > 0: - raise NotImplementedError( - "estimation.manager.enabled and sample_size > 0 needs probs, not in EET case here" + + choices_df = _ensure_chosen_alts_in_sample( + alt_col_name, + alternatives, + choices_df, + choosers, + probs, + state, + trace_label, ) + del probs + chunk_sizer.log_df(trace_label, "probs", None) + else: # convert to probabilities (utilities exponentiated and normalized to probs) # probs is same shape as utilities, one row per chooser and one column for alternative From 71d9481b175f9caf061c2180171ce619b49d6756 Mon Sep 17 00:00:00 2001 From: Matt Richards Date: Tue, 3 Feb 2026 21:20:12 +1000 Subject: [PATCH 134/135] fix zero probs with new arg types --- activitysim/core/interaction_sample.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index 6f61c009db..66e59eed8b 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -51,9 +51,11 @@ def make_sample_choices_utility_based( utilities.sum(axis=1) <= utilities.shape[1] * logit.UTIL_UNAVAILABLE ) if zero_probs.all(): - return pd.DataFrame( + utils = pd.DataFrame( columns=[alt_col_name, "rand", "prob", choosers.index.name] ) + probs = pd.DataFrame(0.0, index=utils.index, columns=utils.columns) + return utils, probs if zero_probs.any(): # remove from sample utilities = utilities[~zero_probs] From d115b3fbdafd2c192954816d6657c3b1c4f69c6a Mon Sep 17 00:00:00 2001 From: Matt Richards Date: Tue, 3 Feb 2026 21:32:08 +1000 Subject: [PATCH 135/135] minor type checking --- activitysim/core/logit.py | 2 +- activitysim/core/simulate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/activitysim/core/logit.py b/activitysim/core/logit.py index 0030168bb2..816e1d0d26 100644 --- a/activitysim/core/logit.py +++ b/activitysim/core/logit.py @@ -428,7 +428,7 @@ def make_choices_utility_based( utilities: pd.DataFrame, name_mapping=None, nest_spec=None, - trace_label: str = None, + trace_label: str | None = None, trace_choosers=None, allow_bad_probs=False, ) -> tuple[pd.Series, pd.Series]: diff --git a/activitysim/core/simulate.py b/activitysim/core/simulate.py index 74b182914a..7796b0a8f0 100644 --- a/activitysim/core/simulate.py +++ b/activitysim/core/simulate.py @@ -1264,7 +1264,7 @@ def eval_mnl( choosers, spec, locals_d, - custom_chooser: CustomChooser_T, + custom_chooser: CustomChooser_T | None, estimator, log_alt_losers=False, want_logsums=False,