EET on VITM 2 activitysim

activitysim/abm/models/disaggregate_accessibility.py

@@ -15,7 +15,11 @@
 from activitysim.abm.models.util import tour_destination
 from activitysim.abm.tables import shadow_pricing
 from activitysim.core import estimation, los, tracing, util, workflow
-from activitysim.core.configuration.base import PreprocessorSettings, PydanticReadable
+from activitysim.core.configuration.base import (
+    ComputeSettings,
+    PreprocessorSettings,
+    PydanticReadable,
+)
 from activitysim.core.configuration.logit import TourLocationComponentSettings
 from activitysim.core.expressions import assign_columns
 
@@ -184,6 +188,8 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
     If not supplied or None, will default to the chunk size in the location choice model settings.
     """
 
+    compute_settings: ComputeSettings | None = None
+
 
 def read_disaggregate_accessibility_yaml(
     state: workflow.State, file_name
@@ -783,6 +789,11 @@ def get_disaggregate_logsums(
         if disagg_model_settings.explicit_chunk is not None:
             model_settings.explicit_chunk = disagg_model_settings.explicit_chunk
 
+        # Can set compute settings for disaggregate accessibility
+        # Otherwise this will be set to whatever is in the location model settings
+        if disagg_model_settings.compute_settings is not None:
+            model_settings.compute_settings = disagg_model_settings.compute_settings
+
         # Include the suffix tags to pass onto downstream logsum models (e.g., tour mode choice)
         if model_settings.LOGSUM_SETTINGS:
             suffixes = util.concat_suffix_dict(disagg_model_settings.suffixes)

activitysim/abm/models/joint_tour_participation.py

@@ -20,8 +20,8 @@
 )
 from activitysim.core.configuration.base import ComputeSettings, PreprocessorSettings
 from activitysim.core.configuration.logit import LogitComponentSettings
-from activitysim.core.util import assign_in_place, reindex
 from activitysim.core.exceptions import InvalidTravelError
+from activitysim.core.util import assign_in_place, reindex
 
 logger = logging.getLogger(__name__)
 
@@ -127,7 +127,7 @@ def get_tour_satisfaction(candidates, participate):
 
 def participants_chooser(
     state: workflow.State,
-    probs: pd.DataFrame,
+    probs_or_utils: pd.DataFrame,
     choosers: pd.DataFrame,
     spec: pd.DataFrame,
     trace_label: str,
@@ -147,9 +147,10 @@ def participants_chooser(
 
     Parameters
     ----------
-    probs : pandas.DataFrame
+    probs_or_utils : pandas.DataFrame
         Rows for choosers and columns for the alternatives from which they
-        are choosing. Values are expected to be valid probabilities across
+        are choosing. If running with explicit_error_terms, these are utilities.
+        Otherwise, values are expected to be valid probabilities across
         each row, e.g. they should sum to 1.
     choosers : pandas.dataframe
         simple_simulate choosers df
@@ -166,7 +167,7 @@ def participants_chooser(
 
     """
 
-    assert probs.index.equals(choosers.index)
+    assert probs_or_utils.index.equals(choosers.index)
 
     # choice is boolean (participate or not)
     model_settings = JointTourParticipationSettings.read_settings_file(
@@ -202,7 +203,7 @@ def participants_chooser(
                 "%s max iterations exceeded (%s).", trace_label, MAX_ITERATIONS
             )
             diagnostic_cols = ["tour_id", "household_id", "composition", "adult"]
-            unsatisfied_candidates = candidates[diagnostic_cols].join(probs)
+            unsatisfied_candidates = candidates[diagnostic_cols].join(probs_or_utils)
             state.tracing.write_csv(
                 unsatisfied_candidates,
                 file_name="%s.UNSATISFIED" % trace_label,
@@ -215,9 +216,31 @@ def participants_chooser(
                     f"Forcing joint tour participation for {num_tours_remaining} tours."
                 )
                 # anybody with probability > 0 is forced to join the joint tour
-                probs[choice_col] = np.where(probs[choice_col] > 0, 1, 0)
-                non_choice_col = [col for col in probs.columns if col != choice_col][0]
-                probs[non_choice_col] = 1 - probs[choice_col]
+                if state.settings.use_explicit_error_terms:
+                    # need "is valid choice" such that we certainly choose those with non-zero values,
+                    # and do not choose others. Let's use 3.0 as large value here.
+                    probs_or_utils[choice_col] = np.where(
+                        probs_or_utils[choice_col] > logit.UTIL_MIN,
+                        3.0,
+                        logit.UTIL_UNAVAILABLE,
+                    )
+                    non_choice_col = [
+                        col for col in probs_or_utils.columns if col != choice_col
+                    ][0]
+                    probs_or_utils[non_choice_col] = np.where(
+                        probs_or_utils[choice_col] <= logit.UTIL_MIN,
+                        3.0,
+                        logit.UTIL_UNAVAILABLE,
+                    )
+                else:
+                    probs_or_utils[choice_col] = np.where(
+                        probs_or_utils[choice_col] > 0, 1, 0
+                    )
+                    non_choice_col = [
+                        col for col in probs_or_utils.columns if col != choice_col
+                    ][0]
+                    probs_or_utils[non_choice_col] = 1 - probs_or_utils[choice_col]
+
                 if iter > MAX_ITERATIONS + 1:
                     raise InvalidTravelError(
                         f"{num_tours_remaining} tours could not be satisfied even with forcing participation"
@@ -227,8 +250,13 @@ def participants_chooser(
                     f"{num_tours_remaining} tours could not be satisfied after {iter} iterations"
                 )
 
-        choices, rands = logit.make_choices(
-            state, probs, trace_label=trace_label, trace_choosers=choosers
+        choice_function = (
+            logit.make_choices_utility_based
+            if state.settings.use_explicit_error_terms
+            else logit.make_choices
+        )
+        choices, rands = choice_function(
+            state, probs_or_utils, trace_label=trace_label, trace_choosers=choosers
         )
         participate = choices == PARTICIPATE_CHOICE
 
@@ -252,7 +280,7 @@ def participants_chooser(
             rands_list.append(rands[satisfied])
 
             # remove candidates of satisfied tours
-            probs = probs[~satisfied]
+            probs_or_utils = probs_or_utils[~satisfied]
             candidates = candidates[~satisfied]
 
         logger.debug(
@@ -401,6 +429,16 @@ def joint_tour_participation(
         if i not in model_settings.compute_settings.protect_columns:
             model_settings.compute_settings.protect_columns.append(i)
 
+    # TODO EET: this is related to the difference in nested logit and logit choice as per comment in
+    # make_choices_utility_based. As soon as alt_order_array is removed from arguments to
+    # make_choices_explicit_error_term_nl this guard can be removed
+    if state.settings.use_explicit_error_terms:
+        assert (
+            nest_spec is None
+        ), "Nested logit model custom chooser for EET requires name_mapping, currently not implemented in jtp"
+
+    custom_chooser = participants_chooser
+
     choices = simulate.simple_simulate_by_chunk_id(
         state,
         choosers=candidates,
@@ -409,7 +447,7 @@ def joint_tour_participation(
         locals_d=constants,
         trace_label=trace_label,
         trace_choice_name="participation",
-        custom_chooser=participants_chooser,
+        custom_chooser=custom_chooser,
         estimator=estimator,
         compute_settings=model_settings.compute_settings,
     )

activitysim/abm/models/trip_departure_choice.py

@@ -24,10 +24,10 @@
     PreprocessorSettings,
     PydanticCompute,
 )
+from activitysim.core.exceptions import SegmentedSpecificationError
 from activitysim.core.skim_dataset import SkimDataset
 from activitysim.core.skim_dictionary import SkimDict
 from activitysim.core.util import reindex
-from activitysim.core.exceptions import SegmentedSpecificationError
 
 logger = logging.getLogger(__name__)
 
@@ -351,37 +351,51 @@ def choose_tour_leg_pattern(
             column_labels=["alternative", "utility"],
         )
 
-    # convert to probabilities (utilities exponentiated and normalized to probs)
-    # probs is same shape as utilities, one row per chooser and one column for alternative
-    probs = logit.utils_to_probs(
-        state, utilities_df, trace_label=trace_label, trace_choosers=trip_segment
-    )
+    if state.settings.use_explicit_error_terms:
+        utilities_df = logit.validate_utils(
+            state, utilities_df, trace_label=trace_label, trace_choosers=trip_segment
+        )
+        # make choices
+        # positions is series with the chosen alternative represented as a column index in probs
+        # which is an integer between zero and num alternatives in the alternative sample
+        positions, rands = logit.make_choices_utility_based(
+            state, utilities_df, trace_label=trace_label, trace_choosers=trip_segment
+        )
+
+        del utilities_df
+        chunk_sizer.log_df(trace_label, "utilities_df", None)
+    else:
+        # convert to probabilities (utilities exponentiated and normalized to probs)
+        # probs is same shape as utilities, one row per chooser and one column for alternative
+        probs = logit.utils_to_probs(
+            state, utilities_df, trace_label=trace_label, trace_choosers=trip_segment
+        )
 
-    chunk_sizer.log_df(trace_label, "probs", probs)
+        chunk_sizer.log_df(trace_label, "probs", probs)
 
-    del utilities_df
-    chunk_sizer.log_df(trace_label, "utilities_df", None)
+        del utilities_df
+        chunk_sizer.log_df(trace_label, "utilities_df", None)
 
-    if have_trace_targets:
-        state.tracing.trace_df(
-            probs,
-            tracing.extend_trace_label(trace_label, "probs"),
-            column_labels=["alternative", "probability"],
+        if have_trace_targets:
+            state.tracing.trace_df(
+                probs,
+                tracing.extend_trace_label(trace_label, "probs"),
+                column_labels=["alternative", "probability"],
+            )
+
+        # make choices
+        # positions is series with the chosen alternative represented as a column index in probs
+        # which is an integer between zero and num alternatives in the alternative sample
+        positions, rands = logit.make_choices(
+            state, probs, trace_label=trace_label, trace_choosers=trip_segment
         )
 
-    # make choices
-    # positions is series with the chosen alternative represented as a column index in probs
-    # which is an integer between zero and num alternatives in the alternative sample
-    positions, rands = logit.make_choices(
-        state, probs, trace_label=trace_label, trace_choosers=trip_segment
-    )
+        del probs
+        chunk_sizer.log_df(trace_label, "probs", None)
 
     chunk_sizer.log_df(trace_label, "positions", positions)
     chunk_sizer.log_df(trace_label, "rands", rands)
 
-    del probs
-    chunk_sizer.log_df(trace_label, "probs", None)
-
     # shouldn't have chosen any of the dummy pad utilities
     assert positions.max() < max_sample_count
 

activitysim/abm/models/util/cdap.py

@@ -999,11 +999,18 @@ def household_activity_choices(
         # add joint util to util
         utils = utils.add(joint_tour_utils)
 
-    probs = logit.utils_to_probs(state, utils, trace_label=trace_label)
+    if state.settings.use_explicit_error_terms:
+        utils = logit.validate_utils(state, utils, trace_label=trace_label)
 
-    # select an activity pattern alternative for each household based on probability
-    # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs
-    idx_choices, rands = logit.make_choices(state, probs, trace_label=trace_label)
+        idx_choices, rands = logit.make_choices_utility_based(
+            state, utils, trace_label=trace_label
+        )
+    else:
+        probs = logit.utils_to_probs(state, utils, trace_label=trace_label)
+
+        # select an activity pattern alternative for each household based on probability
+        # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs
+        idx_choices, rands = logit.make_choices(state, probs, trace_label=trace_label)
 
     # convert choice expressed as index into alternative name from util column label
     choices = pd.Series(utils.columns[idx_choices].values, index=utils.index)
@@ -1021,16 +1028,20 @@ def household_activity_choices(
             "%s.hhsize%d_utils" % (trace_label, hhsize),
             column_labels=["expression", "household"],
         )
-        state.tracing.trace_df(
-            probs,
-            "%s.hhsize%d_probs" % (trace_label, hhsize),
-            column_labels=["expression", "household"],
-        )
+
+        if not state.settings.use_explicit_error_terms:
+            state.tracing.trace_df(
+                probs,
+                "%s.hhsize%d_probs" % (trace_label, hhsize),
+                column_labels=["expression", "household"],
+            )
+
         state.tracing.trace_df(
             choices,
             "%s.hhsize%d_activity_choices" % (trace_label, hhsize),
             column_labels=["expression", "household"],
         )
+
         state.tracing.trace_df(
             rands, "%s.hhsize%d_rands" % (trace_label, hhsize), columns=[None, "rand"]
         )

activitysim/core/configuration/base.py

@@ -135,6 +135,10 @@ class ComputeSettings(PydanticBase):
     Sharrow settings for a component.
     """
 
+    # Make this more general compute settings and use for explicit error term overrides
+    # Default None work for sub-components defined in getter below (eet_subcomponent)
+    use_explicit_error_terms: None | bool | dict[str, bool] = None
+
     sharrow_skip: bool | dict[str, bool] = False
     """Skip sharrow when evaluating this component.
 
@@ -218,6 +222,13 @@ def should_skip(self, subcomponent: str) -> bool:
         else:
             return bool(self.sharrow_skip)
 
+    def eet_subcomponent(self, subcomponent: str) -> bool:
+        """Check for EET overrides for a particular subcomponent."""
+        if isinstance(self.use_explicit_error_terms, dict):
+            return self.use_explicit_error_terms.get(subcomponent, None)
+        else:
+            return self.use_explicit_error_terms
+
     @contextmanager
     def pandas_option_context(self):
         """Context manager to set pandas options for compute settings."""
@@ -266,6 +277,7 @@ def subcomponent_settings(self, subcomponent: str) -> ComputeSettings:
             use_numba=self.use_numba,
             drop_unused_columns=self.drop_unused_columns,
             protect_columns=self.protect_columns,
+            use_explicit_error_terms=self.eet_subcomponent(subcomponent),
         )
 
 

activitysim/core/configuration/top.py

@@ -701,6 +701,7 @@ def _check_store_skims_in_shm(self):
         "memory_profile",
         "instrument",
         "sharrow",
+        "use_explicit_error_terms",
     )
     """
     Setting to log on startup.
@@ -778,7 +779,18 @@ def _check_store_skims_in_shm(self):
     """
     run checks to validate that YAML settings files are loadable and spec and coefficent csv can be resolved.
 
-    should catch many common errors early, including missing required configurations or specified coefficient labels without defined values.  
+    should catch many common errors early, including missing required configurations or specified coefficient labels without defined values.
+    """
+
+    use_explicit_error_terms: bool = False
+    """
+    Make choice from random utility model by drawing from distribution of unobserved
+    part of utility and taking the maximum of total utility.
+
+    Defaults to standard Monte Carlo method, i.e., calculating probabilities and then
+    drawing a single uniform random number to draw from cumulative probabily.
+
+    .. versionadded:: 1.x
     """
 
     other_settings: dict[str, Any] = None