From d244cc92ee7f35659a36ebacb763ff7356e1fddc Mon Sep 17 00:00:00 2001 From: VincentAuriau Date: Sat, 27 Sep 2025 12:40:16 +0200 Subject: [PATCH 1/4] ADD: auto handling of 'end-of-basket' item for Shopper model --- .../basket_models/base_basket_model.py | 1 - .../basket_models/data/basket_dataset.py | 68 +++++-- choice_learn/basket_models/shopper.py | 182 ++++++++++++++++-- .../test_shopper_on_tripdataset.py | 123 ++++++------ 4 files changed, 267 insertions(+), 107 deletions(-) diff --git a/choice_learn/basket_models/base_basket_model.py b/choice_learn/basket_models/base_basket_model.py index 515a0093..566bfb8f 100644 --- a/choice_learn/basket_models/base_basket_model.py +++ b/choice_learn/basket_models/base_basket_model.py @@ -182,7 +182,6 @@ def compute_batch_utility( """ return - # Not clear def compute_item_likelihood( self, basket: Union[None, np.ndarray] = None, diff --git a/choice_learn/basket_models/data/basket_dataset.py b/choice_learn/basket_models/data/basket_dataset.py index f3ad1349..f3b9fcb2 100644 --- a/choice_learn/basket_models/data/basket_dataset.py +++ b/choice_learn/basket_models/data/basket_dataset.py @@ -93,7 +93,9 @@ def get_items_up_to_index(self, i: int) -> np.ndarray: class TripDataset: """Class for a dataset of trips.""" - def __init__(self, trips: list[Trip], available_items: np.ndarray) -> None: + def __init__( + self, trips: list[Trip], available_items: np.ndarray = None, prices: np.ndarray = None + ) -> None: """Initialize the dataset. Parameters @@ -112,6 +114,7 @@ def __init__(self, trips: list[Trip], available_items: np.ndarray) -> None: self.max_length = max([trip.trip_length for trip in self.trips]) self.n_samples = len(self.get_transactions()) self.available_items = available_items + self.prices = prices def __len__(self) -> int: """Return the number of trips in the dataset. @@ -308,7 +311,7 @@ def get_one_vs_all_augmented_data_from_trip_index( self, trip_index: int, ) -> tuple[np.ndarray]: - """Get augmented data from a trip index. + """Get augmented data from a trip index - following AleaCarta method. Augmented data consists in removing one item from the basket that will be used as a target from the remaining items. It is done for all items, leading to returning: @@ -386,7 +389,7 @@ def get_subbaskets_augmented_data_from_trip_index( self, trip_index: int, ) -> tuple[np.ndarray]: - """Get augmented data from a trip index. + """Get augmented data from a trip index - following Shopper method. Augmented data includes all the transactions obtained sequentially from the trip. In particular, items in the basket are shuffled and sub-baskets are built iteratively @@ -417,11 +420,11 @@ def get_subbaskets_augmented_data_from_trip_index( # Draw a random permutation of the items in the basket without the checkout item 0 # TODO at a later stage: improve by sampling several permutations here - permutation_list = list(permutations(range(length_trip - 1))) + permutation_list = list(permutations(range(length_trip))) permutation = random.sample(permutation_list, 1)[0] # nosec # Permute the basket while keeping the checkout item 0 at the end - permuted_purchases = np.array([trip.purchases[j] for j in permutation] + [0]) + permuted_purchases = np.array([trip.purchases[j] for j in permutation] + [self.n_items]) # Truncate the baskets: for each batch sample, we consider the truncation possibilities # ranging from an empty basket to the basket with all the elements except the checkout item @@ -430,7 +433,7 @@ def get_subbaskets_augmented_data_from_trip_index( padded_truncated_purchases = np.array( [ np.concatenate((permuted_purchases[:i], -1 * np.ones(self.max_length - i))) - for i in range(0, length_trip) + for i in range(0, length_trip + 1) ], dtype=int, ) @@ -447,7 +450,7 @@ def get_subbaskets_augmented_data_from_trip_index( -1 * np.ones(self.max_length - len(permuted_purchases) + i + 1), ) ) - for i in range(0, length_trip) + for i in range(0, length_trip + 1) ], dtype=int, ) @@ -458,6 +461,17 @@ def get_subbaskets_augmented_data_from_trip_index( else: # np.ndarray # Then it is directly the availability matrix assortment = trip.assortment + # end-of-basket item always available + assortment = np.concatenate([assortment, [1.0]]) + + if not (isinstance(trip.prices, np.ndarray) or isinstance(trip.prices, list)): + # Then it is the assortment ID (ie its index in self.available_items) + prices = self.prices[trip.prices] + else: # np.ndarray + # Then it is directly the availability matrix + prices = trip.prices + # end-of-basket item always 0. + prices = np.concatenate([prices, [0.0]]) # Each item is linked to a basket, the future purchases, # a store, a week, prices and an assortment @@ -465,10 +479,10 @@ def get_subbaskets_augmented_data_from_trip_index( permuted_purchases, # Items padded_truncated_purchases, # Baskets padded_future_purchases, # Future purchases - np.full(length_trip, trip.store), # Stores - np.full(length_trip, trip.week), # Weeks - np.tile(trip.prices, (length_trip, 1)), # Prices - np.tile(assortment, (length_trip, 1)), # Available items + np.full(length_trip + 1, trip.store), # Stores + np.full(length_trip + 1, trip.week), # Weeks + np.tile(prices, (length_trip + 1, 1)), # Prices + np.tile(assortment, (length_trip + 1, 1)), # Available items ) def iter_batch( @@ -509,16 +523,28 @@ def iter_batch( trip_indexes = np.random.default_rng().permutation(trip_indexes) # Initialize the buffer - buffer = ( - np.empty(0, dtype=int), # Items - np.empty((0, self.max_length), dtype=int), # Baskets - np.empty((0, self.max_length), dtype=int), # Future purchases - np.empty(0, dtype=int), # Stores - np.empty(0, dtype=int), # Weeks - np.empty((0, self.n_items), dtype=int), # Prices - np.empty((0, self.n_items), dtype=int), # Available items - ) - + if data_method == "shopper": + buffer = ( + np.empty(0, dtype=int), # Items + np.empty((0, self.max_length), dtype=int), # Baskets + np.empty((0, self.max_length), dtype=int), # Future purchases + np.empty(0, dtype=int), # Stores + np.empty(0, dtype=int), # Weeks + np.empty((0, self.n_items + 1), dtype=int), # Prices + np.empty((0, self.n_items + 1), dtype=int), # Available items + ) + elif data_method == "aleacarta": + buffer = ( + np.empty(0, dtype=int), # Items + np.empty((0, self.max_length), dtype=int), # Baskets + np.empty((0, self.max_length), dtype=int), # Future purchases + np.empty(0, dtype=int), # Stores + np.empty(0, dtype=int), # Weeks + np.empty((0, self.n_items), dtype=int), # Prices + np.empty((0, self.n_items), dtype=int), # Available items + ) + else: + raise ValueError(f"Unknown data method: {data_method}") if batch_size == -1: # Get the whole dataset in one batch for trip_index in trip_indexes: diff --git a/choice_learn/basket_models/shopper.py b/choice_learn/basket_models/shopper.py index c926cf2a..db863c9f 100644 --- a/choice_learn/basket_models/shopper.py +++ b/choice_learn/basket_models/shopper.py @@ -6,7 +6,9 @@ import numpy as np import tensorflow as tf +from ..tf_ops import softmax_with_availabilities from .base_basket_model import BaseBasketModel +from .data.basket_dataset import Trip class Shopper(BaseBasketModel): @@ -166,6 +168,14 @@ def instantiate( trainable=True, name="rho", ) + # end-of-basket rho + self.rho_eob = tf.Variable( + tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( + shape=(1, self.latent_sizes["preferences"]) + ), # Dimension for 1 item: latent_sizes["preferences"] + trainable=True, + name="rho_eob", + ) self.alpha = tf.Variable( tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( shape=(n_items, self.latent_sizes["preferences"]) @@ -173,6 +183,13 @@ def instantiate( trainable=True, name="alpha", ) + self.alpha_eob = tf.Variable( # end-of-basket alpha + tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( + shape=(1, self.latent_sizes["preferences"]) + ), # Dimension for 1 item: latent_sizes["preferences"] + trainable=True, + name="alpha_eob", + ) self.theta = tf.Variable( tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( shape=(n_stores, self.latent_sizes["preferences"]) @@ -186,7 +203,7 @@ def instantiate( self.lambda_ = tf.Variable( tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( # No lambda for the checkout item (set to 0 later) - shape=(n_items - 1,) # Dimension for 1 item: 1 + shape=(n_items,) # Dimension for 1 item: 1 ), trainable=True, name="lambda_", @@ -201,6 +218,14 @@ def instantiate( trainable=True, name="beta", ) + self.beta_eob = tf.Variable( + tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( + shape=(1, self.latent_sizes["price"]) + ), # Dimension for 1 item: latent_sizes["price"] + trainable=True, + name="beta_eob", + ) + self.gamma = tf.Variable( tf.random_normal_initializer(mean=0, stddev=1.0, seed=42)( shape=(n_stores, self.latent_sizes["price"]) @@ -218,6 +243,13 @@ def instantiate( trainable=True, name="mu", ) + self.mu_eob = tf.Variable( + tf.random_normal_initializer(mean=0, stddev=0.1, seed=42)( + shape=(1, self.latent_sizes["season"]) + ), # Dimension for 1 item: latent_sizes["season"] + trainable=True, + name="mu_eob", + ) self.delta = tf.Variable( tf.random_normal_initializer(mean=0, stddev=0.1, seed=42)( shape=(52, self.latent_sizes["season"]) @@ -237,16 +269,16 @@ def trainable_weights(self) -> list[tf.Variable]: list[tf.Variable] Latent parameters of the model """ - weights = [self.rho, self.alpha, self.theta] + weights = [self.rho, self.rho_eob, self.alpha, self.alpha_eob, self.theta] if self.item_intercept: weights.append(self.lambda_) if self.price_effects: - weights.extend([self.beta, self.gamma]) + weights.extend([self.beta, self.beta_eob, self.gamma]) if self.seasonal_effects: - weights.extend([self.mu, self.delta]) + weights.extend([self.mu, self.mu_eob, self.delta]) return weights @@ -267,6 +299,113 @@ def train_iter_method(self): """ return "shopper" + # Not clear + def compute_item_likelihood( + self, + basket: Union[None, np.ndarray] = None, + available_items: Union[None, np.ndarray] = None, + store: Union[None, int] = None, + week: Union[None, int] = None, + prices: Union[None, np.ndarray] = None, + trip: Union[None, Trip] = None, + ) -> tf.Tensor: + """Compute the likelihood of all items for a given trip. + + Take as input directly a Trip object or separately basket, available_items, + store, week and prices. + + Parameters + ---------- + basket: np.ndarray or None, optional + ID the of items already in the basket, by default None + available_items: np.ndarray or None, optional + Matrix indicating the availability (1) or not (0) of the products, + by default None + Shape must be (n_items,) + store: int or None, optional + Store id, by default None + week: int or None, optional + Week number, by default None + prices: np.ndarray or None, optional + Prices of all the items in the dataset, by default None + Shape must be (n_items,) + trip: Trip or None, optional + Trip object containing basket, available_items, store, + week and prices, by default None + + Returns + ------- + likelihood: tf.Tensor + Likelihood of all items for a given trip + Shape must be (n_items,) + """ + if trip is None: + # Trip not provided as an argument + # Then basket, available_items, store, week and prices must be provided + if ( + basket is None + or available_items is None + or store is None + or week is None + or prices is None + ): + raise ValueError( + "If trip is None, then basket, available_items, store, week, and " + "prices must be provided as arguments." + ) + + else: + # Trip directly provided as an argument + if isinstance(trip.assortment, int): + # Then it is the assortment ID (ie its index in the attribute + # available_items of the TripDataset), but we do not have the + # the TripDataset as input here + raise ValueError( + "The assortment ID is not enough to compute the likelihood. " + "Please provide the availability matrix directly (array of shape (n_items,) " + "indicating the availability (1) or not (0) of the products)." + ) + + return self.compute_item_likelihood( + basket=trip.purchases, + available_items=trip.assortment, + store=trip.store, + week=trip.week, + prices=trip.prices, + trip=None, + ) + + # Prevent unintended side effects from in-place modifications + available_items_copy = available_items.copy() + for basket_item in basket: + if basket_item != -1: + available_items_copy[basket_item] = 0.0 + + if len(prices) == self.n_items: + prices = np.concatenate([prices, [0.0]], axis=0) + if len(available_items_copy) == self.n_items: + available_items_copy = np.concatenate([available_items_copy, [0.0]], axis=0) + # Compute the utility of all the items + all_utilities = self.compute_batch_utility( + # All items + item_batch=np.arange(self.n_items + 1), + # For each item: same basket / store / week / prices / available items + basket_batch=np.array([basket for _ in range(self.n_items + 1)]), + store_batch=np.array([store for _ in range(self.n_items + 1)]), + week_batch=np.array([week for _ in range(self.n_items + 1)]), + price_batch=prices, + available_item_batch=np.array([available_items_copy for _ in range(self.n_items + 1)]), + ) + + # Softmax on the utilities + return softmax_with_availabilities( + items_logit_by_choice=all_utilities, # Shape: (n_items,) + available_items_by_choice=available_items_copy, # Shape: (n_items,) + axis=-1, + normalize_exit=False, + eps=None, + ) + def thinking_ahead( self, item_batch: Union[np.ndarray, tf.Tensor], @@ -320,7 +459,7 @@ def thinking_ahead( # TODO: avoid a for loop on ragged_basket_batch at a later stage for idx in tf.range(ragged_basket_batch.shape[0]): basket = tf.gather(ragged_basket_batch, idx) - if len(basket) != 0 and basket[-1] == 0: + if len(basket) != 0 and basket[-1] == self.n_items: # No thinking ahead when the basket ends already with the checkout item 0 total_next_step_utilities = tf.tensor_scatter_nd_update( tensor=total_next_step_utilities, indices=[[idx]], updates=[0] @@ -330,7 +469,7 @@ def thinking_ahead( # Basket with the hypothetical current item next_basket = tf.concat([basket, [item_batch[idx]]], axis=0) # Get the list of available items based on the availability matrix - item_ids = tf.range(self.n_items) + item_ids = tf.range(self.n_items + 1) available_mask = tf.equal(available_item_batch[idx], 1) assortment = tf.boolean_mask(item_ids, available_mask) hypothetical_next_purchases = tf.boolean_mask( @@ -349,13 +488,13 @@ def thinking_ahead( # Compute the dot product along the last dimension between the embeddings # of the given store's theta and alpha of all the items hypothetical_store_preferences = tf.reduce_sum( - theta_store[idx] * self.alpha, axis=1 + theta_store[idx] * tf.concat([self.alpha, self.alpha_eob], axis=0), axis=1 ) if self.item_intercept: # Manually enforce the lambda of the checkout item to be 0 # (equivalent to translating the lambda values) - hypothetical_item_intercept = tf.concat([[0.0], self.lambda_], axis=0) + hypothetical_item_intercept = tf.concat([self.lambda_, [0.0]], axis=0) else: hypothetical_item_intercept = tf.zeros_like(hypothetical_store_preferences) @@ -365,7 +504,10 @@ def thinking_ahead( # Compute the dot product along the last dimension between # the embeddings of the given store's gamma and beta # of all the items - * tf.reduce_sum(gamma_store[idx] * self.beta, axis=1) + * tf.reduce_sum( + gamma_store[idx] * tf.concat([self.beta, self.beta_eob], axis=0), + axis=1, + ) * tf.math.log(price_batch[idx] + self.epsilon_price) ) else: @@ -375,7 +517,7 @@ def thinking_ahead( # Compute the dot product along the last dimension between the embeddings # of delta of the given week and mu of all the items hypothetical_seasonal_effects = tf.reduce_sum( - delta_week[idx] * self.mu, axis=1 + delta_week[idx] * tf.concat([self.mu, self.mu_eob], axis=0), axis=1 ) else: hypothetical_seasonal_effects = tf.zeros_like( @@ -404,12 +546,13 @@ def thinking_ahead( for inner_idx in tf.range(len(hypothetical_next_purchases)): next_item_id = tf.gather(hypothetical_next_purchases, inner_idx) rho_next_item = tf.gather( - self.rho, indices=next_item_id + tf.concat([self.rho, self.rho_eob], axis=0), indices=next_item_id ) # Shape: (latent_size,) # Gather the embeddings using a tensor of indices # (before ensure that indices are integers) next_alpha_by_basket = tf.gather( - self.alpha, indices=tf.cast(next_basket, dtype=tf.int32) + tf.concat([self.alpha, self.alpha_eob], axis=0), + indices=tf.cast(next_basket, dtype=tf.int32), ) # Shape: (len(next_basket), latent_size) # Divide the sum of alpha embeddings by the number of items # in the basket of the next step (always > 0) @@ -483,20 +626,20 @@ def compute_batch_utility( available_item_batch = tf.cast(available_item_batch, dtype=tf.int32) theta_store = tf.gather(self.theta, indices=store_batch) - alpha_item = tf.gather(self.alpha, indices=item_batch) + alpha_item = tf.gather(tf.concat([self.alpha, self.alpha_eob], axis=0), indices=item_batch) # Compute the dot product along the last dimension store_preferences = tf.reduce_sum(theta_store * alpha_item, axis=1) if self.item_intercept: # Manually enforce the lambda of the checkout item to be 0 # (equivalent to translating the lambda values) - item_intercept = tf.gather(tf.concat([[0.0], self.lambda_], axis=0), indices=item_batch) + item_intercept = tf.gather(tf.concat([self.lambda_, [0.0]], axis=0), indices=item_batch) else: item_intercept = tf.zeros_like(store_preferences) if self.price_effects: gamma_store = tf.gather(self.gamma, indices=store_batch) - beta_item = tf.gather(self.beta, indices=item_batch) + beta_item = tf.gather(tf.concat([self.beta, self.beta_eob], axis=0), indices=item_batch) # Add epsilon to avoid NaN values (log(0)) price_effects = ( -1 @@ -510,7 +653,7 @@ def compute_batch_utility( if self.seasonal_effects: delta_week = tf.gather(self.delta, indices=week_batch) - mu_item = tf.gather(self.mu, indices=item_batch) + mu_item = tf.gather(tf.concat([self.mu, self.mu_eob], axis=0), indices=item_batch) # Compute the dot product along the last dimension seasonal_effects = tf.reduce_sum(delta_week * mu_item, axis=1) else: @@ -558,7 +701,7 @@ def compute_batch_utility( # Compute the sum of the alpha embeddings for each basket alpha_sum = tf.reduce_sum(alpha_by_basket, axis=1) - rho_item = tf.gather(self.rho, indices=item_batch) + rho_item = tf.gather(tf.concat([self.rho, self.rho_eob], axis=0), indices=item_batch) # Divide each sum of alpha embeddings by the number of items in the corresponding basket # Avoid NaN values (division by 0) @@ -634,7 +777,7 @@ def get_negative_samples( next_item = tf.cast(tf.convert_to_tensor(next_item), dtype=tf.int32) # Get the list of available items based on the availability matrix - item_ids = tf.range(self.n_items) + item_ids = tf.range(self.n_items + 1) available_mask = tf.equal(available_items, 1) assortment = tf.boolean_mask(item_ids, available_mask) @@ -726,6 +869,9 @@ def compute_batch_loss( item_batch = tf.cast(item_batch, dtype=tf.int32) # Negative sampling + print(batch_size, available_item_batch.shape, item_batch.shape) + print(basket_batch.shape, future_batch.shape, item_batch.shape) + print(available_item_batch[batch_size - 1]) negative_samples = tf.reshape( tf.transpose( tf.reshape( diff --git a/tests/integration_tests/basket_models/test_shopper_on_tripdataset.py b/tests/integration_tests/basket_models/test_shopper_on_tripdataset.py index 76505264..9e365eda 100644 --- a/tests/integration_tests/basket_models/test_shopper_on_tripdataset.py +++ b/tests/integration_tests/basket_models/test_shopper_on_tripdataset.py @@ -79,121 +79,123 @@ # with store, week and prices fixed trip_list_2 = [ Trip( - purchases=[0], # Empty basket + purchases=[], # Empty basket store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[1, 0], + purchases=[0], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[2, 0], + purchases=[1], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[3, 0], + purchases=[2], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[1, 2, 0], + purchases=[0, 1], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[1, 3, 0], + purchases=[0, 2], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[2, 1, 0], + purchases=[1, 0], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[2, 3, 0], + purchases=[1, 2], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[3, 1, 0], + purchases=[2, 0], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[3, 2, 0], + purchases=[2, 1], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[1, 2, 3, 0], + purchases=[0, 1, 2], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[1, 3, 2, 0], + purchases=[0, 2, 1], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[2, 1, 3, 0], + purchases=[1, 0, 2], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[2, 3, 1, 0], + purchases=[1, 2, 0], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[3, 1, 2, 0], + purchases=[2, 0, 1], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), Trip( - purchases=[3, 2, 1, 0], + purchases=[2, 1, 0], store=0, week=0, - prices=[1, 100, 170, 110, 150], + prices=0, assortment=0, ), ] # One more item available in the assortment to be able to use negative sampling -available_items_2 = np.expand_dims(np.ones(5), axis=0) -trip_dataset_2 = TripDataset(trips=trip_list_2, available_items=available_items_2) +available_items_2 = np.expand_dims(np.ones(3), axis=0) +trip_dataset_2 = TripDataset( + trips=trip_list_2, available_items=available_items_2, prices=np.array([[10, 20, 30]]) +) n_items_2 = trip_dataset_2.n_items n_stores_2 = trip_dataset_2.n_stores @@ -210,7 +212,7 @@ def test_item_probabilities_sum_to_1() -> None: n_items=n_items_1, n_stores=n_stores_1, ) - model.fit(trip_dataset=trip_dataset_1, val_dataset=trip_dataset_1) + # model.fit(trip_dataset=trip_dataset_1, val_dataset=trip_dataset_1) for trip in trip_dataset_1.trips: # For a given trip, check at each step that the sum of the probabilities for each @@ -244,7 +246,7 @@ def test_ordered_basket_probabilities_sum_to_1() -> None: n_negative_samples=1, ) model.instantiate(n_items=n_items_2, n_stores=n_stores_2) - model.fit(trip_dataset=trip_dataset_2) + # model.fit(trip_dataset=trip_dataset_2) # For a basket {1, 2, 3, 0} of size 3: # compute_ordered_basket_likelihood = 1/3 * 1/3 * 1/2 * 1/1 = 1/18 @@ -255,37 +257,24 @@ def test_ordered_basket_probabilities_sum_to_1() -> None: # - The checkout item must be available # - The checkout item must not be the only item available # (because the proba of an empty basket is 0 and cannot sum to 1) - list_availability_matrices = [ - np.array([1, 1, 1, 1, 1]), - np.array([1, 0, 1, 1, 1]), - np.array([1, 1, 0, 1, 1]), - np.array([1, 1, 1, 0, 1]), - np.array([1, 1, 1, 1, 0]), - np.array([1, 0, 0, 0, 1]), - np.array([1, 0, 0, 1, 0]), - np.array([1, 0, 1, 0, 0]), - np.array([1, 1, 0, 0, 0]), - ] - for availability_matrix in list_availability_matrices: - # Try with different availability matrices - assert ( - np.abs( - np.sum( - [ - model.compute_ordered_basket_likelihood( - basket=trip.purchases, - available_items=availability_matrix, - store=trip.store, - week=trip.week, - prices=trip.prices, - ) - for trip in trip_dataset_2.trips - ] - ) - - 1.0 + assert ( + np.abs( + np.sum( + [ + model.compute_ordered_basket_likelihood( + basket=trip.purchases + [trip_dataset_2.n_items], + available_items=np.ones(trip_dataset_2.n_items + 1), + store=trip.store, + week=trip.week, + prices=np.array([10, 20, 30, 0]), + ) + for trip in trip_dataset_2.trips + ] ) - < 2e-2 + - 1.0 ) + < 2e-2 + ) def test_thinking_ahead() -> None: @@ -303,7 +292,7 @@ def test_thinking_ahead() -> None: store_batch=np.array([0] * batch_size), week_batch=np.array([0] * batch_size), price_batch=np.random.uniform(1, 10, batch_size), - available_item_batch=np.array([np.ones(n_items_1)] * batch_size), + available_item_batch=np.array([np.ones(n_items_1 + 1)] * batch_size), ) @@ -451,7 +440,7 @@ def test_get_negative_samples() -> None: with pytest.raises(tf.errors.InvalidArgumentError): model.get_negative_samples( - available_items=np.ones(n_items_1), + available_items=np.ones(n_items_1 + 1), purchased_items=np.array([1, 2]), future_purchases=np.array([3, 0]), next_item=0, From 9b956a1b10b9524337c25ef071e2a4fb1494814a Mon Sep 17 00:00:00 2001 From: VincentAuriau Date: Sat, 27 Sep 2025 13:29:21 +0200 Subject: [PATCH 2/4] FIX: import in notebook --- notebooks/basket_models/shopper.ipynb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/notebooks/basket_models/shopper.ipynb b/notebooks/basket_models/shopper.ipynb index fba85f70..acaf57e8 100644 --- a/notebooks/basket_models/shopper.ipynb +++ b/notebooks/basket_models/shopper.ipynb @@ -54,7 +54,8 @@ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", - "from choice_learn.basket_models import Shopper, Trip" + "from choice_learn.basket_models import Shopper\n", + "from choice_learn.basket_models.data import Trip" ] }, { @@ -343,11 +344,18 @@ "\n", "print(f\"Likelihood for ordered basket {[3, 1, 0]}: {basket_ordered_likelihood:.4f}.\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "tf_env", + "display_name": "basics", "language": "python", "name": "python3" }, @@ -361,7 +369,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.12.11" } }, "nbformat": 4, From f3b1a0f9674632b2d6e3c97b7c1d763537e3080e Mon Sep 17 00:00:00 2001 From: Michael Teboul Date: Tue, 30 Sep 2025 09:55:37 +0200 Subject: [PATCH 3/4] Add of variable user_id in Trip class --- .../basket_models/data/basket_dataset.py | 108 +++++++++--------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/choice_learn/basket_models/data/basket_dataset.py b/choice_learn/basket_models/data/basket_dataset.py index f3b9fcb2..173be5fc 100644 --- a/choice_learn/basket_models/data/basket_dataset.py +++ b/choice_learn/basket_models/data/basket_dataset.py @@ -25,6 +25,7 @@ def __init__( assortment: Union[int, np.ndarray], store: int = 0, week: int = 0, + user_id: int = 0, ) -> None: """Initialize the trip. @@ -57,6 +58,7 @@ def __init__( self.week = week self.prices = prices self.assortment = assortment + self.user_id = user_id self.trip_length = len(purchases) @@ -69,7 +71,7 @@ def __str__(self) -> str: Representation of the trip """ desc = f"Trip with {self.trip_length} purchases {self.purchases}" - desc += f" at store {self.store} in week {self.week}" + desc += f" at store {self.store} in week {self.week} by user {self.user_id}" desc += f" with prices {self.prices} and assortment {self.assortment}" return desc @@ -93,9 +95,7 @@ def get_items_up_to_index(self, i: int) -> np.ndarray: class TripDataset: """Class for a dataset of trips.""" - def __init__( - self, trips: list[Trip], available_items: np.ndarray = None, prices: np.ndarray = None - ) -> None: + def __init__(self, trips: list[Trip], available_items: np.ndarray) -> None: """Initialize the dataset. Parameters @@ -114,8 +114,7 @@ def __init__( self.max_length = max([trip.trip_length for trip in self.trips]) self.n_samples = len(self.get_transactions()) self.available_items = available_items - self.prices = prices - + def __len__(self) -> int: """Return the number of trips in the dataset. @@ -203,7 +202,7 @@ def get_trip(self, index: int) -> Trip: def get_transactions(self) -> np.ndarray: """Return the transactions of the TripDataset. - One transaction is a triplet (store, trip, item). + One transaction is a quadruplet (store, trip, item, user_id). Returns ------- @@ -217,7 +216,7 @@ def get_transactions(self) -> np.ndarray: trans_id = 0 for i, trip in enumerate(self.trips): for item in trip.purchases: - transactions[trans_id] = (trip.store, i, item) + transactions[trans_id] = (trip.store, i, item, trip.user_id) trans_id += 1 return transactions @@ -273,7 +272,17 @@ def get_all_prices(self) -> np.ndarray: List of price arrays in the dataset """ return np.array([self.trips[i].prices for i in range(len(self))]) + + def get_all_users(self) -> np.ndarray: + """Return the list of all users in the dataset. + Returns + ------- + np.ndarray + List of users in the dataset + """ + return np.array(list({self.trips[i].user_id for i in range(len(self))})) + @property def n_items(self) -> int: """Return the number of items available in the dataset. @@ -295,6 +304,17 @@ def n_stores(self) -> int: Number of stores in the dataset """ return len(self.get_all_stores()) + + @property + def n_users(self) -> int: + """Return the number of users in the dataset. + + Returns + ------- + int + Number of users in the dataset + """ + return len(self.get_all_users()) @property def n_assortments(self) -> int: @@ -311,7 +331,7 @@ def get_one_vs_all_augmented_data_from_trip_index( self, trip_index: int, ) -> tuple[np.ndarray]: - """Get augmented data from a trip index - following AleaCarta method. + """Get augmented data from a trip index. Augmented data consists in removing one item from the basket that will be used as a target from the remaining items. It is done for all items, leading to returning: @@ -321,6 +341,7 @@ def get_one_vs_all_augmented_data_from_trip_index( - weeks, - prices, - available items. + - user_id Parameters ---------- @@ -374,6 +395,7 @@ def get_one_vs_all_augmented_data_from_trip_index( # Then it is directly the availability matrix prices = trip.prices + # Each item is linked to a basket, a store, a week, prices and an assortment return ( permuted_purchases, # Items @@ -383,13 +405,14 @@ def get_one_vs_all_augmented_data_from_trip_index( np.full(length_trip, trip.week), # Weeks np.tile(prices, (length_trip, 1)), # Prices np.tile(assortment, (length_trip, 1)), # Available items + np.full(length_trip, trip.user_id) # User IDs ) def get_subbaskets_augmented_data_from_trip_index( self, trip_index: int, ) -> tuple[np.ndarray]: - """Get augmented data from a trip index - following Shopper method. + """Get augmented data from a trip index. Augmented data includes all the transactions obtained sequentially from the trip. In particular, items in the basket are shuffled and sub-baskets are built iteratively @@ -420,11 +443,11 @@ def get_subbaskets_augmented_data_from_trip_index( # Draw a random permutation of the items in the basket without the checkout item 0 # TODO at a later stage: improve by sampling several permutations here - permutation_list = list(permutations(range(length_trip))) + permutation_list = list(permutations(range(length_trip - 1))) permutation = random.sample(permutation_list, 1)[0] # nosec # Permute the basket while keeping the checkout item 0 at the end - permuted_purchases = np.array([trip.purchases[j] for j in permutation] + [self.n_items]) + permuted_purchases = np.array([trip.purchases[j] for j in permutation] + [0]) # Truncate the baskets: for each batch sample, we consider the truncation possibilities # ranging from an empty basket to the basket with all the elements except the checkout item @@ -433,7 +456,7 @@ def get_subbaskets_augmented_data_from_trip_index( padded_truncated_purchases = np.array( [ np.concatenate((permuted_purchases[:i], -1 * np.ones(self.max_length - i))) - for i in range(0, length_trip + 1) + for i in range(0, length_trip) ], dtype=int, ) @@ -450,7 +473,7 @@ def get_subbaskets_augmented_data_from_trip_index( -1 * np.ones(self.max_length - len(permuted_purchases) + i + 1), ) ) - for i in range(0, length_trip + 1) + for i in range(0, length_trip) ], dtype=int, ) @@ -461,17 +484,6 @@ def get_subbaskets_augmented_data_from_trip_index( else: # np.ndarray # Then it is directly the availability matrix assortment = trip.assortment - # end-of-basket item always available - assortment = np.concatenate([assortment, [1.0]]) - - if not (isinstance(trip.prices, np.ndarray) or isinstance(trip.prices, list)): - # Then it is the assortment ID (ie its index in self.available_items) - prices = self.prices[trip.prices] - else: # np.ndarray - # Then it is directly the availability matrix - prices = trip.prices - # end-of-basket item always 0. - prices = np.concatenate([prices, [0.0]]) # Each item is linked to a basket, the future purchases, # a store, a week, prices and an assortment @@ -479,10 +491,11 @@ def get_subbaskets_augmented_data_from_trip_index( permuted_purchases, # Items padded_truncated_purchases, # Baskets padded_future_purchases, # Future purchases - np.full(length_trip + 1, trip.store), # Stores - np.full(length_trip + 1, trip.week), # Weeks - np.tile(prices, (length_trip + 1, 1)), # Prices - np.tile(assortment, (length_trip + 1, 1)), # Available items + np.full(length_trip, trip.store), # Stores + np.full(length_trip, trip.week), # Weeks + np.tile(trip.prices, (length_trip, 1)), # Prices + np.tile(assortment, (length_trip, 1)), # Available items + np.full(length_trip, trip.user_id), # User IDs ) def iter_batch( @@ -510,8 +523,8 @@ def iter_batch( ------ tuple[np.ndarray] For each item in the batch: item, basket, future purchases, - store, week, prices, available items - Length must 7 + store, week, prices, available items, user_id + Length must be 8 """ # Get trip indexes num_trips = len(self) @@ -523,28 +536,17 @@ def iter_batch( trip_indexes = np.random.default_rng().permutation(trip_indexes) # Initialize the buffer - if data_method == "shopper": - buffer = ( - np.empty(0, dtype=int), # Items - np.empty((0, self.max_length), dtype=int), # Baskets - np.empty((0, self.max_length), dtype=int), # Future purchases - np.empty(0, dtype=int), # Stores - np.empty(0, dtype=int), # Weeks - np.empty((0, self.n_items + 1), dtype=int), # Prices - np.empty((0, self.n_items + 1), dtype=int), # Available items - ) - elif data_method == "aleacarta": - buffer = ( - np.empty(0, dtype=int), # Items - np.empty((0, self.max_length), dtype=int), # Baskets - np.empty((0, self.max_length), dtype=int), # Future purchases - np.empty(0, dtype=int), # Stores - np.empty(0, dtype=int), # Weeks - np.empty((0, self.n_items), dtype=int), # Prices - np.empty((0, self.n_items), dtype=int), # Available items - ) - else: - raise ValueError(f"Unknown data method: {data_method}") + buffer = ( + np.empty(0, dtype=int), # Items + np.empty((0, self.max_length), dtype=int), # Baskets + np.empty((0, self.max_length), dtype=int), # Future purchases + np.empty(0, dtype=int), # Stores + np.empty(0, dtype=int), # Weeks + np.empty((0, self.n_items), dtype=int), # Prices + np.empty((0, self.n_items), dtype=int), # Available items + np.empty(0, dtype=int), # User IDs + ) + if batch_size == -1: # Get the whole dataset in one batch for trip_index in trip_indexes: From 860663c225783e37fec603a184c07b1a52bdc38f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 08:10:57 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- choice_learn/basket_models/data/basket_dataset.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/choice_learn/basket_models/data/basket_dataset.py b/choice_learn/basket_models/data/basket_dataset.py index 173be5fc..d3760214 100644 --- a/choice_learn/basket_models/data/basket_dataset.py +++ b/choice_learn/basket_models/data/basket_dataset.py @@ -114,7 +114,7 @@ def __init__(self, trips: list[Trip], available_items: np.ndarray) -> None: self.max_length = max([trip.trip_length for trip in self.trips]) self.n_samples = len(self.get_transactions()) self.available_items = available_items - + def __len__(self) -> int: """Return the number of trips in the dataset. @@ -272,7 +272,7 @@ def get_all_prices(self) -> np.ndarray: List of price arrays in the dataset """ return np.array([self.trips[i].prices for i in range(len(self))]) - + def get_all_users(self) -> np.ndarray: """Return the list of all users in the dataset. @@ -282,7 +282,7 @@ def get_all_users(self) -> np.ndarray: List of users in the dataset """ return np.array(list({self.trips[i].user_id for i in range(len(self))})) - + @property def n_items(self) -> int: """Return the number of items available in the dataset. @@ -304,7 +304,7 @@ def n_stores(self) -> int: Number of stores in the dataset """ return len(self.get_all_stores()) - + @property def n_users(self) -> int: """Return the number of users in the dataset. @@ -395,7 +395,6 @@ def get_one_vs_all_augmented_data_from_trip_index( # Then it is directly the availability matrix prices = trip.prices - # Each item is linked to a basket, a store, a week, prices and an assortment return ( permuted_purchases, # Items @@ -405,7 +404,7 @@ def get_one_vs_all_augmented_data_from_trip_index( np.full(length_trip, trip.week), # Weeks np.tile(prices, (length_trip, 1)), # Prices np.tile(assortment, (length_trip, 1)), # Available items - np.full(length_trip, trip.user_id) # User IDs + np.full(length_trip, trip.user_id), # User IDs ) def get_subbaskets_augmented_data_from_trip_index(