From 914987e3e4e84533d9b28d85cb04afdf06ff3a51 Mon Sep 17 00:00:00 2001 From: Josh Purtell Date: Mon, 26 Feb 2024 17:54:35 -0500 Subject: [PATCH] add fallback and hardfail logic --- ragatouille/RAGTrainer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ragatouille/RAGTrainer.py b/ragatouille/RAGTrainer.py index 35d5c4e..a9d8296 100644 --- a/ragatouille/RAGTrainer.py +++ b/ragatouille/RAGTrainer.py @@ -155,6 +155,22 @@ def prepare_training_data( mine_hard_negatives=mine_hard_negatives, hard_negative_minimum_rank=hard_negative_minimum_rank, ) + if len(self.data_processor.training_triplets) == 0: + if mine_hard_negatives: + print("Warning: No training triplets were generated with setting mine_hard_negatives=='True'. This may be due to the data being too small or the hard negative miner not being able to find enough hard negatives.") + self.data_processor.process_raw_data( + data_type=data_type, + raw_data=raw_data, + export=True, + data_dir=data_out_path, + num_new_negatives=num_new_negatives, + positive_label=positive_label, + negative_label=negative_label, + mine_hard_negatives=False, + hard_negative_minimum_rank=hard_negative_minimum_rank, + ) + else: + raise ValueError("No training triplets were generated.") self.training_triplets = self.data_processor.training_triplets