diff --git a/seqio/dataset_providers_test.py b/seqio/dataset_providers_test.py index 3e1b2cac..3b61b991 100644 --- a/seqio/dataset_providers_test.py +++ b/seqio/dataset_providers_test.py @@ -1384,6 +1384,7 @@ def _constant_feature_preprocessor(unused_x, val): res = sum(int(item["feature"][0]) for item in mix_ds.as_numpy_iterator()) self.assertEqual(res, 2481) + def test_copy_pretokenized(self): @utils.map_over_dataset def _constant_preprocessor(unused_x, val):