bloom-oscar.yaml

# Process config example for Oscar used in BLOOM

# global parameters
project_name: 'bloom_oscar'
dataset_path: '/path/to/your/dataset'  # path to your dataset directory or file
np: 4  # number of subprocess to process your dataset

export_path: '/path/to/result/dataset.jsonl'

# process schedule
# a list of several process operators with their arguments
process:
  # filter English corpus
  - language_id_score_filter:
      lang: en
      min_score: 0.8

  # basic process for regular English text
  - whitespace_normalization_mapper:
  - punctuation_normalization_mapper:
  - fix_unicode_mapper:
  - remove_words_with_incorrect_substrings_mapper:
  - remove_long_words_mapper:
      max_len: 25

  # basic filter rules for regular English text
  - words_num_filter:
      min_num: 20
      max_num: 100000
  - character_repetition_filter:
      rep_len: 10
      min_ratio: 0.0
      max_ratio: 0.106
  - word_repetition_filter:
      rep_len: 5
      min_ratio: 0.0
      max_ratio: 0.19
  - special_characters_filter:
      min_ratio: 0.0
      max_ratio: 0.4
  - stopwords_filter:
      lang: en
      min_ratio: 0.3
  - flagged_words_filter:
      lang: en
      max_ratio: 0.01
  - perplexity_filter:
      lang: en
      max_ppl: 1500

  # basic deduplication rules for regular English text
  - document_simhash_deduplicator:
      tokenization: space
      window_size: 6
      lowercase: true
      ignore_pattern: '\p{P}'
      num_blocks: 6
      hamming_distance: 4