Bump version and update Readme

huggingface · Dec 20, 2019 · 2266960 · 2266960
1 parent f2b9c30
commit 2266960
Show file tree

Hide file tree

Showing 6 changed files with 9 additions and 9 deletions.
diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.0.7"
+version = "0.0.8"
 authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
 edition = "2018"
 

diff --git a/bindings/python/README.md b/bindings/python/README.md
@@ -61,7 +61,7 @@ bpe = models.BPE.from_files(vocab, merges)
 tokenizer = Tokenizer(bpe)
 
 # Customize pre-tokenization and decoding
-tokenizer.with_pre_tokenizer(pre_tokenizers.ByteLevel.new())
+tokenizer.with_pre_tokenizer(pre_tokenizers.ByteLevel.new(True))
 tokenizer.with_decoder(decoders.ByteLevel.new())
 
 # And then encode:
@@ -85,7 +85,7 @@ from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers
 tokenizer = Tokenizer(models.BPE.empty())
 
 # Customize pre-tokenization and decoding
-tokenizer.with_pre_tokenizer(pre_tokenizers.ByteLevel.new())
+tokenizer.with_pre_tokenizer(pre_tokenizers.ByteLevel.new(True))
 tokenizer.with_decoder(decoders.ByteLevel.new())
 
 # And then train

diff --git a/bindings/python/setup.py b/bindings/python/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="tokenizers",
-    version="0.0.7",
+    version="0.0.8",
     description="Fast and Customizable Tokenizers",
     long_description=open("README.md", "r", encoding="utf-8").read(),
     long_description_content_type="text/markdown",

diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock
diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml
@@ -2,7 +2,7 @@
 authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
 edition = "2018"
 name = "tokenizers-lib"
-version = "0.0.7"
+version = "0.0.8"
 [[bin]]
 name = "cli"
 path = "src/cli.rs"