File tree Expand file tree Collapse file tree 2 files changed +6
-1
lines changed Expand file tree Collapse file tree 2 files changed +6
-1
lines changed Original file line number Diff line number Diff line change @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
4
4
The format is based on [ Keep a Changelog] ( https://keepachangelog.com/en/1.0.0/ ) ,
5
5
and this project adheres to [ Semantic Versioning] ( https://semver.org/spec/v2.0.0.html ) .
6
6
7
+ ## [ 0.10.1]
8
+
9
+ ### Fixed
10
+ - [ #226 ] : Fix the word indexes when there are special tokens
11
+
7
12
## [ 0.10.0]
8
13
9
14
### Changed
Original file line number Diff line number Diff line change 2
2
authors = [" Anthony MOI <m.anthony.moi@gmail.com>" ]
3
3
edition = " 2018"
4
4
name = " tokenizers"
5
- version = " 0.10.0 "
5
+ version = " 0.10.1 "
6
6
homepage = " https://github.com/huggingface/tokenizers"
7
7
repository = " https://github.com/huggingface/tokenizers"
8
8
documentation = " https://docs.rs/tokenizers/"
You can’t perform that action at this time.
0 commit comments