Add Hindi tokenizer to API (#696)

NaturalNode · Aug 26, 2023 · 4bcf7f4 · 4bcf7f4
1 parent bd284d3
commit 4bcf7f4
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 1 deletion.
diff --git a/lib/natural/tokenizers/index.d.ts b/lib/natural/tokenizers/index.d.ts
@@ -83,6 +83,10 @@ export class AggressiveTokenizerId extends Tokenizer {
   tokenize (text: string): string[]
 }
 
+export class AggressiveTokenizerHi extends Tokenizer {
+  tokenize (text: string): string[]
+}
+
 export class AggressiveTokenizer extends Tokenizer {
   tokenize (text: string): string[]
 }

diff --git a/lib/natural/tokenizers/index.js b/lib/natural/tokenizers/index.js
@@ -35,6 +35,7 @@ exports.AggressiveTokenizerNo = require('./aggressive_tokenizer_no')
 exports.AggressiveTokenizerSv = require('./aggressive_tokenizer_sv')
 exports.AggressiveTokenizerVi = require('./aggressive_tokenizer_vi')
 exports.AggressiveTokenizerId = require('./aggressive_tokenizer_id')
+exports.AggressiveTokenizerHi = require('./aggressive_tokenizer_hi')
 exports.AggressiveTokenizer = require('./aggressive_tokenizer')
 exports.CaseTokenizer = require('./tokenizer_case')
 exports.RegexpTokenizer = require('./regexp_tokenizer').RegexpTokenizer

diff --git a/package.json b/package.json
@@ -74,7 +74,6 @@
     "Porter stemmer",
     "Lancaster stemmer",
     "tokenizer",
-    "tokenizer hindi",
     "bigram",
     "trigram",
     "quadgram",