Rewrite of sentence tokenizers

NaturalNode · Aug 8, 2024 · 31ed8f5 · 31ed8f5
1 parent ed002eb
commit 31ed8f5
Show file tree

Hide file tree

Showing 9 changed files with 364 additions and 1,950 deletions.
diff --git a/lib/natural/tokenizers/index.d.ts b/lib/natural/tokenizers/index.d.ts
@@ -136,9 +136,6 @@ export class TokenizerJa extends Tokenizer {
 }
 
 export class SentenceTokenizer extends Tokenizer {
-  tokenize (text: string): string[]
-}
-
-export class SentenceTokenizerNew extends Tokenizer {
+  constructor(abbreviations: string[], sentenceDemarkers?: string[])
   tokenize (text: string): string[]
 }
diff --git a/lib/natural/tokenizers/index.js b/lib/natural/tokenizers/index.js
@@ -46,4 +46,4 @@ exports.WordPunctTokenizer = require('./regexp_tokenizer').WordPunctTokenizer
 exports.TreebankWordTokenizer = require('./treebank_word_tokenizer')
 exports.TokenizerJa = require('./tokenizer_ja')
 exports.SentenceTokenizer = require('./sentence_tokenizer')
-exports.SentenceTokenizerNew = require('./sentence_tokenizer_parser')
+exports.SentenceTokenizerNew = require('./sentence_tokenizer')