Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

基于外挂文件屏蔽词典中的一个词 #115

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

基于外挂文件屏蔽词典中的一个词 #115

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 62 additions & 30 deletions src/main/java/com/huaban/analysis/jieba/WordDictionary.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,25 +169,37 @@ public void loadUserDict(Path userDict, Charset charset) {
BufferedReader br = Files.newBufferedReader(userDict, charset);
long s = System.currentTimeMillis();
int count = 0;
while (br.ready()) {
String line = br.readLine();
String[] tokens = line.split("[\t ]+");

if (tokens.length < 1) {
// Ignore empty line
continue;
if (userDict.toString().contains("shield")) {
while (br.ready()) {
String line = br.readLine();
shieldWord(line);
count++;
}
System.out.println(String.format(Locale.getDefault(), "user dict %s for shield load finished, tot words:%d, time elapsed:%dms", userDict.toString(), count, System.currentTimeMillis() - s));
} else {
while (br.ready()) {
String line = br.readLine();
String[] tokens = line.split("[\t ]+");

String word = tokens[0];

double freq = 3.0d;
if (tokens.length == 2)
freq = Double.valueOf(tokens[1]);
word = addWord(word);
freqs.put(word, Math.log(freq / total));
count++;
if (tokens.length < 1) {
// Ignore empty line
continue;
}


String word = tokens[0];


double freq = 3.0d;
if (tokens.length == 2)
freq = Double.valueOf(tokens[1]);
word = addWord(word);
freqs.put(word, Math.log(freq / total));
count++;
}
System.out.println(String.format(Locale.getDefault(), "user dict %s load finished, tot words:%d, time elapsed:%dms", userDict.toString(), count, System.currentTimeMillis() - s));
}
System.out.println(String.format(Locale.getDefault(), "user dict %s load finished, tot words:%d, time elapsed:%dms", userDict.toString(), count, System.currentTimeMillis() - s));
br.close();
}
catch (IOException e) {
Expand All @@ -202,31 +214,51 @@ public void loadUserDict(String userDictPath, Charset charset) {

long s = System.currentTimeMillis();
int count = 0;
while (br.ready()) {
String line = br.readLine();
String[] tokens = line.split("[\t ]+");

if (tokens.length < 1) {
// Ignore empty line
continue;
if (userDictPath.contains("shield")) {
while (br.ready()) {
String line = br.readLine();
shieldWord(line);
count++;
}
System.out.println(String.format(Locale.getDefault(), "user dict %s for shield load finished, tot words:%d, time elapsed:%dms", userDictPath, count, System.currentTimeMillis() - s));
} else {
while (br.ready()) {
String line = br.readLine();
String[] tokens = line.split("[\t ]+");

String word = tokens[0];

double freq = 3.0d;
if (tokens.length == 2)
freq = Double.valueOf(tokens[1]);
word = addWord(word);
freqs.put(word, Math.log(freq / total));
count++;
if (tokens.length < 1) {
// Ignore empty line
continue;
}


String word = tokens[0];


double freq = 3.0d;
if (tokens.length == 2)
freq = Double.valueOf(tokens[1]);
word = addWord(word);
freqs.put(word, Math.log(freq / total));
count++;
}
System.out.println(String.format(Locale.getDefault(), "user dict %s load finished, tot words:%d, time elapsed:%dms", userDictPath, count, System.currentTimeMillis() - s));
}
System.out.println(String.format(Locale.getDefault(), "user dict %s load finished, tot words:%d, time elapsed:%dms", userDictPath, count, System.currentTimeMillis() - s));
br.close();
}
catch (IOException e) {
System.err.println(String.format(Locale.getDefault(), "%s: load user dict failure!", userDictPath));
}
}

private void shieldWord(String word) {
if (null != word && !"".equals(word.trim())) {
String key = word.trim().toLowerCase(Locale.getDefault());
_dict.disableSegment(key.toCharArray());
freqs.remove(word);
}
}

public DictSegment getTrie() {
return this._dict;
Expand Down