Add functions to get custom segmentations/readings for tokens come from a user dict (#294)

* Add functions to get custom segmentations/readings for tokens come from a user dict

* Rename Yomi to Reading

Author: ikawaha

tokenizer/token.go

@@ -131,6 +131,27 @@ func (t Token) FeatureAt(i int) (string, bool) {
 	return "", false
 }
 
+// UserExtra represents custom segmentation and custom reading for user entries.
+type UserExtra struct {
+	Tokens   []string
+	Readings []string
+}
+
+// UserExtra returns extra data if token comes from a user dict.
+func (t Token) UserExtra() *UserExtra {
+	if t.Class != USER {
+		return nil
+	}
+	tokens := make([]string, len(t.udict.Contents[t.ID].Tokens))
+	copy(tokens, t.udict.Contents[t.ID].Tokens)
+	yomi := make([]string, len(t.udict.Contents[t.ID].Yomi))
+	copy(yomi, t.udict.Contents[t.ID].Yomi)
+	return &UserExtra{
+		Tokens:   tokens,
+		Readings: yomi,
+	}
+}
+
 // POS returns POS elements of features.
 func (t Token) POS() []string {
 	switch t.Class {

tokenizer/token_test.go

@@ -346,6 +346,35 @@ func Test_FeaturesAndPosUserDict(t *testing.T) {
 	}
 }
 
+func Test_FeaturesUserExtra(t *testing.T) {
+	d, err := dict.LoadDictFile(testDictPath)
+	if err != nil {
+		t.Fatalf("unexpected error, %v", err)
+	}
+	tok := Token{
+		ID:      0,
+		Class:   USER,
+		Start:   0,
+		End:     1,
+		Surface: "",
+	}
+	tok.dict = d
+	if udic, err := dict.NewUserDict(userDictSample); err != nil {
+		t.Fatalf("build user dict error: %v", err)
+	} else {
+		tok.udict = udic
+	}
+
+	got := tok.UserExtra()
+	want := &UserExtra{
+		Tokens:   []string{"日本", "経済", "新聞"},
+		Readings: []string{"ニホン", "ケイザイ", "シンブン"},
+	}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("want %v, got %v", want, got)
+	}
+}
+
 func Test_TokenString(t *testing.T) {
 	tok := Token{
 		ID:      123,