Merge pull request #227 from ikawaha/fix/update-uni-dic

ikawaha · web-flow · commit 94630332328b · 2020-12-29T22:52:57.000+09:00
diff --git a/go.mod b/go.mod
@@ -5,5 +5,5 @@ go 1.15
 require (
 	github.com/ikawaha/kagome-dict v1.0.1
 	github.com/ikawaha/kagome-dict/ipa v1.0.1
-	github.com/ikawaha/kagome-dict/uni v1.0.1
+	github.com/ikawaha/kagome-dict/uni v1.1.0
 )
diff --git a/go.sum b/go.sum
@@ -2,8 +2,8 @@ github.com/ikawaha/kagome-dict v1.0.1 h1:CdGx80JEz+nBUP0+m+47pubzOhURYPRX+zXLRrg
 github.com/ikawaha/kagome-dict v1.0.1/go.mod h1:pbeH4qwp/LVWKNePhU4Mh11w/l8uUSXKxfvNwRkj3NY=
 github.com/ikawaha/kagome-dict/ipa v1.0.1 h1:pE98DI+3e8kqUSL50M7/QE5BDk/o63/o9fDanNtgkN0=
 github.com/ikawaha/kagome-dict/ipa v1.0.1/go.mod h1:6KbSbFnEwIlR65gYAowfwOkbze3bnFH1BILpOEGBkvY=
-github.com/ikawaha/kagome-dict/uni v1.0.1 h1:0HfqEmpHyoRroR8whDsn/pu3LgMljpvURrACxJlk6hs=
-github.com/ikawaha/kagome-dict/uni v1.0.1/go.mod h1:nij0TsdXZ/WauqM9X0XW7VpOx/JkiSkiGicN1bJpx2Q=
+github.com/ikawaha/kagome-dict/uni v1.1.0 h1:b6VKvsjfYT8SB6Lo1LBqDHwIugR02lhkCWFy1Vl/kPI=
+github.com/ikawaha/kagome-dict/uni v1.1.0/go.mod h1:nij0TsdXZ/WauqM9X0XW7VpOx/JkiSkiGicN1bJpx2Q=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/tokenizer/tokenizer_test.go b/tokenizer/tokenizer_test.go
@@ -82,11 +82,7 @@ func Test_AnalyzeEmptyInput(t *testing.T) {
 		t.Fatalf("got %v, expected %v", tokens, expected)
 	}
 	for i, tok := range tokens {
-		if tok.ID != expected[i].ID ||
-			tok.Class != expected[i].Class ||
-			tok.Start != expected[i].Start ||
-			tok.End != expected[i].End ||
-			tok.Surface != expected[i].Surface {
+		if !tok.Equal(expected[i]) {
 			t.Errorf("got %v, expected %v", tok, expected[i])
 		}
 	}
@@ -131,17 +127,13 @@ func Test_AnalyzeUnknown(t *testing.T) {
 	expected := []Token{
 		{ID: -1, Surface: "BOS"},
 		{ID: 34, Surface: "ポポピ", Start: 0, End: 3, Class: TokenClass(lattice.UNKNOWN)},
-		{ID: -1, Surface: "EOS", Start: 3, End: 3},
+		{ID: -1, Surface: "EOS", Start: 3, End: 3, Position: 9},
 	}
 	if len(tokens) != len(expected) {
 		t.Fatalf("got %v, expected %v", tokens, expected)
 	}
 	for i, tok := range tokens {
-		if tok.ID != expected[i].ID ||
-			tok.Class != expected[i].Class ||
-			tok.Start != expected[i].Start ||
-			tok.End != expected[i].End ||
-			tok.Surface != expected[i].Surface {
+		if !tok.Equal(expected[i]) {
 			t.Errorf("got %v, expected %v", tok, expected[i])
 		}
 	}
@@ -204,11 +196,7 @@ func Test_AnalyzeWithSearchModeEmptyInput(t *testing.T) {
 		t.Fatalf("got %v, expected %v", tokens, expected)
 	}
 	for i, tok := range tokens {
-		if tok.ID != expected[i].ID ||
-			tok.Class != expected[i].Class ||
-			tok.Start != expected[i].Start ||
-			tok.End != expected[i].End ||
-			tok.Surface != expected[i].Surface {
+		if !tok.Equal(expected[i]) {
 			t.Errorf("got %v, expected %v", tok, expected[i])
 		}
 	}

Original file line number	Diff line number	Diff line change
`@@ -5,5 +5,5 @@ go 1.15`
`5`	`5`	`require (`
`6`	`6`	`github.com/ikawaha/kagome-dict v1.0.1`
`7`	`7`	`github.com/ikawaha/kagome-dict/ipa v1.0.1`
`8`		`- github.com/ikawaha/kagome-dict/uni v1.0.1`
	`8`	`+ github.com/ikawaha/kagome-dict/uni v1.1.0`
`9`	`9`	`)`
Original file line number	Diff line number	Diff line change
`@@ -82,11 +82,7 @@ func Test_AnalyzeEmptyInput(t *testing.T) {`
`82`	`82`	`t.Fatalf("got %v, expected %v", tokens, expected)`
`83`	`83`	`}`
`84`	`84`	`for i, tok := range tokens {`
`85`		`- if tok.ID != expected[i].ID \|\|`
`86`		`- tok.Class != expected[i].Class \|\|`
`87`		`- tok.Start != expected[i].Start \|\|`
`88`		`- tok.End != expected[i].End \|\|`
`89`		`- tok.Surface != expected[i].Surface {`
	`85`	`+ if !tok.Equal(expected[i]) {`
`90`	`86`	`t.Errorf("got %v, expected %v", tok, expected[i])`
`91`	`87`	`}`
`92`	`88`	`}`
`@@ -131,17 +127,13 @@ func Test_AnalyzeUnknown(t *testing.T) {`
`131`	`127`	`expected := []Token{`
`132`	`128`	`{ID: -1, Surface: "BOS"},`
`133`	`129`	`{ID: 34, Surface: "ポポピ", Start: 0, End: 3, Class: TokenClass(lattice.UNKNOWN)},`
`134`		`- {ID: -1, Surface: "EOS", Start: 3, End: 3},`
	`130`	`+ {ID: -1, Surface: "EOS", Start: 3, End: 3, Position: 9},`
`135`	`131`	`}`
`136`	`132`	`if len(tokens) != len(expected) {`
`137`	`133`	`t.Fatalf("got %v, expected %v", tokens, expected)`
`138`	`134`	`}`
`139`	`135`	`for i, tok := range tokens {`
`140`		`- if tok.ID != expected[i].ID \|\|`
`141`		`- tok.Class != expected[i].Class \|\|`
`142`		`- tok.Start != expected[i].Start \|\|`
`143`		`- tok.End != expected[i].End \|\|`
`144`		`- tok.Surface != expected[i].Surface {`
	`136`	`+ if !tok.Equal(expected[i]) {`
`145`	`137`	`t.Errorf("got %v, expected %v", tok, expected[i])`
`146`	`138`	`}`
`147`	`139`	`}`
`@@ -204,11 +196,7 @@ func Test_AnalyzeWithSearchModeEmptyInput(t *testing.T) {`
`204`	`196`	`t.Fatalf("got %v, expected %v", tokens, expected)`
`205`	`197`	`}`
`206`	`198`	`for i, tok := range tokens {`
`207`		`- if tok.ID != expected[i].ID \|\|`
`208`		`- tok.Class != expected[i].Class \|\|`
`209`		`- tok.Start != expected[i].Start \|\|`
`210`		`- tok.End != expected[i].End \|\|`
`211`		`- tok.Surface != expected[i].Surface {`
	`199`	`+ if !tok.Equal(expected[i]) {`
`212`	`200`	`t.Errorf("got %v, expected %v", tok, expected[i])`
`213`	`201`	`}`
`214`	`202`	`}`