Merge pull request #258 from ikawaha/develop

Develop

Author: ikawaha

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        go: ['1.16.x', '1.17.x']
+        go: ['1.17.x', '1.18.x']
     runs-on: ${{ matrix.os }}
 
     steps:

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.17.x'
+          go-version: '1.18.x'
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v3
         with:

app.json

@@ -2,7 +2,7 @@
     "name": "kagome",
     "description": "Kagome Japanese Morphological Analyzer",
     "env": {
-        "GOVERSION": "go1.17",
+        "GOVERSION": "go1.18",
         "GO_INSTALL_PACKAGE_SPEC": "."
     },
     "buildpacks": [

go.mod

@@ -1,7 +1,7 @@
 module github.com/ikawaha/kagome/v2
 
-// +heroku goVersion go1.17
-go 1.17
+// +heroku goVersion go1.18
+go 1.18
 
 require (
 	github.com/ikawaha/kagome-dict v1.0.4

tokenizer/lattice/lattice.go

@@ -41,8 +41,8 @@ var latticePool = sync.Pool{
 // Lattice represents a grid of morph nodes.
 type Lattice struct {
 	Input  string
-	Output []*node
-	list   [][]*node
+	Output []*Node
+	list   [][]*Node
 	dic    *dict.Dict
 	udic   *dict.UserDict
 }
@@ -86,15 +86,15 @@ func (la *Lattice) addNode(pos, id, position, start int, class NodeClass, surfac
 	case USER:
 		// use default cost
 	}
-	n := newNode()
+	n := nodePool.Get().(*Node)
 	n.ID = id
 	n.Position = position
 	n.Start = start
 	n.Class = class
 	n.Cost = 0
 	n.Left, n.Right, n.Weight = int32(m.LeftID), int32(m.RightID), int32(m.Weight)
 	n.Surface = surface
-	n.Prev = nil
+	n.prev = nil
 	p := pos + utf8.RuneCountInString(surface)
 	la.list[p] = append(la.list[p], n)
 }
@@ -105,8 +105,7 @@ func (la *Lattice) Build(inp string) {
 	rc := utf8.RuneCountInString(inp)
 	la.Input = inp
 	if cap(la.list) < rc+2 {
-		const expandRatio = 2
-		la.list = make([][]*node, 0, (rc+2)*expandRatio)
+		la.list = make([][]*Node, 0, rc+2)
 	}
 	la.list = la.list[0 : rc+2]
 
@@ -201,7 +200,7 @@ func kanjiOnly(s string) bool {
 	return s != ""
 }
 
-func additionalCost(n *node) int {
+func additionalCost(n *Node) int {
 	l := utf8.RuneCountInString(n.Surface)
 	if l > searchModeKanjiLength && kanjiOnly(n.Surface) {
 		return (l - searchModeKanjiLength) * searchModeKanjiPenalty
@@ -236,7 +235,7 @@ func (la *Lattice) Forward(m TokenizeMode) {
 				}
 				if j == 0 || int32(totalCost) < la.list[i][index].Cost {
 					la.list[i][index].Cost = int32(totalCost)
-					la.list[i][index].Prev = la.list[target.Start][j]
+					la.list[i][index].prev = la.list[target.Start][j]
 				}
 			}
 		}
@@ -245,26 +244,20 @@ func (la *Lattice) Forward(m TokenizeMode) {
 
 // Backward runs backward algorithm of the Viterbi.
 func (la *Lattice) Backward(m TokenizeMode) {
-	const bufferExpandRatio = 2
 	size := len(la.list)
 	if size == 0 {
 		return
 	}
-	if cap(la.Output) < size {
-		la.Output = make([]*node, 0, size*bufferExpandRatio)
-	} else {
-		la.Output = la.Output[:0]
-	}
-	for p := la.list[size-1][0]; p != nil; p = p.Prev {
+	for p := la.list[size-1][0]; p != nil; p = p.prev {
 		if m != Extended || p.Class != UNKNOWN {
 			la.Output = append(la.Output, p)
 			continue
 		}
 		runeLen := utf8.RuneCountInString(p.Surface)
-		stack := make([]*node, 0, runeLen)
+		stack := make([]*Node, 0, runeLen)
 		i := 0
 		for k, r := range p.Surface {
-			stack = append(stack, &node{
+			stack = append(stack, &Node{
 				ID:       p.ID,
 				Start:    p.Start + i,
 				Class:    DUMMY,
@@ -279,7 +272,7 @@ func (la *Lattice) Backward(m TokenizeMode) {
 	}
 }
 
-func posFeature(d *dict.Dict, u *dict.UserDict, t *node) string {
+func posFeature(d *dict.Dict, u *dict.UserDict, t *Node) string {
 	var ret []string
 	switch t.Class {
 	case KNOWN:
@@ -318,13 +311,13 @@ func posFeature(d *dict.Dict, u *dict.UserDict, t *node) string {
 // Dot outputs a lattice in the graphviz dot format.
 //nolint:gocyclo
 func (la *Lattice) Dot(w io.Writer) {
-	bests := make(map[*node]struct{})
+	bests := make(map[*Node]struct{})
 	for _, n := range la.Output {
 		bests[n] = struct{}{}
 	}
 	type edge struct {
-		from *node
-		to   *node
+		from *Node
+		to   *Node
 	}
 	edges := make([]edge, 0, 1024)
 	for i, size := 1, len(la.list); i < size; i++ {

tokenizer/lattice/lattice_test.go

@@ -21,7 +21,7 @@ func Test_LatticeBuildEmptyInput(t *testing.T) {
 	if la.Input != inp {
 		t.Errorf("got %v, expected %v", la.Input, inp)
 	}
-	boseos := node{ID: -1}
+	boseos := Node{ID: -1}
 	if len(la.list) != 2 {
 		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
 	} else if len(la.list[0]) != 1 || *la.list[0][0] != boseos {
@@ -53,8 +53,8 @@ func Test_LatticeBuild(t *testing.T) {
 	if la.Input != inp {
 		t.Errorf("got %v, expected %v", la.Input, inp)
 	}
-	bos := node{ID: -1}
-	eos := node{ID: -1, Position: len(inp), Start: 1}
+	bos := Node{ID: -1}
+	eos := Node{ID: -1, Position: len(inp), Start: 1}
 	if len(la.list) != 3 {
 		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
 	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
@@ -133,8 +133,8 @@ func Test_LatticeBuildUnknown(t *testing.T) {
 	if la.Input != inp {
 		t.Errorf("got %v, expected %v", la.Input, inp)
 	}
-	bos := node{ID: -1}
-	eos := node{ID: -1, Position: len(inp), Start: 3}
+	bos := Node{ID: -1}
+	eos := Node{ID: -1, Position: len(inp), Start: 3}
 	if len(la.list) != 5 {
 		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
 	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
@@ -216,8 +216,8 @@ func Test_LatticeBuildInvalidInput(t *testing.T) {
 	if la.Input != inp {
 		t.Errorf("got %v, expected %v", la.Input, inp)
 	}
-	bos := node{ID: -1}
-	eos := node{ID: -1, Position: len(inp), Start: 4}
+	bos := Node{ID: -1}
+	eos := Node{ID: -1, Position: len(inp), Start: 4}
 	if len(la.list) != 6 {
 		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
 	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {

tokenizer/lattice/node.go

@@ -31,7 +31,8 @@ func (nc NodeClass) String() string {
 	return "UNDEF"
 }
 
-type node struct {
+// Node is a lattice node.
+type Node struct {
 	ID       int
 	Position int // byte position
 	Start    int // rune position
@@ -41,15 +42,11 @@ type node struct {
 	Right    int32
 	Weight   int32
 	Surface  string
-	Prev     *node
+	prev     *Node
 }
 
 var nodePool = sync.Pool{
 	New: func() interface{} {
-		return new(node)
+		return new(Node)
 	},
 }
-
-func newNode() *node {
-	return nodePool.Get().(*node)
-}

tokenizer/lattice/node_test.go

@@ -5,20 +5,19 @@ import (
 )
 
 func Test_NodeClassString(t *testing.T) {
-	pairs := []struct {
-		in  NodeClass
-		out string
+	testdata := []struct {
+		class NodeClass
+		want  string
 	}{
-		{DUMMY, "DUMMY"},
-		{KNOWN, "KNOWN"},
-		{UNKNOWN, "UNKNOWN"},
-		{USER, "USER"},
-		{NodeClass(999), "UNDEF"},
+		{class: DUMMY, want: "DUMMY"},
+		{class: KNOWN, want: "KNOWN"},
+		{class: UNKNOWN, want: "UNKNOWN"},
+		{class: USER, want: "USER"},
+		{class: NodeClass(999), want: "UNDEF"},
 	}
-
-	for _, p := range pairs {
-		if p.in.String() != p.out {
-			t.Errorf("got %v, expected %v", p.in.String(), p.out)
+	for _, p := range testdata {
+		if got := p.class.String(); got != p.want {
+			t.Errorf("got %v, expected %v", got, p.want)
 		}
 	}
 }

tokenizer/token.go

@@ -165,6 +165,29 @@ func (t Token) POS() []string {
 	return nil
 }
 
+// EqualFeatures returns true, if the features of tokens are equal.
+func (t Token) EqualFeatures(tt Token) bool {
+	return EqualFeatures(t.Features(), tt.Features())
+}
+
+// EqualPOS returns true, if the POSs of tokens are equal.
+func (t Token) EqualPOS(tt Token) bool {
+	return EqualFeatures(t.POS(), tt.POS())
+}
+
+// EqualFeatures returns true, if the features are equal.
+func EqualFeatures(lhs, rhs []string) bool {
+	if len(lhs) != len(rhs) {
+		return false
+	}
+	for i := 0; i < len(lhs); i++ {
+		if lhs[i] != rhs[i] {
+			return false
+		}
+	}
+	return true
+}
+
 // InflectionalType returns the inflectional type feature if exists.
 func (t Token) InflectionalType() (string, bool) {
 	return t.pickupFromFeatures(dict.InflectionalType)