Fix a bug when the option to skip white space is disabled (#231)

Author: ikawaha

filter/sentence_splitter.go

@@ -62,19 +62,22 @@ func (s SentenceSplitter) ScanSentences(data []byte, atEOF bool) (advance int, t
 		r, size := utf8.DecodeRune(data[p:])
 		if s.SkipWhiteSpace && unicode.IsSpace(r) {
 			p += size
-			if head {
+			switch {
+			case head:
 				start, end = p, p
-			} else if s.isDelim(r) {
+			case s.isDelim(r):
 				return p, data[start:end], nil
-			} else if s.DoubleLineFeedSplit && r == '\n' {
+			case s.DoubleLineFeedSplit && r == '\n':
 				if nn {
 					return p, data[start:end], nil
 				}
 				nn = true
+			case nn:
+				nn = false
 			}
 			continue
 		}
-		head, nn = false, false // clear flags
+		head = false
 		if end != p {
 			for i := 0; i < size; i++ {
 				data[end+i] = data[p+i]
@@ -83,6 +86,12 @@ func (s SentenceSplitter) ScanSentences(data []byte, atEOF bool) (advance int, t
 		p += size
 		end += size
 		rcount++
+		if s.DoubleLineFeedSplit && r == '\n' {
+			if nn {
+				return p, data[start:end], nil
+			}
+			nn = true
+		}
 		if !s.isDelim(r) && rcount < s.MaxRuneLen {
 			continue
 		}

filter/sentence_splitter_test.go

@@ -109,6 +109,122 @@ func Test_DelimWhiteSpace(t *testing.T) {
 	}
 }
 
+func Test_DisableSkipWhiteSpace(t *testing.T) {
+	testdata := []struct {
+		input  string
+		expect []string
+	}{
+		{
+			input:  "",
+			expect: []string{},
+		},
+		{
+			input:  "あああ",
+			expect: []string{"あああ"},
+		},
+		{
+			input:  "   ",
+			expect: []string{"   "},
+		},
+		{
+			input:  " x x x ",
+			expect: []string{" ", "x ", "x ", "x "},
+		},
+		{
+			input:  "こんにちはは「さようなら 」　\U0001f363が好き",
+			expect: []string{"こんにちはは「さようなら 」　", "\U0001f363が好き"},
+		},
+		{
+			input:  "こんにちは \t\tさようなら　おはよう おやすみ   ",
+			expect: []string{"こんにちは ", "\t\tさようなら　", "おはよう ", "おやすみ   "},
+		},
+		{
+			input:  "すももも\t\n\n \tももも\n\nもものうち\n\n",
+			expect: []string{"すももも\t\n\n", " ", "\tももも\n\n", "もものうち\n\n"},
+		},
+	}
+
+	s := filter.SentenceSplitter{
+		Delim:               []rune{' ', '　'}, // white spaces
+		Follower:            []rune{'.', '｣', '」', '』', ')', '）', '｝', '}', '〉', '》'},
+		SkipWhiteSpace:      false,
+		DoubleLineFeedSplit: true,
+		MaxRuneLen:          256,
+	}
+	for _, d := range testdata {
+		scanner := bufio.NewScanner(strings.NewReader(d.input))
+		scanner.Split(s.ScanSentences)
+		r := make([]string, 0, len(d.expect))
+		for scanner.Scan() {
+			r = append(r, scanner.Text())
+		}
+		if !reflect.DeepEqual(r, d.expect) {
+			t.Errorf("input %q, got %#v, expected %#v", d.input, r, d.expect)
+		}
+		if got := strings.Join(r, ""); got != d.input {
+			t.Errorf("got len=%d %q, input len=%d %q", len(got), got, len(d.input), d.input)
+		}
+	}
+}
+
+func Test_DisableSkipWhiteSpaceAndDoubleLineFeedSplit(t *testing.T) {
+	testdata := []struct {
+		input  string
+		expect []string
+	}{
+		{
+			input:  "",
+			expect: []string{},
+		},
+		{
+			input:  "あああ",
+			expect: []string{"あああ"},
+		},
+		{
+			input:  "   ",
+			expect: []string{"   "},
+		},
+		{
+			input:  " x x x ",
+			expect: []string{" ", "x ", "x ", "x "},
+		},
+		{
+			input:  "こんにちはは「さようなら 」　\U0001f363が好き",
+			expect: []string{"こんにちはは「さようなら 」　", "\U0001f363が好き"},
+		},
+		{
+			input:  "こんにちは \t\tさようなら　おはよう おやすみ   ",
+			expect: []string{"こんにちは ", "\t\tさようなら　", "おはよう ", "おやすみ   "},
+		},
+		{
+			input:  "すももも\t\n\n \tももも\n\nもものうち\n\n",
+			expect: []string{"すももも\t\n\n ", "\tももも\n\nもものうち\n\n"},
+		},
+	}
+
+	s := filter.SentenceSplitter{
+		Delim:               []rune{' ', '　'}, // white spaces
+		Follower:            []rune{'.', '｣', '」', '』', ')', '）', '｝', '}', '〉', '》'},
+		SkipWhiteSpace:      false,
+		DoubleLineFeedSplit: false,
+		MaxRuneLen:          256,
+	}
+	for _, d := range testdata {
+		scanner := bufio.NewScanner(strings.NewReader(d.input))
+		scanner.Split(s.ScanSentences)
+		r := make([]string, 0, len(d.expect))
+		for scanner.Scan() {
+			r = append(r, scanner.Text())
+		}
+		if !reflect.DeepEqual(r, d.expect) {
+			t.Errorf("input %q, got %#v, expected %#v", d.input, r, d.expect)
+		}
+		if got := strings.Join(r, ""); got != d.input {
+			t.Errorf("got len=%d %q, input len=%d %q", len(got), got, len(d.input), d.input)
+		}
+	}
+}
+
 func Test_ScanSentences(t *testing.T) {
 	testdata := []struct {
 		atEnd   bool