bufio: fix handling of empty tokens at end of line/file

Fixes issue 8672.

LGTM=rsc
R=golang-codereviews, rsc
CC=golang-codereviews
https://codereview.appspot.com/145390043

2 files changed, 39 insertions, 1 deletions

diff --git a/src/bufio/scan.go b/src/bufio/scan.go
index c5714f331..a41451524 100644
--- a/src/bufio/scan.go
+++ b/src/bufio/scan.go
@@ -112,7 +112,9 @@ func (s *Scanner) Scan() bool {
 	// Loop until we have a token.
 	for {
 		// See if we can get a token with what we already have.
-		if s.end > s.start {
+		// If we've run out of data but have an error, give the split function
+		// a chance to recover any remaining, possibly empty token.
+		if s.end > s.start || s.err != nil {
 			advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
 			if err != nil {
 				s.setErr(err)
diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go
index 3ddb25acf..1454a8113 100644
--- a/src/bufio/scan_test.go
+++ b/src/bufio/scan_test.go
@@ -419,3 +419,39 @@ func TestScanWordsExcessiveWhiteSpace(t *testing.T) {
 		t.Fatalf("unexpected token: %v", token)
 	}
 }
+
+// Test that empty tokens, including at end of line or end of file, are found by the scanner.
+// Issue 8672: Could miss final empty token.
+
+func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	for i := 0; i < len(data); i++ {
+		if data[i] == ',' {
+			return i + 1, data[:i], nil
+		}
+	}
+	if !atEOF {
+		return 0, nil, nil
+	}
+	return 0, data, nil
+}
+
+func TestEmptyTokens(t *testing.T) {
+	s := NewScanner(strings.NewReader("1,2,3,"))
+	values := []string{"1", "2", "3", ""}
+	s.Split(commaSplit)
+	var i int
+	for i = 0; i < len(values); i++ {
+		if !s.Scan() {
+			break
+		}
+		if s.Text() != values[i] {
+			t.Errorf("%d: expected %q got %q", i, values[i], s.Text())
+		}
+	}
+	if i != len(values) {
+		t.Errorf("got %d fields, expected %d", i, len(values))
+	}
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+}