utf8.String: move to exp/utf8string.String

R=golang-dev, bradfitz
CC=golang-dev
http://codereview.appspot.com/5528115

21 files changed, 63 insertions, 35 deletions

diff --git a/doc/go1.html b/doc/go1.html
index 2168ba562..af9bbd779 100644
--- a/doc/go1.html
+++ b/doc/go1.html
@@ -923,6 +923,10 @@ Several packages have moved under <code>exp</code> at the time of Go 1's release
 </ul>
 
 <p>
+Also, the <code>utf8.String</code> type has been moved to its own package, <code>exp/utf8string</code>.
+</p>
+
+<p>
 All these packages are available under the same names, with <code>exp/</code> prefixed: <code>exp/ebnf</code> etc.
 </p>
 
@@ -935,7 +939,7 @@ Also, the <code>gotype</code> command now resides in <code>exp/gotype</code>, wh
 <em>Updating</em>:
 Code that uses packages in <code>exp</code> will need to be updated by hand,
 or else compiled from an installation that has <code>exp</code> available.
-Gofix will warn about such uses.
+Gofix or the compiler will complain about such uses.
 <br>
 <font color="red">TODO: gofix should warn about such uses.</font>
 </p>
diff --git a/doc/go1.tmpl b/doc/go1.tmpl
index 21256ed7f..985170411 100644
--- a/doc/go1.tmpl
+++ b/doc/go1.tmpl
@@ -827,6 +827,10 @@ Several packages have moved under <code>exp</code> at the time of Go 1's release
 </ul>
 
 <p>
+Also, the <code>utf8.String</code> type has been moved to its own package, <code>exp/utf8string</code>.
+</p>
+
+<p>
 All these packages are available under the same names, with <code>exp/</code> prefixed: <code>exp/ebnf</code> etc.
 </p>
 
@@ -839,7 +843,7 @@ Also, the <code>gotype</code> command now resides in <code>exp/gotype</code>, wh
 <em>Updating</em>:
 Code that uses packages in <code>exp</code> will need to be updated by hand,
 or else compiled from an installation that has <code>exp</code> available.
-Gofix will warn about such uses.
+Gofix or the compiler will complain about such uses.
 <br>
 <font color="red">TODO: gofix should warn about such uses.</font>
 </p>
diff --git a/src/buildscript/darwin_386.sh b/src/buildscript/darwin_386.sh
index 42dbf5e17..6c8b997f0 100755
--- a/src/buildscript/darwin_386.sh
+++ b/src/buildscript/darwin_386.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/darwin_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/darwin_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/darwin_386/unicode/utf8.a
diff --git a/src/buildscript/darwin_amd64.sh b/src/buildscript/darwin_amd64.sh
index 0766ff262..417e03fde 100755
--- a/src/buildscript/darwin_amd64.sh
+++ b/src/buildscript/darwin_amd64.sh
@@ -124,7 +124,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/darwin_amd64/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.6
 mkdir -p "$GOROOT"/pkg/darwin_amd64/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/darwin_amd64/unicode/utf8.a
diff --git a/src/buildscript/freebsd_386.sh b/src/buildscript/freebsd_386.sh
index 310f3dcf7..2427f9143 100755
--- a/src/buildscript/freebsd_386.sh
+++ b/src/buildscript/freebsd_386.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/freebsd_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/freebsd_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/freebsd_386/unicode/utf8.a
diff --git a/src/buildscript/freebsd_amd64.sh b/src/buildscript/freebsd_amd64.sh
index 9a91017ac..6f632b4a5 100755
--- a/src/buildscript/freebsd_amd64.sh
+++ b/src/buildscript/freebsd_amd64.sh
@@ -124,7 +124,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/freebsd_amd64/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.6
 mkdir -p "$GOROOT"/pkg/freebsd_amd64/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/freebsd_amd64/unicode/utf8.a
diff --git a/src/buildscript/linux_386.sh b/src/buildscript/linux_386.sh
index 26009320c..4deb775f7 100755
--- a/src/buildscript/linux_386.sh
+++ b/src/buildscript/linux_386.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/linux_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/linux_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/linux_386/unicode/utf8.a
diff --git a/src/buildscript/linux_amd64.sh b/src/buildscript/linux_amd64.sh
index f2eae0a59..da27acc3f 100755
--- a/src/buildscript/linux_amd64.sh
+++ b/src/buildscript/linux_amd64.sh
@@ -124,7 +124,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/linux_amd64/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.6
 mkdir -p "$GOROOT"/pkg/linux_amd64/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/linux_amd64/unicode/utf8.a
diff --git a/src/buildscript/linux_arm.sh b/src/buildscript/linux_arm.sh
index 7589a5719..aa68cf802 100755
--- a/src/buildscript/linux_arm.sh
+++ b/src/buildscript/linux_arm.sh
@@ -129,7 +129,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/linux_arm/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-5g -o "$WORK"/unicode/utf8/_obj/_go_.5 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+5g -o "$WORK"/unicode/utf8/_obj/_go_.5 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.5
 mkdir -p "$GOROOT"/pkg/linux_arm/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/linux_arm/unicode/utf8.a
diff --git a/src/buildscript/netbsd_386.sh b/src/buildscript/netbsd_386.sh
index 8f9491c80..5c822f345 100755
--- a/src/buildscript/netbsd_386.sh
+++ b/src/buildscript/netbsd_386.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/netbsd_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/netbsd_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/netbsd_386/unicode/utf8.a
diff --git a/src/buildscript/netbsd_amd64.sh b/src/buildscript/netbsd_amd64.sh
index be0053d06..0bfea9c11 100755
--- a/src/buildscript/netbsd_amd64.sh
+++ b/src/buildscript/netbsd_amd64.sh
@@ -124,7 +124,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/netbsd_amd64/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.6
 mkdir -p "$GOROOT"/pkg/netbsd_amd64/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/netbsd_amd64/unicode/utf8.a
diff --git a/src/buildscript/openbsd_386.sh b/src/buildscript/openbsd_386.sh
index 30ddec6a5..be38d3944 100755
--- a/src/buildscript/openbsd_386.sh
+++ b/src/buildscript/openbsd_386.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/openbsd_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/openbsd_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/openbsd_386/unicode/utf8.a
diff --git a/src/buildscript/openbsd_amd64.sh b/src/buildscript/openbsd_amd64.sh
index c315d59e1..00b32a139 100755
--- a/src/buildscript/openbsd_amd64.sh
+++ b/src/buildscript/openbsd_amd64.sh
@@ -124,7 +124,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/openbsd_amd64/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.6
 mkdir -p "$GOROOT"/pkg/openbsd_amd64/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/openbsd_amd64/unicode/utf8.a
diff --git a/src/buildscript/plan9_386.sh b/src/buildscript/plan9_386.sh
index 10a24724f..d1c3ec647 100755
--- a/src/buildscript/plan9_386.sh
+++ b/src/buildscript/plan9_386.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/plan9_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/plan9_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/plan9_386/unicode/utf8.a
diff --git a/src/buildscript/windows_386.sh b/src/buildscript/windows_386.sh
index b5e79288b..13885d8a2 100755
--- a/src/buildscript/windows_386.sh
+++ b/src/buildscript/windows_386.sh
@@ -128,7 +128,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/windows_386/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+8g -o "$WORK"/unicode/utf8/_obj/_go_.8 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.8
 mkdir -p "$GOROOT"/pkg/windows_386/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/windows_386/unicode/utf8.a
diff --git a/src/buildscript/windows_amd64.sh b/src/buildscript/windows_amd64.sh
index bf87f6af9..6eb6349c9 100755
--- a/src/buildscript/windows_amd64.sh
+++ b/src/buildscript/windows_amd64.sh
@@ -126,7 +126,7 @@ cp "$WORK"/unicode.a "$GOROOT"/pkg/windows_amd64/unicode.a
 
 mkdir -p "$WORK"/unicode/utf8/_obj/
 cd "$GOROOT"/src/pkg/unicode/utf8
-6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./string.go ./utf8.go
+6g -o "$WORK"/unicode/utf8/_obj/_go_.6 -p unicode/utf8 -I "$WORK" ./utf8.go
 gopack grc "$WORK"/unicode/utf8.a "$WORK"/unicode/utf8/_obj/_go_.6
 mkdir -p "$GOROOT"/pkg/windows_amd64/unicode/
 cp "$WORK"/unicode/utf8.a "$GOROOT"/pkg/windows_amd64/unicode/utf8.a
diff --git a/src/pkg/exp/utf8string/Makefile b/src/pkg/exp/utf8string/Makefile
new file mode 100644
index 000000000..49c192861
--- /dev/null
+++ b/src/pkg/exp/utf8string/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=exp/utf8string
+GOFILES=\
+	string.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/unicode/utf8/string.go b/src/pkg/exp/utf8string/string.go
index 443decf05..da1e2de1e 100644
--- a/src/pkg/unicode/utf8/string.go
+++ b/src/pkg/exp/utf8string/string.go
@@ -2,9 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package utf8
+// Package utf8string provides an efficient way to index strings by rune rather than by byte.
+package utf8string
 
-import "errors"
+import (
+	"errors"
+	"unicode/utf8"
+)
 
 // String wraps a regular string with a small structure that provides more
 // efficient indexing by code point index, as opposed to byte index.
@@ -37,10 +41,10 @@ func (s *String) Init(contents string) *String {
 	s.bytePos = 0
 	s.runePos = 0
 	for i := 0; i < len(contents); i++ {
-		if contents[i] >= RuneSelf {
+		if contents[i] >= utf8.RuneSelf {
 			// Not ASCII.
-			s.numRunes = RuneCountInString(contents)
-			_, s.width = DecodeRuneInString(contents)
+			s.numRunes = utf8.RuneCountInString(contents)
+			_, s.width = utf8.DecodeRuneInString(contents)
 			s.nonASCII = i
 			return s
 		}
@@ -121,7 +125,7 @@ func (s *String) At(i int) rune {
 	switch {
 
 	case i == s.runePos-1: // backing up one rune
-		r, s.width = DecodeLastRuneInString(s.str[0:s.bytePos])
+		r, s.width = utf8.DecodeLastRuneInString(s.str[0:s.bytePos])
 		s.runePos = i
 		s.bytePos -= s.width
 		return r
@@ -130,16 +134,16 @@ func (s *String) At(i int) rune {
 		s.bytePos += s.width
 		fallthrough
 	case i == s.runePos:
-		r, s.width = DecodeRuneInString(s.str[s.bytePos:])
+		r, s.width = utf8.DecodeRuneInString(s.str[s.bytePos:])
 		return r
 	case i == 0: // start of string
-		r, s.width = DecodeRuneInString(s.str)
+		r, s.width = utf8.DecodeRuneInString(s.str)
 		s.runePos = 0
 		s.bytePos = 0
 		return r
 
 	case i == s.numRunes-1: // last rune in string
-		r, s.width = DecodeLastRuneInString(s.str)
+		r, s.width = utf8.DecodeLastRuneInString(s.str)
 		s.runePos = i
 		s.bytePos = len(s.str) - s.width
 		return r
@@ -175,7 +179,7 @@ func (s *String) At(i int) rune {
 	if forward {
 		// TODO: Is it much faster to use a range loop for this scan?
 		for {
-			r, s.width = DecodeRuneInString(s.str[s.bytePos:])
+			r, s.width = utf8.DecodeRuneInString(s.str[s.bytePos:])
 			if s.runePos == i {
 				break
 			}
@@ -184,7 +188,7 @@ func (s *String) At(i int) rune {
 		}
 	} else {
 		for {
-			r, s.width = DecodeLastRuneInString(s.str[0:s.bytePos])
+			r, s.width = utf8.DecodeLastRuneInString(s.str[0:s.bytePos])
 			s.runePos--
 			s.bytePos -= s.width
 			if s.runePos == i {
diff --git a/src/pkg/unicode/utf8/string_test.go b/src/pkg/exp/utf8string/string_test.go
index 2c139bea9..28511b2f5 100644
--- a/src/pkg/unicode/utf8/string_test.go
+++ b/src/pkg/exp/utf8string/string_test.go
@@ -2,14 +2,23 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package utf8_test
+package utf8string
 
 import (
 	"math/rand"
 	"testing"
-	. "unicode/utf8"
+	"unicode/utf8"
 )
 
+var testStrings = []string{
+	"",
+	"abcd",
+	"☺☻☹",
+	"日a本b語ç日ð本Ê語þ日¥本¼語i日©",
+	"日a本b語ç日ð本Ê語þ日¥本¼語i日©日a本b語ç日ð本Ê語þ日¥本¼語i日©日a本b語ç日ð本Ê語þ日¥本¼語i日©",
+	"\x80\x80\x80\x80",
+}
+
 func TestScanForwards(t *testing.T) {
 	for _, s := range testStrings {
 		runes := []rune(s)
@@ -106,7 +115,7 @@ func TestLimitSliceAccess(t *testing.T) {
 		if str.Slice(0, 0) != "" {
 			t.Error("failure with empty slice at beginning")
 		}
-		nr := RuneCountInString(s)
+		nr := utf8.RuneCountInString(s)
 		if str.Slice(nr, nr) != "" {
 			t.Error("failure with empty slice at end")
 		}
diff --git a/src/pkg/unicode/utf8/Makefile b/src/pkg/unicode/utf8/Makefile
index 9863433b0..b6c36f07b 100644
--- a/src/pkg/unicode/utf8/Makefile
+++ b/src/pkg/unicode/utf8/Makefile
@@ -6,7 +6,6 @@ include ../../../Make.inc
 
 TARG=unicode/utf8
 GOFILES=\
-	string.go\
 	utf8.go\
 
 include ../../../Make.pkg
diff --git a/test/alias.go b/test/alias.go
index 410a44e6c..639a9cabb 100644
--- a/test/alias.go
+++ b/test/alias.go
@@ -7,7 +7,7 @@
 package main
 
 // Test that error messages say what the source file says
-// (uint8 vs byte).
+// (uint8 vs byte, int32 vs. rune).
 
 import (
 	"fmt"
@@ -29,7 +29,4 @@ func main() {
 	ff.Format(fs, x) // ERROR "rune"
 
 	utf8.RuneStart(x) // ERROR "byte"
-
-	var s utf8.String
-	s.At(x) // ERROR "int"
 }