runtime/race: better handling of atomic operations

This change fixes the last known false negative of the race detector --
detection of races between mutating atomic operations and non-atomic operations.
Race runtime already has functions for precise modelling of various atomic operations,
so this change just forwards all atomic ops to race runtime
instead of poor man modeling in sync/atomic package.
Performance is also improved -- full sync/atomic tests run in 60s instead of 85s now.

LGTM=khr
R=golang-codereviews, khr
CC=golang-codereviews, rsc
https://codereview.appspot.com/111310044

3 files changed, 8 insertions, 278 deletions

diff --git a/src/pkg/sync/atomic/doc.go b/src/pkg/sync/atomic/doc.go
index 17ba72fa1..10fb8c917 100644
--- a/src/pkg/sync/atomic/doc.go
+++ b/src/pkg/sync/atomic/doc.go
@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !race
-
 // Package atomic provides low-level atomic memory primitives
 // useful for implementing synchronization algorithms.
 //
diff --git a/src/pkg/sync/atomic/race.go b/src/pkg/sync/atomic/race.go
deleted file mode 100644
index 6cbbf12cb..000000000
--- a/src/pkg/sync/atomic/race.go
+++ /dev/null
@@ -1,276 +0,0 @@
-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build race
-
-package atomic
-
-import (
-	"runtime"
-	"unsafe"
-)
-
-// We use runtime.RaceRead() inside of atomic operations to catch races
-// between atomic and non-atomic operations.  It will also catch races
-// between Mutex.Lock() and mutex overwrite (mu = Mutex{}).  Since we use
-// only RaceRead() we won't catch races with non-atomic loads.
-// Otherwise (if we use RaceWrite()) we will report races
-// between atomic operations (false positives).
-
-var mtx uint32 = 1 // same for all
-
-func SwapInt32(addr *int32, new int32) (old int32) {
-	return int32(SwapUint32((*uint32)(unsafe.Pointer(addr)), uint32(new)))
-}
-
-func SwapUint32(addr *uint32, new uint32) (old uint32) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	old = *addr
-	*addr = new
-	runtime.RaceReleaseMerge(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func SwapInt64(addr *int64, new int64) (old int64) {
-	return int64(SwapUint64((*uint64)(unsafe.Pointer(addr)), uint64(new)))
-}
-
-func SwapUint64(addr *uint64, new uint64) (old uint64) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	old = *addr
-	*addr = new
-	runtime.RaceReleaseMerge(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func SwapUintptr(addr *uintptr, new uintptr) (old uintptr) {
-	return uintptr(SwapPointer((*unsafe.Pointer)(unsafe.Pointer(addr)), unsafe.Pointer(new)))
-}
-
-func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	old = *addr
-	*addr = new
-	runtime.RaceReleaseMerge(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func CompareAndSwapInt32(val *int32, old, new int32) bool {
-	return CompareAndSwapUint32((*uint32)(unsafe.Pointer(val)), uint32(old), uint32(new))
-}
-
-func CompareAndSwapUint32(val *uint32, old, new uint32) (swapped bool) {
-	_ = *val
-	swapped = false
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	if *val == old {
-		*val = new
-		swapped = true
-		runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	}
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func CompareAndSwapInt64(val *int64, old, new int64) bool {
-	return CompareAndSwapUint64((*uint64)(unsafe.Pointer(val)), uint64(old), uint64(new))
-}
-
-func CompareAndSwapUint64(val *uint64, old, new uint64) (swapped bool) {
-	_ = *val
-	swapped = false
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	if *val == old {
-		*val = new
-		swapped = true
-		runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	}
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func CompareAndSwapPointer(val *unsafe.Pointer, old, new unsafe.Pointer) (swapped bool) {
-	_ = *val
-	swapped = false
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	if *val == old {
-		*val = new
-		swapped = true
-		runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	}
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func CompareAndSwapUintptr(val *uintptr, old, new uintptr) (swapped bool) {
-	_ = *val
-	swapped = false
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	if *val == old {
-		*val = new
-		swapped = true
-		runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	}
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func AddInt32(val *int32, delta int32) int32 {
-	return int32(AddUint32((*uint32)(unsafe.Pointer(val)), uint32(delta)))
-}
-
-func AddUint32(val *uint32, delta uint32) (new uint32) {
-	_ = *val
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	*val = *val + delta
-	new = *val
-	runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	runtime.RaceSemrelease(&mtx)
-
-	return
-}
-
-func AddInt64(val *int64, delta int64) int64 {
-	return int64(AddUint64((*uint64)(unsafe.Pointer(val)), uint64(delta)))
-}
-
-func AddUint64(val *uint64, delta uint64) (new uint64) {
-	_ = *val
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	*val = *val + delta
-	new = *val
-	runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	runtime.RaceSemrelease(&mtx)
-
-	return
-}
-
-func AddUintptr(val *uintptr, delta uintptr) (new uintptr) {
-	_ = *val
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(val))
-	runtime.RaceAcquire(unsafe.Pointer(val))
-	*val = *val + delta
-	new = *val
-	runtime.RaceReleaseMerge(unsafe.Pointer(val))
-	runtime.RaceSemrelease(&mtx)
-
-	return
-}
-
-func LoadInt32(addr *int32) int32 {
-	return int32(LoadUint32((*uint32)(unsafe.Pointer(addr))))
-}
-
-func LoadUint32(addr *uint32) (val uint32) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	val = *addr
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func LoadInt64(addr *int64) int64 {
-	return int64(LoadUint64((*uint64)(unsafe.Pointer(addr))))
-}
-
-func LoadUint64(addr *uint64) (val uint64) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	val = *addr
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	val = *addr
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func LoadUintptr(addr *uintptr) (val uintptr) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	runtime.RaceAcquire(unsafe.Pointer(addr))
-	val = *addr
-	runtime.RaceSemrelease(&mtx)
-	return
-}
-
-func StoreInt32(addr *int32, val int32) {
-	StoreUint32((*uint32)(unsafe.Pointer(addr)), uint32(val))
-}
-
-func StoreUint32(addr *uint32, val uint32) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	*addr = val
-	runtime.RaceRelease(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-}
-
-func StoreInt64(addr *int64, val int64) {
-	StoreUint64((*uint64)(unsafe.Pointer(addr)), uint64(val))
-}
-
-func StoreUint64(addr *uint64, val uint64) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	*addr = val
-	runtime.RaceRelease(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-}
-
-func StorePointer(addr *unsafe.Pointer, val unsafe.Pointer) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	*addr = val
-	runtime.RaceRelease(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-}
-
-func StoreUintptr(addr *uintptr, val uintptr) {
-	_ = *addr
-	runtime.RaceSemacquire(&mtx)
-	runtime.RaceRead(unsafe.Pointer(addr))
-	*addr = val
-	runtime.RaceRelease(unsafe.Pointer(addr))
-	runtime.RaceSemrelease(&mtx)
-}
diff --git a/src/pkg/sync/atomic/race.s b/src/pkg/sync/atomic/race.s
new file mode 100644
index 000000000..4dadc9ed7
--- /dev/null
+++ b/src/pkg/sync/atomic/race.s
@@ -0,0 +1,8 @@
+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build race
+
+// This file is here only to allow external functions.
+// The operations are implemented in src/pkg/runtime/race_amd64.s