summaryrefslogtreecommitdiff
path: root/src/runtime
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime')
-rw-r--r--src/runtime/asm_386.s21
-rw-r--r--src/runtime/asm_amd64.s22
-rw-r--r--src/runtime/asm_amd64p32.s2
-rw-r--r--src/runtime/asm_arm.s28
-rw-r--r--src/runtime/cgo/callbacks.c3
-rw-r--r--src/runtime/cgo/dragonfly.c4
-rw-r--r--src/runtime/cgo/freebsd.c4
-rw-r--r--src/runtime/cgo/gcc_setenv.c7
-rw-r--r--src/runtime/cgo/netbsd.c4
-rw-r--r--src/runtime/cgo/openbsd.c4
-rw-r--r--src/runtime/cgo/setenv.c3
-rw-r--r--src/runtime/cgocall.go12
-rw-r--r--src/runtime/crash_test.go88
-rw-r--r--src/runtime/debug.go30
-rw-r--r--src/runtime/debug/debug.s (renamed from src/runtime/debug/debug.c)0
-rw-r--r--src/runtime/defs_windows.go1
-rw-r--r--src/runtime/defs_windows_386.h1
-rw-r--r--src/runtime/defs_windows_amd64.h1
-rw-r--r--src/runtime/env_posix.go13
-rw-r--r--src/runtime/error.go22
-rw-r--r--src/runtime/heapdump.c28
-rw-r--r--src/runtime/malloc.c28
-rw-r--r--src/runtime/malloc.go2
-rw-r--r--src/runtime/malloc.h5
-rw-r--r--src/runtime/mcache.c10
-rw-r--r--src/runtime/mem.go43
-rw-r--r--src/runtime/mem_plan9.c1
-rw-r--r--src/runtime/mem_windows.c12
-rw-r--r--src/runtime/mgc0.c157
-rw-r--r--src/runtime/mheap.c4
-rw-r--r--src/runtime/os_windows.c12
-rw-r--r--src/runtime/os_windows_386.c28
-rw-r--r--src/runtime/os_windows_amd64.c25
-rw-r--r--src/runtime/panic.go80
-rw-r--r--src/runtime/proc.c164
-rw-r--r--src/runtime/proc.go8
-rw-r--r--src/runtime/rt0_linux_386.s2
-rw-r--r--src/runtime/rt0_linux_arm.s2
-rw-r--r--src/runtime/rt0_plan9_386.s8
-rw-r--r--src/runtime/rt0_plan9_amd64.s8
-rw-r--r--src/runtime/rt0_solaris_amd64.s2
-rw-r--r--src/runtime/rt0_windows_386.s2
-rw-r--r--src/runtime/rt0_windows_amd64.s2
-rw-r--r--src/runtime/runtime.c47
-rw-r--r--src/runtime/runtime.go11
-rw-r--r--src/runtime/runtime.h44
-rw-r--r--src/runtime/runtime_test.go26
-rw-r--r--src/runtime/sema.go5
-rw-r--r--src/runtime/signals_darwin.h3
-rw-r--r--src/runtime/signals_dragonfly.h3
-rw-r--r--src/runtime/signals_freebsd.h3
-rw-r--r--src/runtime/signals_linux.h3
-rw-r--r--src/runtime/signals_nacl.h3
-rw-r--r--src/runtime/signals_netbsd.h3
-rw-r--r--src/runtime/signals_openbsd.h3
-rw-r--r--src/runtime/signals_plan9.h3
-rw-r--r--src/runtime/signals_solaris.h3
-rw-r--r--src/runtime/stack.c92
-rw-r--r--src/runtime/stack.h21
-rw-r--r--src/runtime/stack_test.go4
-rw-r--r--src/runtime/stubs.go7
-rw-r--r--src/runtime/sys_dragonfly_386.s2
-rw-r--r--src/runtime/sys_freebsd_386.s2
-rw-r--r--src/runtime/sys_netbsd_386.s2
-rw-r--r--src/runtime/sys_openbsd_386.s2
-rw-r--r--src/runtime/sys_windows_386.s2
-rw-r--r--src/runtime/sys_windows_amd64.s2
-rw-r--r--src/runtime/syscall_windows_test.go6
-rw-r--r--src/runtime/thunk.s20
-rw-r--r--src/runtime/tls_arm.s6
-rw-r--r--src/runtime/traceback.go48
-rw-r--r--src/runtime/vdso_linux_amd64.c9
72 files changed, 759 insertions, 529 deletions
diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index 21065b6d6..1495246a2 100644
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -646,15 +646,13 @@ TEXT gosave<>(SB),NOSPLIT,$0
// Call fn(arg) on the scheduler stack,
// aligned appropriately for the gcc ABI.
// See cgocall.c for more details.
-TEXT runtime·asmcgocall(SB),NOSPLIT,$0-8
- GO_ARGS
+TEXT ·asmcgocall(SB),NOSPLIT,$0-8
MOVL fn+0(FP), AX
MOVL arg+4(FP), BX
CALL asmcgocall<>(SB)
RET
-TEXT runtime·asmcgocall_errno(SB),NOSPLIT,$0-12
- GO_ARGS
+TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-12
MOVL fn+0(FP), AX
MOVL arg+4(FP), BX
CALL asmcgocall<>(SB)
@@ -714,8 +712,7 @@ TEXT runtime·cgocallback(SB),NOSPLIT,$12-12
// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize)
// See cgocall.c for more details.
-TEXT runtime·cgocallback_gofunc(SB),NOSPLIT,$12-12
- GO_ARGS
+TEXT ·cgocallback_gofunc(SB),NOSPLIT,$12-12
NO_LOCAL_POINTERS
// If g is nil, Go did not create the current thread.
@@ -906,8 +903,6 @@ TEXT runtime·emptyfunc(SB),0,$0-0
TEXT runtime·abort(SB),NOSPLIT,$0-0
INT $0x3
-GLOBL runtime·tls0(SB), $32
-
// hash function using AES hardware instructions
TEXT runtime·aeshash(SB),NOSPLIT,$0-16
MOVL p+0(FP), AX // ptr to data
@@ -2280,3 +2275,13 @@ TEXT runtime·fastrand1(SB), NOSPLIT, $0-4
TEXT runtime·return0(SB), NOSPLIT, $0
MOVL $0, AX
RET
+
+// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
+// Must obey the gcc calling convention.
+TEXT _cgo_topofstack(SB),NOSPLIT,$0
+ get_tls(CX)
+ MOVL g(CX), AX
+ MOVL g_m(AX), AX
+ MOVL m_curg(AX), AX
+ MOVL (g_stack+stack_hi)(AX), AX
+ RET
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index da29f61ed..3f7f60841 100644
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -623,15 +623,13 @@ TEXT gosave<>(SB),NOSPLIT,$0
// Call fn(arg) on the scheduler stack,
// aligned appropriately for the gcc ABI.
// See cgocall.c for more details.
-TEXT runtime·asmcgocall(SB),NOSPLIT,$0-16
- GO_ARGS
+TEXT ·asmcgocall(SB),NOSPLIT,$0-16
MOVQ fn+0(FP), AX
MOVQ arg+8(FP), BX
CALL asmcgocall<>(SB)
RET
-TEXT runtime·asmcgocall_errno(SB),NOSPLIT,$0-20
- GO_ARGS
+TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-20
MOVQ fn+0(FP), AX
MOVQ arg+8(FP), BX
CALL asmcgocall<>(SB)
@@ -700,8 +698,7 @@ TEXT runtime·cgocallback(SB),NOSPLIT,$24-24
// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize)
// See cgocall.c for more details.
-TEXT runtime·cgocallback_gofunc(SB),NOSPLIT,$8-24
- GO_ARGS
+TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-24
NO_LOCAL_POINTERS
// If g is nil, Go did not create the current thread.
@@ -874,8 +871,6 @@ TEXT runtime·gocputicks(SB),NOSPLIT,$0-8
MOVQ AX, ret+0(FP)
RET
-GLOBL runtime·tls0(SB), $64
-
// hash function using AES hardware instructions
TEXT runtime·aeshash(SB),NOSPLIT,$0-32
MOVQ p+0(FP), AX // ptr to data
@@ -2225,3 +2220,14 @@ TEXT runtime·fastrand1(SB), NOSPLIT, $0-4
TEXT runtime·return0(SB), NOSPLIT, $0
MOVL $0, AX
RET
+
+
+// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
+// Must obey the gcc calling convention.
+TEXT _cgo_topofstack(SB),NOSPLIT,$0
+ get_tls(CX)
+ MOVQ g(CX), AX
+ MOVQ g_m(AX), AX
+ MOVQ m_curg(AX), AX
+ MOVQ (g_stack+stack_hi)(AX), AX
+ RET
diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s
index bbbd886a5..13a164256 100644
--- a/src/runtime/asm_amd64p32.s
+++ b/src/runtime/asm_amd64p32.s
@@ -674,8 +674,6 @@ TEXT runtime·gocputicks(SB),NOSPLIT,$0-8
MOVQ AX, ret+0(FP)
RET
-GLOBL runtime·tls0(SB), $64
-
// hash function using AES hardware instructions
// For now, our one amd64p32 system (NaCl) does not
// support using AES instructions, so have not bothered to
diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s
index 3e78d9114..36fb022f9 100644
--- a/src/runtime/asm_arm.s
+++ b/src/runtime/asm_arm.s
@@ -96,7 +96,7 @@ TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
#ifdef GOOS_nacl
WORD $0xe125be7f // BKPT 0x5bef, NACL_INSTR_ARM_BREAKPOINT
#else
- WORD $0xe1200071 // BKPT 0x0001
+ WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
#endif
RET
@@ -480,15 +480,13 @@ TEXT gosave<>(SB),NOSPLIT,$0
// Call fn(arg) on the scheduler stack,
// aligned appropriately for the gcc ABI.
// See cgocall.c for more details.
-TEXT runtime·asmcgocall(SB),NOSPLIT,$0-8
- GO_ARGS
+TEXT ·asmcgocall(SB),NOSPLIT,$0-8
MOVW fn+0(FP), R1
MOVW arg+4(FP), R0
BL asmcgocall<>(SB)
RET
-TEXT runtime·asmcgocall_errno(SB),NOSPLIT,$0-12
- GO_ARGS
+TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-12
MOVW fn+0(FP), R1
MOVW arg+4(FP), R0
BL asmcgocall<>(SB)
@@ -551,8 +549,7 @@ TEXT runtime·cgocallback(SB),NOSPLIT,$12-12
// cgocallback_gofunc(void (*fn)(void*), void *frame, uintptr framesize)
// See cgocall.c for more details.
-TEXT runtime·cgocallback_gofunc(SB),NOSPLIT,$8-12
- GO_ARGS
+TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-12
NO_LOCAL_POINTERS
// Load m and g from thread-local storage.
@@ -1303,3 +1300,20 @@ yieldloop:
RET
SUB $1, R1
B yieldloop
+
+// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
+// Must obey the gcc calling convention.
+TEXT _cgo_topofstack(SB),NOSPLIT,$8
+ // R11 and g register are clobbered by load_g. They are
+ // callee-save in the gcc calling convention, so save them here.
+ MOVW R11, saveR11-4(SP)
+ MOVW g, saveG-8(SP)
+
+ BL runtime·load_g(SB)
+ MOVW g_m(g), R0
+ MOVW m_curg(R0), R0
+ MOVW (g_stack+stack_hi)(R0), R0
+
+ MOVW saveG-8(SP), g
+ MOVW saveR11-4(SP), R11
+ RET
diff --git a/src/runtime/cgo/callbacks.c b/src/runtime/cgo/callbacks.c
index 16614d03d..282beeea8 100644
--- a/src/runtime/cgo/callbacks.c
+++ b/src/runtime/cgo/callbacks.c
@@ -78,3 +78,6 @@ void (*_cgo_free)(void*) = x_cgo_free;
#pragma cgo_import_static x_cgo_thread_start
extern void x_cgo_thread_start(void*);
void (*_cgo_thread_start)(void*) = x_cgo_thread_start;
+
+#pragma cgo_export_static _cgo_topofstack
+#pragma cgo_export_dynamic _cgo_topofstack
diff --git a/src/runtime/cgo/dragonfly.c b/src/runtime/cgo/dragonfly.c
index acf53e265..3c95ff354 100644
--- a/src/runtime/cgo/dragonfly.c
+++ b/src/runtime/cgo/dragonfly.c
@@ -2,11 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
// Supply environ and __progname, because we don't
// link against the standard DragonFly crt0.o and the
// libc dynamic library needs them.
+#pragma dataflag NOPTR
char *environ[1];
+#pragma dataflag NOPTR
char *__progname;
#pragma dynexport environ environ
diff --git a/src/runtime/cgo/freebsd.c b/src/runtime/cgo/freebsd.c
index dfcfa3a21..aefc481e6 100644
--- a/src/runtime/cgo/freebsd.c
+++ b/src/runtime/cgo/freebsd.c
@@ -2,11 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
// Supply environ and __progname, because we don't
// link against the standard FreeBSD crt0.o and the
// libc dynamic library needs them.
+#pragma dataflag NOPTR
char *environ[1];
+#pragma dataflag NOPTR
char *__progname;
#pragma dynexport environ environ
diff --git a/src/runtime/cgo/gcc_setenv.c b/src/runtime/cgo/gcc_setenv.c
index 8b128b946..af0fc5d8d 100644
--- a/src/runtime/cgo/gcc_setenv.c
+++ b/src/runtime/cgo/gcc_setenv.c
@@ -14,3 +14,10 @@ x_cgo_setenv(char **arg)
{
setenv(arg[0], arg[1], 1);
}
+
+/* Stub for calling unsetenv */
+void
+x_cgo_unsetenv(char *arg)
+{
+ unsetenv(arg);
+}
diff --git a/src/runtime/cgo/netbsd.c b/src/runtime/cgo/netbsd.c
index b6403f686..de38bb770 100644
--- a/src/runtime/cgo/netbsd.c
+++ b/src/runtime/cgo/netbsd.c
@@ -2,11 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
// Supply environ and __progname, because we don't
// link against the standard NetBSD crt0.o and the
// libc dynamic library needs them.
+#pragma dataflag NOPTR
char *environ[1];
+#pragma dataflag NOPTR
char *__progname;
#pragma dynexport environ environ
diff --git a/src/runtime/cgo/openbsd.c b/src/runtime/cgo/openbsd.c
index 84e9f9eff..7c2b6c173 100644
--- a/src/runtime/cgo/openbsd.c
+++ b/src/runtime/cgo/openbsd.c
@@ -2,11 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
// Supply environ, __progname and __guard_local, because
// we don't link against the standard OpenBSD crt0.o and
// the libc dynamic library needs them.
+#pragma dataflag NOPTR
char *environ[1];
+#pragma dataflag NOPTR
char *__progname;
long __guard_local;
diff --git a/src/runtime/cgo/setenv.c b/src/runtime/cgo/setenv.c
index ee529904f..76d88cbf1 100644
--- a/src/runtime/cgo/setenv.c
+++ b/src/runtime/cgo/setenv.c
@@ -5,6 +5,9 @@
// +build darwin dragonfly freebsd linux netbsd openbsd
#pragma cgo_import_static x_cgo_setenv
+#pragma cgo_import_static x_cgo_unsetenv
void x_cgo_setenv(char**);
void (*runtime·_cgo_setenv)(char**) = x_cgo_setenv;
+void x_cgo_unsetenv(char**);
+void (*runtime·_cgo_unsetenv)(char**) = x_cgo_unsetenv;
diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go
index a21474b01..7fd91469e 100644
--- a/src/runtime/cgocall.go
+++ b/src/runtime/cgocall.go
@@ -177,14 +177,22 @@ func cfree(p unsafe.Pointer) {
// Call from C back to Go.
//go:nosplit
func cgocallbackg() {
- if gp := getg(); gp != gp.m.curg {
+ gp := getg()
+ if gp != gp.m.curg {
println("runtime: bad g in cgocallback")
exit(2)
}
+ // entersyscall saves the caller's SP to allow the GC to trace the Go
+ // stack. However, since we're returning to an earlier stack frame and
+ // need to pair with the entersyscall() call made by cgocall, we must
+ // save syscall* and let reentersyscall restore them.
+ savedsp := unsafe.Pointer(gp.syscallsp)
+ savedpc := gp.syscallpc
exitsyscall() // coming out of cgo call
cgocallbackg1()
- entersyscall() // going back to cgo call
+ // going back to cgo call
+ reentersyscall(savedpc, savedsp)
}
func cgocallbackg1() {
diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go
index d1577fb5f..783b4c48f 100644
--- a/src/runtime/crash_test.go
+++ b/src/runtime/crash_test.go
@@ -412,3 +412,91 @@ func main() {
runtime.Breakpoint()
}
`
+
+func TestGoexitInPanic(t *testing.T) {
+ // see issue 8774: this code used to trigger an infinite recursion
+ output := executeTest(t, goexitInPanicSource, nil)
+ want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
+ if !strings.HasPrefix(output, want) {
+ t.Fatalf("output does not start with %q:\n%s", want, output)
+ }
+}
+
+const goexitInPanicSource = `
+package main
+import "runtime"
+func main() {
+ go func() {
+ defer func() {
+ runtime.Goexit()
+ }()
+ panic("hello")
+ }()
+ runtime.Goexit()
+}
+`
+
+func TestPanicAfterGoexit(t *testing.T) {
+ // an uncaught panic should still work after goexit
+ output := executeTest(t, panicAfterGoexitSource, nil)
+ want := "panic: hello"
+ if !strings.HasPrefix(output, want) {
+ t.Fatalf("output does not start with %q:\n%s", want, output)
+ }
+}
+
+const panicAfterGoexitSource = `
+package main
+import "runtime"
+func main() {
+ defer func() {
+ panic("hello")
+ }()
+ runtime.Goexit()
+}
+`
+
+func TestRecoveredPanicAfterGoexit(t *testing.T) {
+ output := executeTest(t, recoveredPanicAfterGoexitSource, nil)
+ want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
+ if !strings.HasPrefix(output, want) {
+ t.Fatalf("output does not start with %q:\n%s", want, output)
+ }
+}
+
+const recoveredPanicAfterGoexitSource = `
+package main
+import "runtime"
+func main() {
+ defer func() {
+ defer func() {
+ r := recover()
+ if r == nil {
+ panic("bad recover")
+ }
+ }()
+ panic("hello")
+ }()
+ runtime.Goexit()
+}
+`
+
+func TestRecoverBeforePanicAfterGoexit(t *testing.T) {
+ // 1. defer a function that recovers
+ // 2. defer a function that panics
+ // 3. call goexit
+ // Goexit should run the #2 defer. Its panic
+ // should be caught by the #1 defer, and execution
+ // should resume in the caller. Like the Goexit
+ // never happened!
+ defer func() {
+ r := recover()
+ if r == nil {
+ panic("bad recover")
+ }
+ }()
+ defer func() {
+ panic("hello")
+ }()
+ runtime.Goexit()
+}
diff --git a/src/runtime/debug.go b/src/runtime/debug.go
index bb4bd60ed..4414dd55d 100644
--- a/src/runtime/debug.go
+++ b/src/runtime/debug.go
@@ -24,15 +24,29 @@ func UnlockOSThread()
// The number of logical CPUs on the local machine can be queried with NumCPU.
// This call will go away when the scheduler improves.
func GOMAXPROCS(n int) int {
- g := getg()
- g.m.scalararg[0] = uintptr(n)
- onM(gomaxprocs_m)
- n = int(g.m.scalararg[0])
- g.m.scalararg[0] = 0
- return n
-}
+ if n > _MaxGomaxprocs {
+ n = _MaxGomaxprocs
+ }
+ lock(&sched.lock)
+ ret := int(gomaxprocs)
+ unlock(&sched.lock)
+ if n <= 0 || n == ret {
+ return ret
+ }
-func gomaxprocs_m() // proc.c
+ semacquire(&worldsema, false)
+ gp := getg()
+ gp.m.gcing = 1
+ onM(stoptheworld)
+
+ // newprocs will be processed by starttheworld
+ newprocs = int32(n)
+
+ gp.m.gcing = 0
+ semrelease(&worldsema)
+ onM(starttheworld)
+ return ret
+}
// NumCPU returns the number of logical CPUs on the local machine.
func NumCPU() int {
diff --git a/src/runtime/debug/debug.c b/src/runtime/debug/debug.s
index a7292c477..a7292c477 100644
--- a/src/runtime/debug/debug.c
+++ b/src/runtime/debug/debug.s
diff --git a/src/runtime/defs_windows.go b/src/runtime/defs_windows.go
index 01aea92de..cb0f54d8a 100644
--- a/src/runtime/defs_windows.go
+++ b/src/runtime/defs_windows.go
@@ -49,7 +49,6 @@ const (
CONTEXT_FULL = C.CONTEXT_FULL
EXCEPTION_ACCESS_VIOLATION = C.STATUS_ACCESS_VIOLATION
- EXCEPTION_BREAKPOINT = C.STATUS_BREAKPOINT
EXCEPTION_FLT_DENORMAL_OPERAND = C.STATUS_FLOAT_DENORMAL_OPERAND
EXCEPTION_FLT_DIVIDE_BY_ZERO = C.STATUS_FLOAT_DIVIDE_BY_ZERO
EXCEPTION_FLT_INEXACT_RESULT = C.STATUS_FLOAT_INEXACT_RESULT
diff --git a/src/runtime/defs_windows_386.h b/src/runtime/defs_windows_386.h
index db3629a1d..295e422c6 100644
--- a/src/runtime/defs_windows_386.h
+++ b/src/runtime/defs_windows_386.h
@@ -22,7 +22,6 @@ enum {
CONTEXT_FULL = 0x10007,
EXCEPTION_ACCESS_VIOLATION = 0xc0000005,
- EXCEPTION_BREAKPOINT = 0x80000003,
EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d,
EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e,
EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f,
diff --git a/src/runtime/defs_windows_amd64.h b/src/runtime/defs_windows_amd64.h
index fe26f5a84..2516c8412 100644
--- a/src/runtime/defs_windows_amd64.h
+++ b/src/runtime/defs_windows_amd64.h
@@ -22,7 +22,6 @@ enum {
CONTEXT_FULL = 0x10000b,
EXCEPTION_ACCESS_VIOLATION = 0xc0000005,
- EXCEPTION_BREAKPOINT = 0x80000003,
EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d,
EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e,
EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f,
diff --git a/src/runtime/env_posix.go b/src/runtime/env_posix.go
index 6c04f6cc7..dd57872d7 100644
--- a/src/runtime/env_posix.go
+++ b/src/runtime/env_posix.go
@@ -32,7 +32,8 @@ func gogetenv(key string) string {
return ""
}
-var _cgo_setenv uintptr // pointer to C function
+var _cgo_setenv uintptr // pointer to C function
+var _cgo_unsetenv uintptr // pointer to C function
// Update the C environment if cgo is loaded.
// Called from syscall.Setenv.
@@ -44,6 +45,16 @@ func syscall_setenv_c(k string, v string) {
asmcgocall(unsafe.Pointer(_cgo_setenv), unsafe.Pointer(&arg))
}
+// Update the C environment if cgo is loaded.
+// Called from syscall.unsetenv.
+func syscall_unsetenv_c(k string) {
+ if _cgo_unsetenv == 0 {
+ return
+ }
+ arg := [1]unsafe.Pointer{cstring(k)}
+ asmcgocall(unsafe.Pointer(_cgo_unsetenv), unsafe.Pointer(&arg))
+}
+
func cstring(s string) unsafe.Pointer {
p := make([]byte, len(s)+1)
sp := (*_string)(unsafe.Pointer(&s))
diff --git a/src/runtime/error.go b/src/runtime/error.go
index 3ea93680c..0b40c702b 100644
--- a/src/runtime/error.go
+++ b/src/runtime/error.go
@@ -71,28 +71,6 @@ func (e errorString) Error() string {
return "runtime error: " + string(e)
}
-// For calling from C.
-func newErrorString(s string, ret *interface{}) {
- *ret = errorString(s)
-}
-
-// An errorCString represents a runtime error described by a single C string.
-// Not "type errorCString unsafe.Pointer" because of http://golang.org/issue/7084.
-// Not uintptr because we want to avoid an allocation if interfaces can't hold
-// uintptrs directly (and cstr _is_ a pointer).
-type errorCString struct{ cstr unsafe.Pointer }
-
-func (e errorCString) RuntimeError() {}
-
-func (e errorCString) Error() string {
- return "runtime error: " + gostringnocopy((*byte)(e.cstr))
-}
-
-// For calling from C.
-func newErrorCString(s unsafe.Pointer, ret *interface{}) {
- *ret = errorCString{s}
-}
-
type stringer interface {
String() string
}
diff --git a/src/runtime/heapdump.c b/src/runtime/heapdump.c
index 8bbc7d8a5..54b9666b5 100644
--- a/src/runtime/heapdump.c
+++ b/src/runtime/heapdump.c
@@ -59,6 +59,8 @@ static BitVector makeheapobjbv(byte *p, uintptr size);
// fd to write the dump to.
static uintptr dumpfd;
+
+#pragma dataflag NOPTR /* tmpbuf not a heap pointer at least */
static byte *tmpbuf;
static uintptr tmpbufsize;
@@ -109,6 +111,7 @@ typedef struct TypeCacheBucket TypeCacheBucket;
struct TypeCacheBucket {
Type *t[TypeCacheAssoc];
};
+#pragma dataflag NOPTR /* only initialized and used while world is stopped */
static TypeCacheBucket typecache[TypeCacheBuckets];
// dump a uint64 in a varint format parseable by encoding/binary
@@ -737,33 +740,16 @@ mdump(void)
flush();
}
-static void writeheapdump_m(void);
-
-#pragma textflag NOSPLIT
void
-runtime∕debug·WriteHeapDump(uintptr fd)
-{
- void (*fn)(void);
-
- g->m->scalararg[0] = fd;
- fn = writeheapdump_m;
- runtime·onM(&fn);
-}
-
-static void
-writeheapdump_m(void)
+runtime·writeheapdump_m(void)
{
uintptr fd;
fd = g->m->scalararg[0];
g->m->scalararg[0] = 0;
- // Stop the world.
runtime·casgstatus(g->m->curg, Grunning, Gwaiting);
g->waitreason = runtime·gostringnocopy((byte*)"dumping heap");
- runtime·semacquire(&runtime·worldsema, false);
- g->m->gcing = 1;
- runtime·stoptheworld();
// Update stats so we can dump them.
// As a side effect, flushes all the MCaches so the MSpan.freelist
@@ -784,13 +770,7 @@ writeheapdump_m(void)
tmpbufsize = 0;
}
- // Start up the world again.
- g->m->gcing = 0;
- g->m->locks++;
- runtime·semrelease(&runtime·worldsema);
- runtime·starttheworld();
runtime·casgstatus(g->m->curg, Gwaiting, Grunning);
- g->m->locks--;
}
// dumpint() the kind & offset of each field in an object.
diff --git a/src/runtime/malloc.c b/src/runtime/malloc.c
index cfb698ac2..b79c30b72 100644
--- a/src/runtime/malloc.c
+++ b/src/runtime/malloc.c
@@ -79,6 +79,8 @@ runtime·purgecachedstats(MCache *c)
h = &runtime·mheap;
mstats.heap_alloc += c->local_cachealloc;
c->local_cachealloc = 0;
+ mstats.tinyallocs += c->local_tinyallocs;
+ c->local_tinyallocs = 0;
mstats.nlookup += c->local_nlookup;
c->local_nlookup = 0;
h->largefree += c->local_largefree;
@@ -92,9 +94,10 @@ runtime·purgecachedstats(MCache *c)
}
// Size of the trailing by_size array differs between Go and C,
+// and all data after by_size is local to C, not exported to Go.
// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
// sizeof_C_MStats is what C thinks about size of Go struct.
-uintptr runtime·sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeof(mstats.by_size[0]);
+uintptr runtime·sizeof_C_MStats = offsetof(MStats, by_size[61]);
#define MaxArena32 (2U<<30)
@@ -326,29 +329,6 @@ runtime·MHeap_SysAlloc(MHeap *h, uintptr n)
return p;
}
-// Runtime stubs.
-
-static void*
-cnew(Type *typ, intgo n)
-{
- if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size))
- runtime·panicstring("runtime: allocation size out of range");
- return runtime·mallocgc(typ->size*n, typ, typ->kind&KindNoPointers ? FlagNoScan : 0);
-}
-
-// same as runtime·new, but callable from C
-void*
-runtime·cnew(Type *typ)
-{
- return cnew(typ, 1);
-}
-
-void*
-runtime·cnewarray(Type *typ, intgo n)
-{
- return cnew(typ, n);
-}
-
void
runtime·setFinalizer_m(void)
{
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index acf6b48f8..fc22cc29e 100644
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -103,7 +103,6 @@ func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer {
// standalone escaping variables. On a json benchmark
// the allocator reduces number of allocations by ~12% and
// reduces heap size by ~20%.
-
tinysize := uintptr(c.tinysize)
if size <= tinysize {
tiny := unsafe.Pointer(c.tiny)
@@ -121,6 +120,7 @@ func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer {
x = tiny
c.tiny = (*byte)(add(x, size))
c.tinysize -= uintptr(size1)
+ c.local_tinyallocs++
if debugMalloc {
mp := acquirem()
if mp.mallocing == 0 {
diff --git a/src/runtime/malloc.h b/src/runtime/malloc.h
index 413870c9f..520c783df 100644
--- a/src/runtime/malloc.h
+++ b/src/runtime/malloc.h
@@ -279,6 +279,8 @@ struct MStats
uint64 nmalloc;
uint64 nfree;
} by_size[NumSizeClasses];
+
+ uint64 tinyallocs; // number of tiny allocations that didn't cause actual allocation; not exported to Go directly
};
#define mstats runtime·memstats
@@ -332,6 +334,7 @@ struct MCache
// See "Tiny allocator" comment in malloc.goc.
byte* tiny;
uintptr tinysize;
+ uintptr local_tinyallocs; // number of tiny allocs not counted in other stats
// The rest is not accessed on every malloc.
MSpan* alloc[NumSizeClasses]; // spans to allocate from
@@ -527,8 +530,6 @@ uintptr runtime·sweepone(void);
void runtime·markspan(void *v, uintptr size, uintptr n, bool leftover);
void runtime·unmarkspan(void *v, uintptr size);
void runtime·purgecachedstats(MCache*);
-void* runtime·cnew(Type*);
-void* runtime·cnewarray(Type*, intgo);
void runtime·tracealloc(void*, uintptr, Type*);
void runtime·tracefree(void*, uintptr);
void runtime·tracegc(void);
diff --git a/src/runtime/mcache.c b/src/runtime/mcache.c
index bb1fc5403..5fdbe3266 100644
--- a/src/runtime/mcache.c
+++ b/src/runtime/mcache.c
@@ -13,7 +13,7 @@
extern volatile intgo runtime·MemProfileRate;
// dummy MSpan that contains no free objects.
-static MSpan emptymspan;
+MSpan runtime·emptymspan;
MCache*
runtime·allocmcache(void)
@@ -27,7 +27,7 @@ runtime·allocmcache(void)
runtime·unlock(&runtime·mheap.lock);
runtime·memclr((byte*)c, sizeof(*c));
for(i = 0; i < NumSizeClasses; i++)
- c->alloc[i] = &emptymspan;
+ c->alloc[i] = &runtime·emptymspan;
// Set first allocation sample size.
rate = runtime·MemProfileRate;
@@ -83,7 +83,7 @@ runtime·MCache_Refill(MCache *c, int32 sizeclass)
s = c->alloc[sizeclass];
if(s->freelist != nil)
runtime·throw("refill on a nonempty span");
- if(s != &emptymspan)
+ if(s != &runtime·emptymspan)
s->incache = false;
// Get a new cached span from the central lists.
@@ -107,9 +107,9 @@ runtime·MCache_ReleaseAll(MCache *c)
for(i=0; i<NumSizeClasses; i++) {
s = c->alloc[i];
- if(s != &emptymspan) {
+ if(s != &runtime·emptymspan) {
runtime·MCentral_UncacheSpan(&runtime·mheap.central[i].mcentral, s);
- c->alloc[i] = &emptymspan;
+ c->alloc[i] = &runtime·emptymspan;
}
}
}
diff --git a/src/runtime/mem.go b/src/runtime/mem.go
index 34391b2eb..438f22ec0 100644
--- a/src/runtime/mem.go
+++ b/src/runtime/mem.go
@@ -41,8 +41,8 @@ type MemStats struct {
OtherSys uint64 // other system allocations
// Garbage collector statistics.
- NextGC uint64 // next run in HeapAlloc time (bytes)
- LastGC uint64 // last run in absolute time (ns)
+ NextGC uint64 // next collection will happen when HeapAlloc ≥ this amount
+ LastGC uint64 // end time of last collection (nanoseconds since 1970)
PauseTotalNs uint64
PauseNs [256]uint64 // circular buffer of recent GC pause times, most recent at [(NumGC+255)%256]
NumGC uint32
@@ -64,9 +64,44 @@ func init() {
var memStats MemStats
if sizeof_C_MStats != unsafe.Sizeof(memStats) {
println(sizeof_C_MStats, unsafe.Sizeof(memStats))
- panic("MStats vs MemStatsType size mismatch")
+ gothrow("MStats vs MemStatsType size mismatch")
}
}
// ReadMemStats populates m with memory allocator statistics.
-func ReadMemStats(m *MemStats)
+func ReadMemStats(m *MemStats) {
+ // Have to acquire worldsema to stop the world,
+ // because stoptheworld can only be used by
+ // one goroutine at a time, and there might be
+ // a pending garbage collection already calling it.
+ semacquire(&worldsema, false)
+ gp := getg()
+ gp.m.gcing = 1
+ onM(stoptheworld)
+
+ gp.m.ptrarg[0] = noescape(unsafe.Pointer(m))
+ onM(readmemstats_m)
+
+ gp.m.gcing = 0
+ gp.m.locks++
+ semrelease(&worldsema)
+ onM(starttheworld)
+ gp.m.locks--
+}
+
+// Implementation of runtime/debug.WriteHeapDump
+func writeHeapDump(fd uintptr) {
+ semacquire(&worldsema, false)
+ gp := getg()
+ gp.m.gcing = 1
+ onM(stoptheworld)
+
+ gp.m.scalararg[0] = fd
+ onM(writeheapdump_m)
+
+ gp.m.gcing = 0
+ gp.m.locks++
+ semrelease(&worldsema)
+ onM(starttheworld)
+ gp.m.locks--
+}
diff --git a/src/runtime/mem_plan9.c b/src/runtime/mem_plan9.c
index 402869f39..d673d6f83 100644
--- a/src/runtime/mem_plan9.c
+++ b/src/runtime/mem_plan9.c
@@ -10,6 +10,7 @@
#include "textflag.h"
extern byte runtime·end[];
+#pragma dataflag NOPTR
static byte *bloc = { runtime·end };
static Mutex memlock;
diff --git a/src/runtime/mem_windows.c b/src/runtime/mem_windows.c
index 7bc028bf3..6ea992020 100644
--- a/src/runtime/mem_windows.c
+++ b/src/runtime/mem_windows.c
@@ -68,10 +68,22 @@ void
runtime·SysUsed(void *v, uintptr n)
{
void *r;
+ uintptr small;
r = runtime·stdcall4(runtime·VirtualAlloc, (uintptr)v, n, MEM_COMMIT, PAGE_READWRITE);
if(r != v)
runtime·throw("runtime: failed to commit pages");
+
+ // Commit failed. See SysUnused.
+ while(n > 0) {
+ small = n;
+ while(small >= 4096 && runtime·stdcall4(runtime·VirtualAlloc, (uintptr)v, small, MEM_COMMIT, PAGE_READWRITE) == nil)
+ small = (small / 2) & ~(4096-1);
+ if(small < 4096)
+ runtime·throw("runtime: failed to decommit pages");
+ v = (byte*)v + small;
+ n -= small;
+ }
}
void
diff --git a/src/runtime/mgc0.c b/src/runtime/mgc0.c
index b4cd3474d..39fae9bbe 100644
--- a/src/runtime/mgc0.c
+++ b/src/runtime/mgc0.c
@@ -119,7 +119,7 @@ FinBlock* runtime·finc; // cache of free blocks
static byte finptrmask[FinBlockSize/PtrSize/PointersPerByte];
bool runtime·fingwait;
bool runtime·fingwake;
-static FinBlock *allfin; // list of all blocks
+FinBlock *runtime·allfin; // list of all blocks
BitVector runtime·gcdatamask;
BitVector runtime·gcbssmask;
@@ -146,7 +146,8 @@ static void slottombits(byte*, Markbits*);
void runtime·bgsweep(void);
static FuncVal bgsweepv = {runtime·bgsweep};
-static struct {
+typedef struct WorkData WorkData;
+struct WorkData {
uint64 full; // lock-free list of full blocks
uint64 empty; // lock-free list of empty blocks
byte pad0[CacheLineSize]; // prevents false-sharing between full/empty and nproc/nwait
@@ -160,7 +161,8 @@ static struct {
// Copy of mheap.allspans for marker or sweeper.
MSpan** spans;
uint32 nspan;
-} work;
+};
+WorkData runtime·work;
// Is address b in the known heap. If it doesn't have a valid gcmap
// returns false. For example pointers into stacks will return false.
@@ -286,8 +288,7 @@ greyobject(byte *obj, Markbits *mbits, Workbuf *wbuf)
// but the object it shares the byte with is already marked,
// then all the possible concurrent updates are trying to set the same bit,
// so we can use a non-atomic update.
- if((mbits->xbits&(bitMask|(bitMask<<gcBits))) != (bitBoundary|(bitBoundary<<gcBits)) ||
- work.nproc == 1)
+ if((mbits->xbits&(bitMask|(bitMask<<gcBits))) != (bitBoundary|(bitBoundary<<gcBits)) || runtime·work.nproc == 1)
*mbits->bitp = mbits->xbits | (bitMarked<<mbits->shift);
else
runtime·atomicor8(mbits->bitp, bitMarked<<mbits->shift);
@@ -424,7 +425,7 @@ scanblock(byte *b, uintptr n, byte *ptrmask)
}
// If another proc wants a pointer, give it some.
- if(work.nwait > 0 && wbuf->nobj > 4 && work.full == 0) {
+ if(runtime·work.nwait > 0 && wbuf->nobj > 4 && runtime·work.full == 0) {
wbuf = handoff(wbuf);
}
@@ -461,18 +462,18 @@ markroot(ParFor *desc, uint32 i)
break;
case RootFinalizers:
- for(fb=allfin; fb; fb=fb->alllink)
+ for(fb=runtime·allfin; fb; fb=fb->alllink)
scanblock((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), finptrmask);
break;
case RootSpans:
// mark MSpan.specials
sg = runtime·mheap.sweepgen;
- for(spanidx=0; spanidx<work.nspan; spanidx++) {
+ for(spanidx=0; spanidx<runtime·work.nspan; spanidx++) {
Special *sp;
SpecialFinalizer *spf;
- s = work.spans[spanidx];
+ s = runtime·work.spans[spanidx];
if(s->state != MSpanInUse)
continue;
if(s->sweepgen != sg) {
@@ -507,7 +508,7 @@ markroot(ParFor *desc, uint32 i)
// needed only to output in traceback
status = runtime·readgstatus(gp); // We are not in a scan state
if((status == Gwaiting || status == Gsyscall) && gp->waitsince == 0)
- gp->waitsince = work.tstart;
+ gp->waitsince = runtime·work.tstart;
// Shrink a stack if not much of it is being used.
runtime·shrinkstack(gp);
if(runtime·readgstatus(gp) == Gdead)
@@ -554,7 +555,7 @@ getempty(Workbuf *b)
MCache *c;
if(b != nil)
- runtime·lfstackpush(&work.full, &b->node);
+ runtime·lfstackpush(&runtime·work.full, &b->node);
b = nil;
c = g->m->mcache;
if(c->gcworkbuf != nil) {
@@ -562,7 +563,7 @@ getempty(Workbuf *b)
c->gcworkbuf = nil;
}
if(b == nil)
- b = (Workbuf*)runtime·lfstackpop(&work.empty);
+ b = (Workbuf*)runtime·lfstackpop(&runtime·work.empty);
if(b == nil) {
b = runtime·persistentalloc(sizeof(*b), CacheLineSize, &mstats.gc_sys);
b->nobj = 0;
@@ -585,7 +586,7 @@ putempty(Workbuf *b)
c->gcworkbuf = b;
return;
}
- runtime·lfstackpush(&work.empty, &b->node);
+ runtime·lfstackpush(&runtime·work.empty, &b->node);
}
// Get an partially empty work buffer from the mcache structure
@@ -619,7 +620,7 @@ putpartial(Workbuf *b)
runtime·throw("putpartial: c->gcworkbuf is not nil\n");
- runtime·lfstackpush(&work.full, &b->node);
+ runtime·lfstackpush(&runtime·work.full, &b->node);
}
void
@@ -650,22 +651,22 @@ getfull(Workbuf *b)
if(b != nil) {
if(b->nobj != 0)
runtime·printf("runtime:getfull: b->nobj=%D not 0.", b->nobj);
- runtime·lfstackpush(&work.empty, &b->node);
+ runtime·lfstackpush(&runtime·work.empty, &b->node);
}
- b = (Workbuf*)runtime·lfstackpop(&work.full);
- if(b != nil || work.nproc == 1)
+ b = (Workbuf*)runtime·lfstackpop(&runtime·work.full);
+ if(b != nil || runtime·work.nproc == 1)
return b;
- runtime·xadd(&work.nwait, +1);
+ runtime·xadd(&runtime·work.nwait, +1);
for(i=0;; i++) {
- if(work.full != 0) {
- runtime·xadd(&work.nwait, -1);
- b = (Workbuf*)runtime·lfstackpop(&work.full);
+ if(runtime·work.full != 0) {
+ runtime·xadd(&runtime·work.nwait, -1);
+ b = (Workbuf*)runtime·lfstackpop(&runtime·work.full);
if(b != nil)
return b;
- runtime·xadd(&work.nwait, +1);
+ runtime·xadd(&runtime·work.nwait, +1);
}
- if(work.nwait == work.nproc)
+ if(runtime·work.nwait == runtime·work.nproc)
return nil;
if(i < 10) {
g->m->gcstats.nprocyield++;
@@ -696,7 +697,7 @@ handoff(Workbuf *b)
g->m->gcstats.nhandoffcnt += n;
// Put b on full list - let first half of b get stolen.
- runtime·lfstackpush(&work.full, &b->node);
+ runtime·lfstackpush(&runtime·work.full, &b->node);
return b1;
}
@@ -856,6 +857,7 @@ runtime·gcphasework(G *gp)
gp->gcworkdone = true;
}
+#pragma dataflag NOPTR
static byte finalizer1[] = {
// Each Finalizer is 5 words, ptr ptr uintptr ptr ptr.
// Each byte describes 4 words.
@@ -890,8 +892,8 @@ runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType
if(runtime·finc == nil) {
runtime·finc = runtime·persistentalloc(FinBlockSize, 0, &mstats.gc_sys);
runtime·finc->cap = (FinBlockSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1;
- runtime·finc->alllink = allfin;
- allfin = runtime·finc;
+ runtime·finc->alllink = runtime·allfin;
+ runtime·allfin = runtime·finc;
if(finptrmask[0] == 0) {
// Build pointer mask for Finalizer array in block.
// Check assumptions made in finalizer1 array above.
@@ -931,7 +933,7 @@ runtime·iterate_finq(void (*callback)(FuncVal*, byte*, uintptr, Type*, PtrType*
Finalizer *f;
uintptr i;
- for(fb = allfin; fb; fb = fb->alllink) {
+ for(fb = runtime·allfin; fb; fb = fb->alllink) {
for(i = 0; i < fb->cnt; i++) {
f = &fb->fin[i];
callback(f->fn, f->arg, f->nret, f->fint, f->ot);
@@ -1155,8 +1157,8 @@ runtime·MSpan_Sweep(MSpan *s, bool preserve)
// State of background runtime·sweep.
// Protected by runtime·gclock.
-// Must match mgc0.go.
-struct
+typedef struct SweepData SweepData;
+struct SweepData
{
G* g;
bool parked;
@@ -1165,7 +1167,8 @@ struct
uint32 nbgsweep;
uint32 npausesweep;
-} runtime·sweep;
+};
+SweepData runtime·sweep;
// sweeps one span
// returns number of pages returned to heap, or -1 if there is nothing to sweep
@@ -1182,12 +1185,12 @@ runtime·sweepone(void)
sg = runtime·mheap.sweepgen;
for(;;) {
idx = runtime·xadd(&runtime·sweep.spanidx, 1) - 1;
- if(idx >= work.nspan) {
+ if(idx >= runtime·work.nspan) {
runtime·mheap.sweepdone = true;
g->m->locks--;
return -1;
}
- s = work.spans[idx];
+ s = runtime·work.spans[idx];
if(s->state != MSpanInUse) {
s->sweepgen = sg;
continue;
@@ -1236,12 +1239,12 @@ runtime·gchelper(void)
gchelperstart();
// parallel mark for over gc roots
- runtime·parfordo(work.markfor);
+ runtime·parfordo(runtime·work.markfor);
if(runtime·gcphase != GCscan)
scanblock(nil, 0, nil); // blocks in getfull
- nproc = work.nproc; // work.nproc can change right after we increment work.ndone
- if(runtime·xadd(&work.ndone, +1) == nproc-1)
- runtime·notewakeup(&work.alldone);
+ nproc = runtime·work.nproc; // work.nproc can change right after we increment work.ndone
+ if(runtime·xadd(&runtime·work.ndone, +1) == nproc-1)
+ runtime·notewakeup(&runtime·work.alldone);
g->m->traceback = 0;
}
@@ -1361,6 +1364,7 @@ runtime·updatememstats(GCStats *stats)
mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i];
smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i];
}
+ mstats.nfree += mstats.tinyallocs;
mstats.nmalloc += mstats.nfree;
// Calculate derived stats.
@@ -1378,7 +1382,6 @@ struct gc_args
};
static void gc(struct gc_args *args);
-static void mgc(G *gp);
int32
runtime·readgogc(void)
@@ -1399,7 +1402,7 @@ runtime·gcinit(void)
if(sizeof(Workbuf) != WorkbufSize)
runtime·throw("runtime: size of Workbuf is suboptimal");
- work.markfor = runtime·parforalloc(MaxGcproc);
+ runtime·work.markfor = runtime·parforalloc(MaxGcproc);
runtime·gcpercent = runtime·readgogc();
runtime·gcdatamask = unrollglobgcprog(runtime·gcdata, runtime·edata - runtime·data);
runtime·gcbssmask = unrollglobgcprog(runtime·gcbss, runtime·ebss - runtime·bss);
@@ -1436,7 +1439,7 @@ gc(struct gc_args *args)
g->m->traceback = 2;
t0 = args->start_time;
- work.tstart = args->start_time;
+ runtime·work.tstart = args->start_time;
t1 = 0;
if(runtime·debug.gctrace)
@@ -1456,24 +1459,24 @@ gc(struct gc_args *args)
// Even if this is stop-the-world, a concurrent exitsyscall can allocate a stack from heap.
runtime·lock(&runtime·mheap.lock);
// Free the old cached sweep array if necessary.
- if(work.spans != nil && work.spans != runtime·mheap.allspans)
- runtime·SysFree(work.spans, work.nspan*sizeof(work.spans[0]), &mstats.other_sys);
+ if(runtime·work.spans != nil && runtime·work.spans != runtime·mheap.allspans)
+ runtime·SysFree(runtime·work.spans, runtime·work.nspan*sizeof(runtime·work.spans[0]), &mstats.other_sys);
// Cache the current array for marking.
runtime·mheap.gcspans = runtime·mheap.allspans;
- work.spans = runtime·mheap.allspans;
- work.nspan = runtime·mheap.nspan;
+ runtime·work.spans = runtime·mheap.allspans;
+ runtime·work.nspan = runtime·mheap.nspan;
runtime·unlock(&runtime·mheap.lock);
oldphase = runtime·gcphase;
- work.nwait = 0;
- work.ndone = 0;
- work.nproc = runtime·gcprocs();
+ runtime·work.nwait = 0;
+ runtime·work.ndone = 0;
+ runtime·work.nproc = runtime·gcprocs();
runtime·gcphase = GCmark; //^^ vv
- runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allglen, nil, false, markroot);
- if(work.nproc > 1) {
- runtime·noteclear(&work.alldone);
- runtime·helpgc(work.nproc);
+ runtime·parforsetup(runtime·work.markfor, runtime·work.nproc, RootCount + runtime·allglen, nil, false, markroot);
+ if(runtime·work.nproc > 1) {
+ runtime·noteclear(&runtime·work.alldone);
+ runtime·helpgc(runtime·work.nproc);
}
t2 = 0;
@@ -1481,7 +1484,7 @@ gc(struct gc_args *args)
t2 = runtime·nanotime();
gchelperstart();
- runtime·parfordo(work.markfor);
+ runtime·parfordo(runtime·work.markfor);
scanblock(nil, 0, nil);
runtime·gcphase = oldphase; //^^ vv
@@ -1489,8 +1492,8 @@ gc(struct gc_args *args)
if(runtime·debug.gctrace)
t3 = runtime·nanotime();
- if(work.nproc > 1)
- runtime·notesleep(&work.alldone);
+ if(runtime·work.nproc > 1)
+ runtime·notesleep(&runtime·work.alldone);
cachestats();
// next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
@@ -1517,19 +1520,21 @@ gc(struct gc_args *args)
}
obj = mstats.nmalloc - mstats.nfree;
- stats.nprocyield += work.markfor->nprocyield;
- stats.nosyield += work.markfor->nosyield;
- stats.nsleep += work.markfor->nsleep;
+ stats.nprocyield += runtime·work.markfor->nprocyield;
+ stats.nosyield += runtime·work.markfor->nosyield;
+ stats.nsleep += runtime·work.markfor->nsleep;
runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects,"
+ " %d goroutines,"
" %d/%d/%d sweeps,"
" %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
- mstats.numgc, work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t3-t2)/1000, (t4-t3)/1000,
+ mstats.numgc, runtime·work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t3-t2)/1000, (t4-t3)/1000,
heap0>>20, heap1>>20, obj,
mstats.nmalloc, mstats.nfree,
- work.nspan, runtime·sweep.nbgsweep, runtime·sweep.npausesweep,
+ runtime·gcount(),
+ runtime·work.nspan, runtime·sweep.nbgsweep, runtime·sweep.npausesweep,
stats.nhandoff, stats.nhandoffcnt,
- work.markfor->nsteal, work.markfor->nstealcnt,
+ runtime·work.markfor->nsteal, runtime·work.markfor->nstealcnt,
stats.nprocyield, stats.nosyield, stats.nsleep);
runtime·sweep.nbgsweep = runtime·sweep.npausesweep = 0;
}
@@ -1538,14 +1543,14 @@ gc(struct gc_args *args)
// Even if this is still stop-the-world, a concurrent exitsyscall can allocate a stack from heap.
runtime·lock(&runtime·mheap.lock);
// Free the old cached mark array if necessary.
- if(work.spans != nil && work.spans != runtime·mheap.allspans)
- runtime·SysFree(work.spans, work.nspan*sizeof(work.spans[0]), &mstats.other_sys);
+ if(runtime·work.spans != nil && runtime·work.spans != runtime·mheap.allspans)
+ runtime·SysFree(runtime·work.spans, runtime·work.nspan*sizeof(runtime·work.spans[0]), &mstats.other_sys);
// Cache the current array for sweeping.
runtime·mheap.gcspans = runtime·mheap.allspans;
runtime·mheap.sweepgen += 2;
runtime·mheap.sweepdone = false;
- work.spans = runtime·mheap.allspans;
- work.nspan = runtime·mheap.nspan;
+ runtime·work.spans = runtime·mheap.allspans;
+ runtime·work.nspan = runtime·mheap.nspan;
runtime·sweep.spanidx = 0;
runtime·unlock(&runtime·mheap.lock);
@@ -1573,32 +1578,14 @@ extern uintptr runtime·sizeof_C_MStats;
static void readmemstats_m(void);
-#pragma textflag NOSPLIT
void
-runtime·ReadMemStats(MStats *stats)
-{
- void (*fn)(void);
-
- g->m->ptrarg[0] = stats;
- fn = readmemstats_m;
- runtime·onM(&fn);
-}
-
-static void
-readmemstats_m(void)
+runtime·readmemstats_m(void)
{
MStats *stats;
stats = g->m->ptrarg[0];
g->m->ptrarg[0] = nil;
- // Have to acquire worldsema to stop the world,
- // because stoptheworld can only be used by
- // one goroutine at a time, and there might be
- // a pending garbage collection already calling it.
- runtime·semacquire(&runtime·worldsema, false);
- g->m->gcing = 1;
- runtime·stoptheworld();
runtime·updatememstats(nil);
// Size of the trailing by_size array differs between Go and C,
// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
@@ -1608,12 +1595,6 @@ readmemstats_m(void)
stats->stacks_sys = stats->stacks_inuse;
stats->heap_inuse -= stats->stacks_inuse;
stats->heap_sys -= stats->stacks_inuse;
-
- g->m->gcing = 0;
- g->m->locks++;
- runtime·semrelease(&runtime·worldsema);
- runtime·starttheworld();
- g->m->locks--;
}
static void readgcstats_m(void);
diff --git a/src/runtime/mheap.c b/src/runtime/mheap.c
index 902a5c71a..bb203d5ce 100644
--- a/src/runtime/mheap.c
+++ b/src/runtime/mheap.c
@@ -184,6 +184,8 @@ mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large)
// transfer stats from cache to global
mstats.heap_alloc += g->m->mcache->local_cachealloc;
g->m->mcache->local_cachealloc = 0;
+ mstats.tinyallocs += g->m->mcache->local_tinyallocs;
+ g->m->mcache->local_tinyallocs = 0;
s = MHeap_AllocSpanLocked(h, npage);
if(s != nil) {
@@ -465,6 +467,8 @@ mheap_free(MHeap *h, MSpan *s, int32 acct)
runtime·lock(&h->lock);
mstats.heap_alloc += g->m->mcache->local_cachealloc;
g->m->mcache->local_cachealloc = 0;
+ mstats.tinyallocs += g->m->mcache->local_tinyallocs;
+ g->m->mcache->local_tinyallocs = 0;
if(acct) {
mstats.heap_alloc -= s->npages<<PageShift;
mstats.heap_objects--;
diff --git a/src/runtime/os_windows.c b/src/runtime/os_windows.c
index 6c8f137ee..77f99062c 100644
--- a/src/runtime/os_windows.c
+++ b/src/runtime/os_windows.c
@@ -72,6 +72,7 @@ extern void *runtime·WaitForSingleObject;
extern void *runtime·WriteFile;
extern void *runtime·timeBeginPeriod;
+#pragma dataflag NOPTR
void *runtime·GetQueuedCompletionStatusEx;
extern uintptr runtime·externalthreadhandlerp;
@@ -147,7 +148,7 @@ runtime·get_random_data(byte **rnd, int32 *rnd_len)
void
runtime·goenvs(void)
{
- extern Slice syscall·envs;
+ extern Slice runtime·envs;
uint16 *env;
String *s;
@@ -160,8 +161,8 @@ runtime·goenvs(void)
for(p=env; *p; n++)
p += runtime·findnullw(p)+1;
- syscall·envs = runtime·makeStringSlice(n);
- s = (String*)syscall·envs.array;
+ runtime·envs = runtime·makeStringSlice(n);
+ s = (String*)runtime·envs.array;
p = env;
for(i=0; i<n; i++) {
@@ -278,6 +279,8 @@ runtime·minit(void)
void
runtime·unminit(void)
{
+ runtime·stdcall1(runtime·CloseHandle, (uintptr)g->m->thread);
+ g->m->thread = nil;
}
// Described in http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/
@@ -287,7 +290,9 @@ typedef struct KSYSTEM_TIME {
int32 High2Time;
} KSYSTEM_TIME;
+#pragma dataflag NOPTR
const KSYSTEM_TIME* INTERRUPT_TIME = (KSYSTEM_TIME*)0x7ffe0008;
+#pragma dataflag NOPTR
const KSYSTEM_TIME* SYSTEM_TIME = (KSYSTEM_TIME*)0x7ffe0014;
static void badsystime(void);
@@ -498,6 +503,7 @@ runtime·ctrlhandler1(uint32 type)
extern void runtime·dosigprof(Context *r, G *gp, M *mp);
extern void runtime·profileloop(void);
+#pragma dataflag NOPTR
static void *profiletimer;
static void
diff --git a/src/runtime/os_windows_386.c b/src/runtime/os_windows_386.c
index 15a5ea5d1..e2ae8db27 100644
--- a/src/runtime/os_windows_386.c
+++ b/src/runtime/os_windows_386.c
@@ -24,8 +24,6 @@ runtime·dumpregs(Context *r)
runtime·printf("gs %x\n", r->SegGs);
}
-#define DBG_PRINTEXCEPTION_C 0x40010006
-
// Called by sigtramp from Windows VEH handler.
// Return value signals whether the exception has been handled (-1)
// or should be made available to other handlers in the chain (0).
@@ -36,37 +34,11 @@ runtime·sighandler(ExceptionRecord *info, Context *r, G *gp)
uintptr *sp;
extern byte runtime·text[], runtime·etext[];
- if(info->ExceptionCode == DBG_PRINTEXCEPTION_C) {
- // This exception is intended to be caught by debuggers.
- // There is a not-very-informational message like
- // "Invalid parameter passed to C runtime function"
- // sitting at info->ExceptionInformation[0] (a wchar_t*),
- // with length info->ExceptionInformation[1].
- // The default behavior is to ignore this exception,
- // but somehow returning 0 here (meaning keep going)
- // makes the program crash instead. Maybe Windows has no
- // other handler registered? In any event, ignore it.
- return -1;
- }
-
// Only handle exception if executing instructions in Go binary
// (not Windows library code).
if(r->Eip < (uint32)runtime·text || (uint32)runtime·etext < r->Eip)
return 0;
- switch(info->ExceptionCode) {
- case EXCEPTION_BREAKPOINT:
- // It is unclear whether this is needed, unclear whether it
- // would work, and unclear how to test it. Leave out for now.
- // This only handles breakpoint instructions written in the
- // assembly sources, not breakpoints set by a debugger, and
- // there are very few of the former.
- //
- // r->Eip--; // because 8l generates 2 bytes for INT3
- // return 0;
- break;
- }
-
if(gp != nil && runtime·issigpanic(info->ExceptionCode)) {
// Make it look like a call to the signal func.
// Have to pass arguments out of band since
diff --git a/src/runtime/os_windows_amd64.c b/src/runtime/os_windows_amd64.c
index 9a69d73c0..261880d45 100644
--- a/src/runtime/os_windows_amd64.c
+++ b/src/runtime/os_windows_amd64.c
@@ -32,8 +32,6 @@ runtime·dumpregs(Context *r)
runtime·printf("gs %X\n", (uint64)r->SegGs);
}
-#define DBG_PRINTEXCEPTION_C 0x40010006
-
// Called by sigtramp from Windows VEH handler.
// Return value signals whether the exception has been handled (-1)
// or should be made available to other handlers in the chain (0).
@@ -44,34 +42,11 @@ runtime·sighandler(ExceptionRecord *info, Context *r, G *gp)
uintptr *sp;
extern byte runtime·text[], runtime·etext[];
- if(info->ExceptionCode == DBG_PRINTEXCEPTION_C) {
- // This exception is intended to be caught by debuggers.
- // There is a not-very-informational message like
- // "Invalid parameter passed to C runtime function"
- // sitting at info->ExceptionInformation[0] (a wchar_t*),
- // with length info->ExceptionInformation[1].
- // The default behavior is to ignore this exception,
- // but somehow returning 0 here (meaning keep going)
- // makes the program crash instead. Maybe Windows has no
- // other handler registered? In any event, ignore it.
- return -1;
- }
-
// Only handle exception if executing instructions in Go binary
// (not Windows library code).
if(r->Rip < (uint64)runtime·text || (uint64)runtime·etext < r->Rip)
return 0;
- switch(info->ExceptionCode) {
- case EXCEPTION_BREAKPOINT:
- // It is unclear whether this is needed, unclear whether it
- // would work, and unclear how to test it. Leave out for now.
- // This only handles breakpoint instructions written in the
- // assembly sources, not breakpoints set by a debugger, and
- // there are very few of the former.
- break;
- }
-
if(gp != nil && runtime·issigpanic(info->ExceptionCode)) {
// Make it look like a call to the signal func.
// Have to pass arguments out of band since
diff --git a/src/runtime/panic.go b/src/runtime/panic.go
index 12c85e7ca..7eb2d6055 100644
--- a/src/runtime/panic.go
+++ b/src/runtime/panic.go
@@ -238,7 +238,8 @@ func deferreturn(arg0 uintptr) {
}
// Goexit terminates the goroutine that calls it. No other goroutine is affected.
-// Goexit runs all deferred calls before terminating the goroutine.
+// Goexit runs all deferred calls before terminating the goroutine. Because Goexit
+// is not panic, however, any recover calls in those deferred functions will return nil.
//
// Calling Goexit from the main goroutine terminates that goroutine
// without func main returning. Since func main has not returned,
@@ -246,11 +247,27 @@ func deferreturn(arg0 uintptr) {
// If all other goroutines exit, the program crashes.
func Goexit() {
// Run all deferred functions for the current goroutine.
+ // This code is similar to gopanic, see that implementation
+ // for detailed comments.
gp := getg()
- for gp._defer != nil {
+ for {
d := gp._defer
+ if d == nil {
+ break
+ }
+ if d.started {
+ if d._panic != nil {
+ d._panic.aborted = true
+ }
+ gp._defer = d.link
+ freedefer(d)
+ continue
+ }
d.started = true
reflectcall(unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz))
+ if gp._defer != d {
+ gothrow("bad defer entry in Goexit")
+ }
gp._defer = d.link
freedefer(d)
// Note: we ignore recovers here because Goexit isn't a panic
@@ -280,6 +297,35 @@ func gopanic(e interface{}) {
if gp.m.curg != gp {
gothrow("panic on m stack")
}
+
+ // m.softfloat is set during software floating point.
+ // It increments m.locks to avoid preemption.
+ // We moved the memory loads out, so there shouldn't be
+ // any reason for it to panic anymore.
+ if gp.m.softfloat != 0 {
+ gp.m.locks--
+ gp.m.softfloat = 0
+ gothrow("panic during softfloat")
+ }
+ if gp.m.mallocing != 0 {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ gothrow("panic during malloc")
+ }
+ if gp.m.gcing != 0 {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ gothrow("panic during gc")
+ }
+ if gp.m.locks != 0 {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ gothrow("panic holding locks")
+ }
+
var p _panic
p.arg = e
p.link = gp._panic
@@ -430,33 +476,3 @@ func gothrow(s string) {
dopanic(0)
*(*int)(nil) = 0 // not reached
}
-
-func panicstring(s *int8) {
- // m.softfloat is set during software floating point,
- // which might cause a fault during a memory load.
- // It increments m.locks to avoid preemption.
- // If we're panicking, the software floating point frames
- // will be unwound, so decrement m.locks as they would.
- gp := getg()
- if gp.m.softfloat != 0 {
- gp.m.locks--
- gp.m.softfloat = 0
- }
-
- if gp.m.mallocing != 0 {
- print("panic: ", s, "\n")
- gothrow("panic during malloc")
- }
- if gp.m.gcing != 0 {
- print("panic: ", s, "\n")
- gothrow("panic during gc")
- }
- if gp.m.locks != 0 {
- print("panic: ", s, "\n")
- gothrow("panic holding locks")
- }
-
- var err interface{}
- newErrorCString(unsafe.Pointer(s), &err)
- gopanic(err)
-}
diff --git a/src/runtime/proc.c b/src/runtime/proc.c
index 1f1044d1d..ea50ff43a 100644
--- a/src/runtime/proc.c
+++ b/src/runtime/proc.c
@@ -24,42 +24,6 @@
//
// Design doc at http://golang.org/s/go11sched.
-typedef struct Sched Sched;
-struct Sched {
- Mutex lock;
-
- uint64 goidgen;
-
- M* midle; // idle m's waiting for work
- int32 nmidle; // number of idle m's waiting for work
- int32 nmidlelocked; // number of locked m's waiting for work
- int32 mcount; // number of m's that have been created
- int32 maxmcount; // maximum number of m's allowed (or die)
-
- P* pidle; // idle P's
- uint32 npidle;
- uint32 nmspinning;
-
- // Global runnable queue.
- G* runqhead;
- G* runqtail;
- int32 runqsize;
-
- // Global cache of dead G's.
- Mutex gflock;
- G* gfree;
- int32 ngfree;
-
- uint32 gcwaiting; // gc is waiting to run
- int32 stopwait;
- Note stopnote;
- uint32 sysmonwait;
- Note sysmonnote;
- uint64 lastpoll;
-
- int32 profilehz; // cpu profiling rate
-};
-
enum
{
// Number of goroutine ids to grab from runtime·sched.goidgen to local per-P cache at once.
@@ -67,7 +31,7 @@ enum
GoidCacheBatch = 16,
};
-Sched runtime·sched;
+SchedT runtime·sched;
int32 runtime·gomaxprocs;
uint32 runtime·needextram;
bool runtime·iscgo;
@@ -79,7 +43,7 @@ M* runtime·extram;
P* runtime·allp[MaxGomaxprocs+1];
int8* runtime·goos;
int32 runtime·ncpu;
-static int32 newprocs;
+int32 runtime·newprocs;
Mutex runtime·allglock; // the following vars are protected by this lock or by stoptheworld
G** runtime·allg;
@@ -138,9 +102,9 @@ extern String runtime·buildVersion;
#pragma cgo_export_static main
// Filled in by dynamic linker when Cgo is available.
-void* _cgo_init;
-void* _cgo_malloc;
-void* _cgo_free;
+void (*_cgo_init)(void);
+void (*_cgo_malloc)(void);
+void (*_cgo_free)(void);
// Copy for Go code.
void* runtime·cgoMalloc;
@@ -159,7 +123,6 @@ runtime·schedinit(void)
{
int32 n, procs;
byte *p;
- Eface i;
// raceinit must be the first call to race detector.
// In particular, it must be done before mallocinit below calls racemapshadow.
@@ -168,17 +131,12 @@ runtime·schedinit(void)
runtime·sched.maxmcount = 10000;
+ runtime·tracebackinit();
runtime·symtabinit();
runtime·stackinit();
runtime·mallocinit();
mcommoninit(g->m);
- // Initialize the itable value for newErrorCString,
- // so that the next time it gets called, possibly
- // in a fault during a garbage collection, it will not
- // need to allocated memory.
- runtime·newErrorCString(0, &i);
-
runtime·goargs();
runtime·goenvs();
runtime·parsedebugvars();
@@ -764,9 +722,9 @@ runtime·starttheworld(void)
injectglist(gp);
add = needaddgcproc();
runtime·lock(&runtime·sched.lock);
- if(newprocs) {
- procresize(newprocs);
- newprocs = 0;
+ if(runtime·newprocs) {
+ procresize(runtime·newprocs);
+ runtime·newprocs = 0;
} else
procresize(runtime·gomaxprocs);
runtime·sched.gcwaiting = 0;
@@ -896,24 +854,19 @@ struct CgoThreadStart
void (*fn)(void);
};
+M *runtime·newM(void); // in proc.go
+
// Allocate a new m unassociated with any thread.
// Can use p for allocation context if needed.
M*
runtime·allocm(P *p)
{
M *mp;
- static Type *mtype; // The Go type M
g->m->locks++; // disable GC because it can be called from sysmon
if(g->m->p == nil)
acquirep(p); // temporarily borrow p for mallocs in this function
- if(mtype == nil) {
- Eface e;
- runtime·gc_m_ptr(&e);
- mtype = ((PtrType*)e.type)->elem;
- }
-
- mp = runtime·cnew(mtype);
+ mp = runtime·newM();
mcommoninit(mp);
// In case of cgo or Solaris, pthread_create will make us a stack.
@@ -933,19 +886,12 @@ runtime·allocm(P *p)
return mp;
}
+G *runtime·newG(void); // in proc.go
+
static G*
allocg(void)
{
- G *gp;
- static Type *gtype;
-
- if(gtype == nil) {
- Eface e;
- runtime·gc_g_ptr(&e);
- gtype = ((PtrType*)e.type)->elem;
- }
- gp = runtime·cnew(gtype);
- return gp;
+ return runtime·newG();
}
static M* lockextra(bool nilokay);
@@ -1744,9 +1690,9 @@ goexit0(G *gp)
#pragma textflag NOSPLIT
static void
-save(void *pc, uintptr sp)
+save(uintptr pc, uintptr sp)
{
- g->sched.pc = (uintptr)pc;
+ g->sched.pc = pc;
g->sched.sp = sp;
g->sched.lr = 0;
g->sched.ret = 0;
@@ -1774,9 +1720,15 @@ static void entersyscall_gcwait(void);
// In practice, this means that we make the fast path run through
// entersyscall doing no-split things, and the slow path has to use onM
// to run bigger things on the m stack.
+//
+// reentersyscall is the entry point used by cgo callbacks, where explicitly
+// saved SP and PC are restored. This is needed when exitsyscall will be called
+// from a function further up in the call stack than the parent, as g->syscallsp
+// must always point to a valid stack frame. entersyscall below is the normal
+// entry point for syscalls, which obtains the SP and PC from the caller.
#pragma textflag NOSPLIT
void
-·entersyscall(int32 dummy)
+runtime·reentersyscall(uintptr pc, uintptr sp)
{
void (*fn)(void);
@@ -1792,9 +1744,9 @@ void
g->throwsplit = 1;
// Leave SP around for GC and traceback.
- save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
- g->syscallsp = g->sched.sp;
- g->syscallpc = g->sched.pc;
+ save(pc, sp);
+ g->syscallsp = sp;
+ g->syscallpc = pc;
runtime·casgstatus(g, Grunning, Gsyscall);
if(g->syscallsp < g->stack.lo || g->stack.hi < g->syscallsp) {
fn = entersyscall_bad;
@@ -1804,7 +1756,7 @@ void
if(runtime·atomicload(&runtime·sched.sysmonwait)) { // TODO: fast atomic
fn = entersyscall_sysmon;
runtime·onM(&fn);
- save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
+ save(pc, sp);
}
g->m->mcache = nil;
@@ -1813,7 +1765,7 @@ void
if(runtime·sched.gcwaiting) {
fn = entersyscall_gcwait;
runtime·onM(&fn);
- save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
+ save(pc, sp);
}
// Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched).
@@ -1823,6 +1775,14 @@ void
g->m->locks--;
}
+// Standard syscall entry used by the go syscall library and normal cgo calls.
+#pragma textflag NOSPLIT
+void
+·entersyscall(int32 dummy)
+{
+ runtime·reentersyscall((uintptr)runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
+}
+
static void
entersyscall_bad(void)
{
@@ -1870,7 +1830,7 @@ void
g->stackguard0 = StackPreempt; // see comment in entersyscall
// Leave SP around for GC and traceback.
- save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
+ save((uintptr)runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
g->syscallsp = g->sched.sp;
g->syscallpc = g->sched.pc;
runtime·casgstatus(g, Grunning, Gsyscall);
@@ -1883,7 +1843,7 @@ void
runtime·onM(&fn);
// Resave for traceback during blocked call.
- save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
+ save((uintptr)runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
g->m->locks--;
}
@@ -1900,12 +1860,15 @@ entersyscallblock_handoff(void)
// from the low-level system calls used by the runtime.
#pragma textflag NOSPLIT
void
-runtime·exitsyscall(void)
+·exitsyscall(int32 dummy)
{
void (*fn)(G*);
g->m->locks++; // see comment in entersyscall
+ if(runtime·getcallersp(&dummy) > g->syscallsp)
+ runtime·throw("exitsyscall: syscall frame is no longer valid");
+
g->waitsince = 0;
if(exitsyscallfast()) {
// There's a cpu for us, so we can run.
@@ -2199,11 +2162,11 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp
siz = narg + nret;
siz = (siz+7) & ~7;
- // We could instead create a secondary stack frame
- // and make it look like goexit was on the original but
- // the call to the actual goroutine function was split.
+ // We could allocate a larger initial stack if necessary.
// Not worth it: this is almost always an error.
- if(siz > StackMin - 1024)
+ // 4*sizeof(uintreg): extra space added below
+ // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall).
+ if(siz >= StackMin - 4*sizeof(uintreg) - sizeof(uintreg))
runtime·throw("runtime.newproc: function arguments too large for new goroutine");
p = g->m->p;
@@ -2365,39 +2328,6 @@ runtime·Breakpoint(void)
runtime·breakpoint();
}
-// Implementation of runtime.GOMAXPROCS.
-// delete when scheduler is even stronger
-void
-runtime·gomaxprocs_m(void)
-{
- int32 n, ret;
-
- n = g->m->scalararg[0];
- g->m->scalararg[0] = 0;
-
- if(n > MaxGomaxprocs)
- n = MaxGomaxprocs;
- runtime·lock(&runtime·sched.lock);
- ret = runtime·gomaxprocs;
- if(n <= 0 || n == ret) {
- runtime·unlock(&runtime·sched.lock);
- g->m->scalararg[0] = ret;
- return;
- }
- runtime·unlock(&runtime·sched.lock);
-
- runtime·semacquire(&runtime·worldsema, false);
- g->m->gcing = 1;
- runtime·stoptheworld();
- newprocs = n;
- g->m->gcing = 0;
- runtime·semrelease(&runtime·worldsema);
- runtime·starttheworld();
-
- g->m->scalararg[0] = ret;
- return;
-}
-
// lockOSThread is called by runtime.LockOSThread and runtime.lockOSThread below
// after they modify m->locked. Do not allow preemption during this call,
// or else the m might be different in this function than in the caller.
diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index eefe8239f..b8ea62b05 100644
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -215,6 +215,14 @@ func newP() *p {
return new(p)
}
+func newM() *m {
+ return new(m)
+}
+
+func newG() *g {
+ return new(g)
+}
+
func allgadd(gp *g) {
if readgstatus(gp) == _Gidle {
gothrow("allgadd: bad status Gidle")
diff --git a/src/runtime/rt0_linux_386.s b/src/runtime/rt0_linux_386.s
index 74ddc94da..352e594d5 100644
--- a/src/runtime/rt0_linux_386.s
+++ b/src/runtime/rt0_linux_386.s
@@ -21,5 +21,5 @@ TEXT _fallback_vdso(SB),NOSPLIT,$0
RET
DATA runtime·_vdso(SB)/4, $_fallback_vdso(SB)
-GLOBL runtime·_vdso(SB), $4
+GLOBL runtime·_vdso(SB), NOPTR, $4
diff --git a/src/runtime/rt0_linux_arm.s b/src/runtime/rt0_linux_arm.s
index 8af3d3505..5f521d24b 100644
--- a/src/runtime/rt0_linux_arm.s
+++ b/src/runtime/rt0_linux_arm.s
@@ -77,7 +77,7 @@ DATA bad_abi_msg+0x18(SB)/8, $" run on "
DATA bad_abi_msg+0x20(SB)/8, $"EABI ker"
DATA bad_abi_msg+0x28(SB)/4, $"nels"
DATA bad_abi_msg+0x2c(SB)/1, $0xa
-GLOBL bad_abi_msg(SB), $45
+GLOBL bad_abi_msg(SB), RODATA, $45
TEXT oabi_syscall<>(SB),NOSPLIT,$-4
ADD $1, PC, R4
diff --git a/src/runtime/rt0_plan9_386.s b/src/runtime/rt0_plan9_386.s
index 7e2887b85..c451299ee 100644
--- a/src/runtime/rt0_plan9_386.s
+++ b/src/runtime/rt0_plan9_386.s
@@ -17,7 +17,7 @@ TEXT _rt0_386_plan9(SB),NOSPLIT,$12
CALL runtime·rt0_go(SB)
DATA runtime·isplan9(SB)/4, $1
-GLOBL runtime·isplan9(SB), $4
-GLOBL _tos(SB), $4
-GLOBL _privates(SB), $4
-GLOBL _nprivates(SB), $4
+GLOBL runtime·isplan9(SB), NOPTR, $4
+GLOBL _tos(SB), NOPTR, $4
+GLOBL _privates(SB), NOPTR, $4
+GLOBL _nprivates(SB), NOPTR, $4
diff --git a/src/runtime/rt0_plan9_amd64.s b/src/runtime/rt0_plan9_amd64.s
index a372a0ba8..ec2d9ec82 100644
--- a/src/runtime/rt0_plan9_amd64.s
+++ b/src/runtime/rt0_plan9_amd64.s
@@ -15,7 +15,7 @@ TEXT _rt0_amd64_plan9(SB),NOSPLIT,$24
JMP AX
DATA runtime·isplan9(SB)/4, $1
-GLOBL runtime·isplan9(SB), $4
-GLOBL _tos(SB), $8
-GLOBL _privates(SB), $8
-GLOBL _nprivates(SB), $4
+GLOBL runtime·isplan9(SB), NOPTR, $4
+GLOBL _tos(SB), NOPTR, $8
+GLOBL _privates(SB), NOPTR, $8
+GLOBL _nprivates(SB), NOPTR, $4
diff --git a/src/runtime/rt0_solaris_amd64.s b/src/runtime/rt0_solaris_amd64.s
index 92a9fc295..5997cbf8e 100644
--- a/src/runtime/rt0_solaris_amd64.s
+++ b/src/runtime/rt0_solaris_amd64.s
@@ -15,4 +15,4 @@ TEXT main(SB),NOSPLIT,$-8
JMP AX
DATA runtime·issolaris(SB)/4, $1
-GLOBL runtime·issolaris(SB), $4
+GLOBL runtime·issolaris(SB), NOPTR, $4
diff --git a/src/runtime/rt0_windows_386.s b/src/runtime/rt0_windows_386.s
index 00604372f..3c2deda90 100644
--- a/src/runtime/rt0_windows_386.s
+++ b/src/runtime/rt0_windows_386.s
@@ -17,4 +17,4 @@ TEXT main(SB),NOSPLIT,$0
DATA runtime·iswindows(SB)/4, $1
-GLOBL runtime·iswindows(SB), $4
+GLOBL runtime·iswindows(SB), NOPTR, $4
diff --git a/src/runtime/rt0_windows_amd64.s b/src/runtime/rt0_windows_amd64.s
index 890a570d1..197f52e11 100644
--- a/src/runtime/rt0_windows_amd64.s
+++ b/src/runtime/rt0_windows_amd64.s
@@ -16,4 +16,4 @@ TEXT main(SB),NOSPLIT,$-8
JMP AX
DATA runtime·iswindows(SB)/4, $1
-GLOBL runtime·iswindows(SB), $4
+GLOBL runtime·iswindows(SB), NOPTR, $4
diff --git a/src/runtime/runtime.c b/src/runtime/runtime.c
index ae754dc5c..b3503fb90 100644
--- a/src/runtime/runtime.c
+++ b/src/runtime/runtime.c
@@ -62,10 +62,12 @@ runtime·mchr(byte *p, byte c, byte *ep)
}
static int32 argc;
+
+#pragma dataflag NOPTR /* argv not a heap pointer */
static uint8** argv;
-Slice os·Args;
-Slice syscall·envs;
+extern Slice runtime·argslice;
+extern Slice runtime·envs;
void (*runtime·sysargs)(int32, uint8**);
@@ -97,8 +99,8 @@ runtime·goargs(void)
if(Windows)
return;
- os·Args = runtime·makeStringSlice(argc);
- s = (String*)os·Args.array;
+ runtime·argslice = runtime·makeStringSlice(argc);
+ s = (String*)runtime·argslice.array;
for(i=0; i<argc; i++)
s[i] = runtime·gostringnocopy(argv[i]);
}
@@ -112,8 +114,8 @@ runtime·goenvs_unix(void)
for(n=0; argv[argc+1+n] != 0; n++)
;
- syscall·envs = runtime·makeStringSlice(n);
- s = (String*)syscall·envs.array;
+ runtime·envs = runtime·makeStringSlice(n);
+ s = (String*)runtime·envs.array;
for(i=0; i<n; i++)
s[i] = runtime·gostringnocopy(argv[argc+1+i]);
}
@@ -122,7 +124,7 @@ runtime·goenvs_unix(void)
Slice
runtime·environ()
{
- return syscall·envs;
+ return runtime·envs;
}
int32
@@ -267,10 +269,15 @@ runtime·check(void)
#pragma dataflag NOPTR
DebugVars runtime·debug;
-static struct {
+typedef struct DbgVar DbgVar;
+struct DbgVar
+{
int8* name;
int32* value;
-} dbgvar[] = {
+};
+
+#pragma dataflag NOPTR /* dbgvar has no heap pointers */
+static DbgVar dbgvar[] = {
{"allocfreetrace", &runtime·debug.allocfreetrace},
{"efence", &runtime·debug.efence},
{"gctrace", &runtime·debug.gctrace},
@@ -287,18 +294,18 @@ runtime·parsedebugvars(void)
intgo i, n;
p = runtime·getenv("GODEBUG");
- if(p == nil)
- return;
- for(;;) {
- for(i=0; i<nelem(dbgvar); i++) {
- n = runtime·findnull((byte*)dbgvar[i].name);
- if(runtime·mcmp(p, (byte*)dbgvar[i].name, n) == 0 && p[n] == '=')
- *dbgvar[i].value = runtime·atoi(p+n+1);
+ if(p != nil){
+ for(;;) {
+ for(i=0; i<nelem(dbgvar); i++) {
+ n = runtime·findnull((byte*)dbgvar[i].name);
+ if(runtime·mcmp(p, (byte*)dbgvar[i].name, n) == 0 && p[n] == '=')
+ *dbgvar[i].value = runtime·atoi(p+n+1);
+ }
+ p = runtime·strstr(p, (byte*)",");
+ if(p == nil)
+ break;
+ p++;
}
- p = runtime·strstr(p, (byte*)",");
- if(p == nil)
- break;
- p++;
}
p = runtime·getenv("GOTRACEBACK");
diff --git a/src/runtime/runtime.go b/src/runtime/runtime.go
index dbaea45a6..4e4e1d17a 100644
--- a/src/runtime/runtime.go
+++ b/src/runtime/runtime.go
@@ -9,6 +9,8 @@ var ticks struct {
val uint64
}
+var tls0 [8]uintptr // available storage for m0's TLS; not necessarily used; opaque to GC
+
// Note: Called by runtime/pprof in addition to runtime code.
func tickspersecond() int64 {
r := int64(atomicload64(&ticks.val))
@@ -47,3 +49,12 @@ func parforalloc(nthrmax uint32) *parfor {
nthrmax: nthrmax,
}
}
+
+var envs []string
+var argslice []string
+
+// called from syscall
+func runtime_envs() []string { return envs }
+
+// called from os
+func runtime_args() []string { return argslice }
diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h
index 74d7ba4f5..36748c3a1 100644
--- a/src/runtime/runtime.h
+++ b/src/runtime/runtime.h
@@ -60,6 +60,7 @@ typedef struct SudoG SudoG;
typedef struct Mutex Mutex;
typedef struct M M;
typedef struct P P;
+typedef struct SchedT SchedT;
typedef struct Note Note;
typedef struct Slice Slice;
typedef struct String String;
@@ -434,6 +435,42 @@ enum {
MaxGomaxprocs = 1<<8,
};
+struct SchedT
+{
+ Mutex lock;
+
+ uint64 goidgen;
+
+ M* midle; // idle m's waiting for work
+ int32 nmidle; // number of idle m's waiting for work
+ int32 nmidlelocked; // number of locked m's waiting for work
+ int32 mcount; // number of m's that have been created
+ int32 maxmcount; // maximum number of m's allowed (or die)
+
+ P* pidle; // idle P's
+ uint32 npidle;
+ uint32 nmspinning;
+
+ // Global runnable queue.
+ G* runqhead;
+ G* runqtail;
+ int32 runqsize;
+
+ // Global cache of dead G's.
+ Mutex gflock;
+ G* gfree;
+ int32 ngfree;
+
+ uint32 gcwaiting; // gc is waiting to run
+ int32 stopwait;
+ Note stopnote;
+ uint32 sysmonwait;
+ Note sysmonnote;
+ uint64 lastpoll;
+
+ int32 profilehz; // cpu profiling rate
+};
+
// The m->locked word holds two pieces of state counting active calls to LockOSThread/lockOSThread.
// The low bit (LockExternal) is a boolean reporting whether any LockOSThread call is active.
// External locks are not recursive; a second lock is silently ignored.
@@ -729,6 +766,8 @@ extern DebugVars runtime·debug;
extern uintptr runtime·maxstacksize;
extern Note runtime·signote;
extern ForceGCState runtime·forcegc;
+extern SchedT runtime·sched;
+extern int32 runtime·newprocs;
/*
* common functions and data
@@ -778,7 +817,6 @@ void runtime·goenvs(void);
void runtime·goenvs_unix(void);
void* runtime·getu(void);
void runtime·throw(int8*);
-void runtime·panicstring(int8*);
bool runtime·canpanic(G*);
void runtime·prints(int8*);
void runtime·printf(int8*, ...);
@@ -816,6 +854,7 @@ void runtime·mpreinit(M*);
void runtime·minit(void);
void runtime·unminit(void);
void runtime·signalstack(byte*, int32);
+void runtime·tracebackinit(void);
void runtime·symtabinit(void);
Func* runtime·findfunc(uintptr);
int32 runtime·funcline(Func*, uintptr, String*);
@@ -876,6 +915,7 @@ void runtime·goexit(void);
void runtime·asmcgocall(void (*fn)(void*), void*);
int32 runtime·asmcgocall_errno(void (*fn)(void*), void*);
void runtime·entersyscall(void);
+void runtime·reentersyscall(uintptr, uintptr);
void runtime·entersyscallblock(void);
void runtime·exitsyscall(void);
G* runtime·newproc1(FuncVal*, byte*, int32, int32, void*);
@@ -1037,8 +1077,6 @@ void runtime·panicdivide(void);
*/
void runtime·printany(Eface);
void runtime·newTypeAssertionError(String*, String*, String*, String*, Eface*);
-void runtime·newErrorString(String, Eface*);
-void runtime·newErrorCString(int8*, Eface*);
void runtime·fadd64c(uint64, uint64, uint64*);
void runtime·fsub64c(uint64, uint64, uint64*);
void runtime·fmul64c(uint64, uint64, uint64*);
diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go
index cffc9f7d3..1688364a8 100644
--- a/src/runtime/runtime_test.go
+++ b/src/runtime/runtime_test.go
@@ -157,8 +157,8 @@ var faultAddrs = []uint64{
// or else malformed.
0xffffffffffffffff,
0xfffffffffffff001,
- // no 0xffffffffffff0001; 0xffff0001 is mapped for 32-bit user space on OS X
- // no 0xfffffffffff00001; 0xfff00001 is mapped for 32-bit user space sometimes on Linux
+ 0xffffffffffff0001,
+ 0xfffffffffff00001,
0xffffffffff000001,
0xfffffffff0000001,
0xffffffff00000001,
@@ -182,26 +182,32 @@ func TestSetPanicOnFault(t *testing.T) {
old := debug.SetPanicOnFault(true)
defer debug.SetPanicOnFault(old)
+ nfault := 0
for _, addr := range faultAddrs {
- testSetPanicOnFault(t, uintptr(addr))
+ testSetPanicOnFault(t, uintptr(addr), &nfault)
+ }
+ if nfault == 0 {
+ t.Fatalf("none of the addresses faulted")
}
}
-func testSetPanicOnFault(t *testing.T, addr uintptr) {
+func testSetPanicOnFault(t *testing.T, addr uintptr, nfault *int) {
if GOOS == "nacl" {
t.Skip("nacl doesn't seem to fault on high addresses")
}
defer func() {
- if err := recover(); err == nil {
- t.Fatalf("did not find error in recover")
+ if err := recover(); err != nil {
+ *nfault++
}
}()
- var p *int
- p = (*int)(unsafe.Pointer(addr))
- println(*p)
- t.Fatalf("still here - should have faulted on address %#x", addr)
+ // The read should fault, except that sometimes we hit
+ // addresses that have had C or kernel pages mapped there
+ // readable by user code. So just log the content.
+ // If no addresses fault, we'll fail the test.
+ v := *(*byte)(unsafe.Pointer(addr))
+ t.Logf("addr %#x: %#x\n", addr, v)
}
func eqstring_generic(s1, s2 string) bool {
diff --git a/src/runtime/sema.go b/src/runtime/sema.go
index 142d3082c..a42a29988 100644
--- a/src/runtime/sema.go
+++ b/src/runtime/sema.go
@@ -49,6 +49,11 @@ func asyncsemrelease(addr *uint32) {
// Called from runtime.
func semacquire(addr *uint32, profile bool) {
+ gp := getg()
+ if gp != gp.m.curg {
+ gothrow("semacquire not on the G stack")
+ }
+
// Easy case.
if cansemacquire(addr) {
return
diff --git a/src/runtime/signals_darwin.h b/src/runtime/signals_darwin.h
index 229b58590..8761e1bd9 100644
--- a/src/runtime/signals_darwin.h
+++ b/src/runtime/signals_darwin.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_dragonfly.h b/src/runtime/signals_dragonfly.h
index 4d27e050d..07343a766 100644
--- a/src/runtime/signals_dragonfly.h
+++ b/src/runtime/signals_dragonfly.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_freebsd.h b/src/runtime/signals_freebsd.h
index 8d45c50c3..39e0a947e 100644
--- a/src/runtime/signals_freebsd.h
+++ b/src/runtime/signals_freebsd.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_linux.h b/src/runtime/signals_linux.h
index 368afc1c8..374107609 100644
--- a/src/runtime/signals_linux.h
+++ b/src/runtime/signals_linux.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_nacl.h b/src/runtime/signals_nacl.h
index 229b58590..8761e1bd9 100644
--- a/src/runtime/signals_nacl.h
+++ b/src/runtime/signals_nacl.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_netbsd.h b/src/runtime/signals_netbsd.h
index 7140de86f..950a2fe62 100644
--- a/src/runtime/signals_netbsd.h
+++ b/src/runtime/signals_netbsd.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_openbsd.h b/src/runtime/signals_openbsd.h
index 7140de86f..950a2fe62 100644
--- a/src/runtime/signals_openbsd.h
+++ b/src/runtime/signals_openbsd.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: terminal line hangup",
diff --git a/src/runtime/signals_plan9.h b/src/runtime/signals_plan9.h
index 818f508cf..4ee8e542c 100644
--- a/src/runtime/signals_plan9.h
+++ b/src/runtime/signals_plan9.h
@@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
@@ -16,6 +18,7 @@
// If you add entries to this table, you must respect the prefix ordering
// and also update the constant values is os_plan9.h.
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
// Traps that we cannot be recovered.
T, "sys: trap: debug exception",
diff --git a/src/runtime/signals_solaris.h b/src/runtime/signals_solaris.h
index c272cad29..1f0a65ea6 100644
--- a/src/runtime/signals_solaris.h
+++ b/src/runtime/signals_solaris.h
@@ -2,12 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+#include "textflag.h"
+
#define N SigNotify
#define K SigKill
#define T SigThrow
#define P SigPanic
#define D SigDefault
+#pragma dataflag NOPTR
SigTab runtime·sigtab[] = {
/* 0 */ 0, "SIGNONE: no trap",
/* 1 */ N+K, "SIGHUP: hangup",
diff --git a/src/runtime/stack.c b/src/runtime/stack.c
index 143b645e4..8562b9407 100644
--- a/src/runtime/stack.c
+++ b/src/runtime/stack.c
@@ -32,8 +32,8 @@ enum
// Stacks are assigned an order according to size.
// order = log_2(size/FixedStack)
// There is a free list for each order.
-static MSpan stackpool[NumStackOrders];
-static Mutex stackpoolmu;
+MSpan runtime·stackpool[NumStackOrders];
+Mutex runtime·stackpoolmu;
// TODO: one lock per order?
void
@@ -45,7 +45,7 @@ runtime·stackinit(void)
runtime·throw("cache size must be a multiple of page size");
for(i = 0; i < NumStackOrders; i++)
- runtime·MSpanList_Init(&stackpool[i]);
+ runtime·MSpanList_Init(&runtime·stackpool[i]);
}
// Allocates a stack from the free pool. Must be called with
@@ -58,7 +58,7 @@ poolalloc(uint8 order)
MLink *x;
uintptr i;
- list = &stackpool[order];
+ list = &runtime·stackpool[order];
s = list->next;
if(s == list) {
// no free stacks. Allocate another span worth.
@@ -99,7 +99,7 @@ poolfree(MLink *x, uint8 order)
runtime·throw("freeing stack not in a stack span");
if(s->freelist == nil) {
// s will now have a free stack
- runtime·MSpanList_Insert(&stackpool[order], s);
+ runtime·MSpanList_Insert(&runtime·stackpool[order], s);
}
x->next = s->freelist;
s->freelist = x;
@@ -127,14 +127,14 @@ stackcacherefill(MCache *c, uint8 order)
// Grab half of the allowed capacity (to prevent thrashing).
list = nil;
size = 0;
- runtime·lock(&stackpoolmu);
+ runtime·lock(&runtime·stackpoolmu);
while(size < StackCacheSize/2) {
x = poolalloc(order);
x->next = list;
list = x;
size += FixedStack << order;
}
- runtime·unlock(&stackpoolmu);
+ runtime·unlock(&runtime·stackpoolmu);
c->stackcache[order].list = list;
c->stackcache[order].size = size;
@@ -150,14 +150,14 @@ stackcacherelease(MCache *c, uint8 order)
runtime·printf("stackcacherelease order=%d\n", order);
x = c->stackcache[order].list;
size = c->stackcache[order].size;
- runtime·lock(&stackpoolmu);
+ runtime·lock(&runtime·stackpoolmu);
while(size > StackCacheSize/2) {
y = x->next;
poolfree(x, order);
x = y;
size -= FixedStack << order;
}
- runtime·unlock(&stackpoolmu);
+ runtime·unlock(&runtime·stackpoolmu);
c->stackcache[order].list = x;
c->stackcache[order].size = size;
}
@@ -170,7 +170,7 @@ runtime·stackcache_clear(MCache *c)
if(StackDebug >= 1)
runtime·printf("stackcache clear\n");
- runtime·lock(&stackpoolmu);
+ runtime·lock(&runtime·stackpoolmu);
for(order = 0; order < NumStackOrders; order++) {
x = c->stackcache[order].list;
while(x != nil) {
@@ -181,7 +181,7 @@ runtime·stackcache_clear(MCache *c)
c->stackcache[order].list = nil;
c->stackcache[order].size = 0;
}
- runtime·unlock(&stackpoolmu);
+ runtime·unlock(&runtime·stackpoolmu);
}
Stack
@@ -227,9 +227,9 @@ runtime·stackalloc(uint32 n)
// procresize. Just get a stack from the global pool.
// Also don't touch stackcache during gc
// as it's flushed concurrently.
- runtime·lock(&stackpoolmu);
+ runtime·lock(&runtime·stackpoolmu);
x = poolalloc(order);
- runtime·unlock(&stackpoolmu);
+ runtime·unlock(&runtime·stackpoolmu);
} else {
x = c->stackcache[order].list;
if(x == nil) {
@@ -289,9 +289,9 @@ runtime·stackfree(Stack stk)
x = (MLink*)v;
c = g->m->mcache;
if(c == nil || g->m->gcing || g->m->helpgc) {
- runtime·lock(&stackpoolmu);
+ runtime·lock(&runtime·stackpoolmu);
poolfree(x, order);
- runtime·unlock(&stackpoolmu);
+ runtime·unlock(&runtime·stackpoolmu);
} else {
if(c->stackcache[order].size >= StackCacheSize)
stackcacherelease(c, order);
@@ -463,7 +463,7 @@ adjustframe(Stkframe *frame, void *arg)
StackMap *stackmap;
int32 pcdata;
BitVector bv;
- uintptr targetpc;
+ uintptr targetpc, size, minsize;
adjinfo = arg;
targetpc = frame->continpc;
@@ -486,27 +486,47 @@ adjustframe(Stkframe *frame, void *arg)
if(pcdata == -1)
pcdata = 0; // in prologue
- // adjust local pointers
- if((byte*)frame->varp != (byte*)frame->sp) {
+ // Adjust local variables if stack frame has been allocated.
+ size = frame->varp - frame->sp;
+ if(thechar != '6' && thechar != '8')
+ minsize = sizeof(uintptr);
+ else
+ minsize = 0;
+ if(size > minsize) {
stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
- if(stackmap == nil)
- runtime·throw("no locals info");
- if(stackmap->n <= 0)
- runtime·throw("locals size info only");
+ if(stackmap == nil || stackmap->n <= 0) {
+ runtime·printf("runtime: frame %s untyped locals %p+%p\n", runtime·funcname(f), (byte*)(frame->varp-size), size);
+ runtime·throw("missing stackmap");
+ }
+ // Locals bitmap information, scan just the pointers in locals.
+ if(pcdata < 0 || pcdata >= stackmap->n) {
+ // don't know where we are
+ runtime·printf("runtime: pcdata is %d and %d locals stack map entries for %s (targetpc=%p)\n",
+ pcdata, stackmap->n, runtime·funcname(f), targetpc);
+ runtime·throw("bad symbol table");
+ }
bv = runtime·stackmapdata(stackmap, pcdata);
+ size = (bv.n * PtrSize) / BitsPerPointer;
if(StackDebug >= 3)
runtime·printf(" locals\n");
- adjustpointers((byte**)frame->varp - bv.n / BitsPerPointer, &bv, adjinfo, f);
+ adjustpointers((byte**)(frame->varp - size), &bv, adjinfo, f);
}
- // adjust inargs and outargs
- if(frame->arglen != 0) {
+
+ // Adjust arguments.
+ if(frame->arglen > 0) {
if(frame->argmap != nil) {
bv = *frame->argmap;
} else {
stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
- if(stackmap == nil) {
- runtime·printf("size %d\n", (int32)frame->arglen);
- runtime·throw("no arg info");
+ if(stackmap == nil || stackmap->n <= 0) {
+ runtime·printf("runtime: frame %s untyped args %p+%p\n", runtime·funcname(f), frame->argp, (uintptr)frame->arglen);
+ runtime·throw("missing stackmap");
+ }
+ if(pcdata < 0 || pcdata >= stackmap->n) {
+ // don't know where we are
+ runtime·printf("runtime: pcdata is %d and %d args stack map entries for %s (targetpc=%p)\n",
+ pcdata, stackmap->n, runtime·funcname(f), targetpc);
+ runtime·throw("bad symbol table");
}
bv = runtime·stackmapdata(stackmap, pcdata);
}
@@ -675,7 +695,7 @@ runtime·newstack(void)
runtime·traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g);
runtime·throw("runtime: wrong goroutine in newstack");
}
- if(g->throwsplit)
+ if(g->m->curg->throwsplit)
runtime·throw("runtime: stack split at bad time");
// The goroutine must be executing in order to call newstack,
@@ -786,8 +806,16 @@ runtime·shrinkstack(G *gp)
{
uintptr used, oldsize, newsize;
- if(runtime·readgstatus(gp) == Gdead)
+ if(runtime·readgstatus(gp) == Gdead) {
+ if(gp->stack.lo != 0) {
+ // Free whole stack - it will get reallocated
+ // if G is used again.
+ runtime·stackfree(gp->stack);
+ gp->stack.lo = 0;
+ gp->stack.hi = 0;
+ }
return;
+ }
if(gp->stack.lo == 0)
runtime·throw("missing stack in shrinkstack");
@@ -799,7 +827,9 @@ runtime·shrinkstack(G *gp)
if(used >= oldsize / 4)
return; // still using at least 1/4 of the segment.
- if(gp->syscallsp != 0) // TODO: can we handle this case?
+ // We can't copy the stack if we're in a syscall.
+ // The syscall might have pointers into the stack.
+ if(gp->syscallsp != 0)
return;
#ifdef GOOS_windows
diff --git a/src/runtime/stack.h b/src/runtime/stack.h
index b30e32216..f97dc4ed8 100644
--- a/src/runtime/stack.h
+++ b/src/runtime/stack.h
@@ -69,16 +69,19 @@ enum {
#endif // Plan 9
#endif // Windows
- // The amount of extra stack to allocate beyond the size
- // needed for the single frame that triggered the split.
- StackExtra = 2048,
+ // The minimum size of stack used by Go code
+ StackMin = 2048,
- // The minimum stack segment size to allocate.
- // If the amount needed for the splitting frame + StackExtra
- // is less than this number, the stack will have this size instead.
- StackMin = 8192,
- StackSystemRounded = StackSystem + (-StackSystem & (StackMin-1)),
- FixedStack = StackMin + StackSystemRounded,
+ // The minimum stack size to allocate.
+ // The hackery here rounds FixedStack0 up to a power of 2.
+ FixedStack0 = StackMin + StackSystem,
+ FixedStack1 = FixedStack0 - 1,
+ FixedStack2 = FixedStack1 | (FixedStack1 >> 1),
+ FixedStack3 = FixedStack2 | (FixedStack2 >> 2),
+ FixedStack4 = FixedStack3 | (FixedStack3 >> 4),
+ FixedStack5 = FixedStack4 | (FixedStack4 >> 8),
+ FixedStack6 = FixedStack5 | (FixedStack5 >> 16),
+ FixedStack = FixedStack6 + 1,
// Functions that need frames bigger than this use an extra
// instruction to do the stack split check, to avoid overflow
diff --git a/src/runtime/stack_test.go b/src/runtime/stack_test.go
index 3a0802a1c..652c72eee 100644
--- a/src/runtime/stack_test.go
+++ b/src/runtime/stack_test.go
@@ -71,10 +71,6 @@ func TestStackMem(t *testing.T) {
// Test stack growing in different contexts.
func TestStackGrowth(t *testing.T) {
- switch GOARCH {
- case "386", "arm":
- t.Skipf("skipping test on %q; see issue 8083", GOARCH)
- }
t.Parallel()
var wg sync.WaitGroup
diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go
index ff443c4cd..c6a9cf9f5 100644
--- a/src/runtime/stubs.go
+++ b/src/runtime/stubs.go
@@ -119,6 +119,8 @@ func deferproc_m()
func goexit_m()
func startpanic_m()
func dopanic_m()
+func readmemstats_m()
+func writeheapdump_m()
// memclr clears n bytes starting at ptr.
// in memclr_*.s
@@ -146,10 +148,6 @@ func fastrand1() uint32
//go:noescape
func memeq(a, b unsafe.Pointer, size uintptr) bool
-// Code pointers for the nohash/noequal algorithms. Used for producing better error messages.
-var nohashcode uintptr
-var noequalcode uintptr
-
// noescape hides a pointer from escape analysis. noescape is
// the identity function but escape analysis doesn't think the
// output depends on the input. noescape is inlined and currently
@@ -162,6 +160,7 @@ func noescape(p unsafe.Pointer) unsafe.Pointer {
}
func entersyscall()
+func reentersyscall(pc uintptr, sp unsafe.Pointer)
func entersyscallblock()
func exitsyscall()
diff --git a/src/runtime/sys_dragonfly_386.s b/src/runtime/sys_dragonfly_386.s
index dd0e27e26..161eaec19 100644
--- a/src/runtime/sys_dragonfly_386.s
+++ b/src/runtime/sys_dragonfly_386.s
@@ -378,4 +378,4 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$32
NEGL AX
RET
-GLOBL runtime·tlsoffset(SB),$4
+GLOBL runtime·tlsoffset(SB),NOPTR,$4
diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s
index ffc28560e..2c40fc433 100644
--- a/src/runtime/sys_freebsd_386.s
+++ b/src/runtime/sys_freebsd_386.s
@@ -388,4 +388,4 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$32
NEGL AX
RET
-GLOBL runtime·tlsoffset(SB),$4
+GLOBL runtime·tlsoffset(SB),NOPTR,$4
diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s
index 83a76cb34..23f2f6bd1 100644
--- a/src/runtime/sys_netbsd_386.s
+++ b/src/runtime/sys_netbsd_386.s
@@ -350,7 +350,7 @@ TEXT runtime·sysctl(SB),NOSPLIT,$28
MOVL $0, AX
RET
-GLOBL runtime·tlsoffset(SB),$4
+GLOBL runtime·tlsoffset(SB),NOPTR,$4
// int32 runtime·kqueue(void)
TEXT runtime·kqueue(SB),NOSPLIT,$0
diff --git a/src/runtime/sys_openbsd_386.s b/src/runtime/sys_openbsd_386.s
index 12d9c5c6b..5cda7768a 100644
--- a/src/runtime/sys_openbsd_386.s
+++ b/src/runtime/sys_openbsd_386.s
@@ -395,4 +395,4 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$32
NEGL AX
RET
-GLOBL runtime·tlsoffset(SB),$4
+GLOBL runtime·tlsoffset(SB),NOPTR,$4
diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s
index 9b1fc7a20..1bf4d062a 100644
--- a/src/runtime/sys_windows_386.s
+++ b/src/runtime/sys_windows_386.s
@@ -212,7 +212,7 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT,$0
POPL BP
RET
-GLOBL runtime·cbctxts(SB), $4
+GLOBL runtime·cbctxts(SB), NOPTR, $4
TEXT runtime·callbackasm1+0(SB),NOSPLIT,$0
MOVL 0(SP), AX // will use to find our callback context
diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s
index f701d157e..05750398e 100644
--- a/src/runtime/sys_windows_amd64.s
+++ b/src/runtime/sys_windows_amd64.s
@@ -249,7 +249,7 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT,$0
POPQ BP
RET
-GLOBL runtime·cbctxts(SB), $8
+GLOBL runtime·cbctxts(SB), NOPTR, $8
TEXT runtime·callbackasm1(SB),NOSPLIT,$0
// Construct args vector for cgocallback().
diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go
index a82851218..9ed016ccc 100644
--- a/src/runtime/syscall_windows_test.go
+++ b/src/runtime/syscall_windows_test.go
@@ -488,3 +488,9 @@ func TestRegisterClass(t *testing.T) {
t.Fatalf("UnregisterClass failed: %v", err)
}
}
+
+func TestOutputDebugString(t *testing.T) {
+ d := GetDLL(t, "kernel32.dll")
+ p := syscall.StringToUTF16Ptr("testing OutputDebugString")
+ d.Proc("OutputDebugStringW").Call(uintptr(unsafe.Pointer(p)))
+}
diff --git a/src/runtime/thunk.s b/src/runtime/thunk.s
index 3b66cf47d..0a0f147c4 100644
--- a/src/runtime/thunk.s
+++ b/src/runtime/thunk.s
@@ -80,6 +80,9 @@ TEXT reflect·memmove(SB), NOSPLIT, $0-0
TEXT runtime∕debug·freeOSMemory(SB), NOSPLIT, $0-0
JMP runtime·freeOSMemory(SB)
+TEXT runtime∕debug·WriteHeapDump(SB), NOSPLIT, $0-0
+ JMP runtime·writeHeapDump(SB)
+
TEXT net·runtime_pollServerInit(SB),NOSPLIT,$0-0
JMP runtime·netpollServerInit(SB)
@@ -107,6 +110,9 @@ TEXT net·runtime_pollUnblock(SB),NOSPLIT,$0-0
TEXT syscall·setenv_c(SB), NOSPLIT, $0-0
JMP runtime·syscall_setenv_c(SB)
+TEXT syscall·unsetenv_c(SB), NOSPLIT, $0-0
+ JMP runtime·syscall_unsetenv_c(SB)
+
TEXT reflect·makemap(SB),NOSPLIT,$0-0
JMP runtime·reflect_makemap(SB)
@@ -161,5 +167,17 @@ TEXT runtime·main_init(SB),NOSPLIT,$0-0
TEXT runtime·main_main(SB),NOSPLIT,$0-0
JMP main·main(SB)
-TEXT runtime·timenow(SB), NOSPLIT, $0-0
+TEXT runtime·timenow(SB),NOSPLIT,$0-0
JMP time·now(SB)
+
+TEXT sync∕atomic·runtime_procPin(SB),NOSPLIT,$0-0
+ JMP sync·runtime_procPin(SB)
+
+TEXT sync∕atomic·runtime_procUnpin(SB),NOSPLIT,$0-0
+ JMP sync·runtime_procUnpin(SB)
+
+TEXT syscall·runtime_envs(SB),NOSPLIT,$0-0
+ JMP runtime·runtime_envs(SB)
+
+TEXT os·runtime_args(SB),NOSPLIT,$0-0
+ JMP runtime·runtime_args(SB)
diff --git a/src/runtime/tls_arm.s b/src/runtime/tls_arm.s
index 7a247ab19..85c3940bf 100644
--- a/src/runtime/tls_arm.s
+++ b/src/runtime/tls_arm.s
@@ -29,6 +29,9 @@ TEXT runtime·save_g(SB),NOSPLIT,$-4
MOVW g, R0 // preserve R0 across call to setg<>
RET
#endif
+ // If the host does not support MRC the linker will replace it with
+ // a call to runtime.read_tls_fallback which jumps to __kuser_get_tls.
+ // The replacement function saves LR in R11 over the call to read_tls_fallback.
MRC 15, 0, R0, C13, C0, 3 // fetch TLS base pointer
// $runtime.tlsg(SB) is a special linker symbol.
// It is the offset from the TLS base pointer to our
@@ -51,7 +54,8 @@ TEXT runtime·load_g(SB),NOSPLIT,$0
// nothing to do as nacl/arm does not use TLS at all.
RET
#endif
- MRC 15, 0, R0, C13, C0, 3 // fetch TLS base pointer
+ // See save_g
+ MRC 15, 0, R0, C13, C0, 3 // fetch TLS base pointer
// $runtime.tlsg(SB) is a special linker symbol.
// It is the offset from the TLS base pointer to our
// thread-local storage for g.
diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go
index 9e95fa33d..24dc3eea9 100644
--- a/src/runtime/traceback.go
+++ b/src/runtime/traceback.go
@@ -31,20 +31,36 @@ import "unsafe"
const usesLR = GOARCH != "amd64" && GOARCH != "amd64p32" && GOARCH != "386"
var (
- deferprocPC = funcPC(deferproc)
- goexitPC = funcPC(goexit)
- jmpdeferPC = funcPC(jmpdefer)
- mcallPC = funcPC(mcall)
- morestackPC = funcPC(morestack)
- mstartPC = funcPC(mstart)
- newprocPC = funcPC(newproc)
- newstackPC = funcPC(newstack)
- rt0_goPC = funcPC(rt0_go)
- sigpanicPC = funcPC(sigpanic)
+ // initialized in tracebackinit
+ deferprocPC uintptr
+ goexitPC uintptr
+ jmpdeferPC uintptr
+ mcallPC uintptr
+ morestackPC uintptr
+ mstartPC uintptr
+ newprocPC uintptr
+ rt0_goPC uintptr
+ sigpanicPC uintptr
externalthreadhandlerp uintptr // initialized elsewhere
)
+func tracebackinit() {
+ // Go variable initialization happens late during runtime startup.
+ // Instead of initializing the variables above in the declarations,
+ // schedinit calls this function so that the variables are
+ // initialized and available earlier in the startup sequence.
+ deferprocPC = funcPC(deferproc)
+ goexitPC = funcPC(goexit)
+ jmpdeferPC = funcPC(jmpdefer)
+ mcallPC = funcPC(mcall)
+ morestackPC = funcPC(morestack)
+ mstartPC = funcPC(mstart)
+ newprocPC = funcPC(newproc)
+ rt0_goPC = funcPC(rt0_go)
+ sigpanicPC = funcPC(sigpanic)
+}
+
// Traceback over the deferred function calls.
// Report them like calls that have been invoked but not started executing yet.
func tracebackdefers(gp *g, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer) {
@@ -81,6 +97,9 @@ func tracebackdefers(gp *g, callback func(*stkframe, unsafe.Pointer) bool, v uns
// collector (callback != nil). A little clunky to merge these, but avoids
// duplicating the code and all its subtlety.
func gentraceback(pc0 uintptr, sp0 uintptr, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max int, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer, printall bool) int {
+ if goexitPC == 0 {
+ gothrow("gentraceback before goexitPC initialization")
+ }
g := getg()
gotraceback := gotraceback(nil)
if pc0 == ^uintptr(0) && sp0 == ^uintptr(0) { // Signal to fetch saved values from gp.
@@ -499,7 +518,14 @@ func showframe(f *_func, gp *g) bool {
return true
}
- return traceback > 1 || f != nil && contains(name, ".") && !hasprefix(name, "runtime.")
+ return traceback > 1 || f != nil && contains(name, ".") && (!hasprefix(name, "runtime.") || isExportedRuntime(name))
+}
+
+// isExportedRuntime reports whether name is an exported runtime function.
+// It is only for runtime functions, so ASCII A-Z is fine.
+func isExportedRuntime(name string) bool {
+ const n = len("runtime.")
+ return len(name) > n && name[:n] == "runtime." && 'A' <= name[n] && name[n] <= 'Z'
}
var gStatusStrings = [...]string{
diff --git a/src/runtime/vdso_linux_amd64.c b/src/runtime/vdso_linux_amd64.c
index 38e115243..41a41fdd6 100644
--- a/src/runtime/vdso_linux_amd64.c
+++ b/src/runtime/vdso_linux_amd64.c
@@ -3,6 +3,7 @@
// license that can be found in the LICENSE file.
#include "runtime.h"
+#include "textflag.h"
// Look up symbols in the Linux vDSO.
@@ -171,14 +172,18 @@ struct vdso_info {
Elf64_Verdef *verdef;
};
+#pragma dataflag NOPTR
static version_key linux26 = { (byte*)"LINUX_2.6", 0x3ae75f6 };
// initialize with vsyscall fallbacks
+#pragma dataflag NOPTR
void* runtime·__vdso_time_sym = (void*)0xffffffffff600400ULL;
+#pragma dataflag NOPTR
void* runtime·__vdso_gettimeofday_sym = (void*)0xffffffffff600000ULL;
+#pragma dataflag NOPTR
void* runtime·__vdso_clock_gettime_sym = (void*)0;
-#define SYM_KEYS_COUNT 3
+#pragma dataflag NOPTR
static symbol_key sym_keys[] = {
{ (byte*)"__vdso_time", 0xa33c485, &runtime·__vdso_time_sym },
{ (byte*)"__vdso_gettimeofday", 0x315ca59, &runtime·__vdso_gettimeofday_sym },
@@ -301,7 +306,7 @@ vdso_parse_symbols(struct vdso_info *vdso_info, int32 version)
if(vdso_info->valid == false)
return;
- for(i=0; i<SYM_KEYS_COUNT; i++) {
+ for(i=0; i<nelem(sym_keys); i++) {
for(chain = vdso_info->bucket[sym_keys[i].sym_hash % vdso_info->nbucket];
chain != 0; chain = vdso_info->chain[chain]) {