all: merge default branch into dev.garbage

hg was unable to create a CL on the code review server for this,
so I am submitting the merge by hand.
The only manual edits are in mgc0.c, to reapply the
removal of cached/ncached to the new code.

26 files changed, 558 insertions, 295 deletions

diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index 1495246a2..b0ed2d8ce 100644
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -353,7 +353,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
 	JMP	AX
 // Note: can't just "JMP NAME(SB)" - bad inlining results.
 
-TEXT runtime·reflectcall(SB), NOSPLIT, $0-16
+TEXT ·reflectcall(SB), NOSPLIT, $0-16
 	MOVL	argsize+8(FP), CX
 	DISPATCH(runtime·call16, 16)
 	DISPATCH(runtime·call32, 32)
@@ -385,21 +385,9 @@ TEXT runtime·reflectcall(SB), NOSPLIT, $0-16
 	MOVL	$runtime·badreflectcall(SB), AX
 	JMP	AX
 
-// Argument map for the callXX frames.  Each has one stack map.
-DATA gcargs_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gcargs_reflectcall<>+0x04(SB)/4, $8  // 4 words
-DATA gcargs_reflectcall<>+0x08(SB)/1, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsScalar<<4)+(const_BitsScalar<<6))
-GLOBL gcargs_reflectcall<>(SB),RODATA,$12
-
-// callXX frames have no locals
-DATA gclocals_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gclocals_reflectcall<>+0x04(SB)/4, $0  // 0 locals
-GLOBL gclocals_reflectcall<>(SB),RODATA,$8
-
 #define CALLFN(NAME,MAXSIZE)			\
-TEXT NAME(SB), WRAPPER, $MAXSIZE-16;	\
-	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_reflectcall<>(SB);	\
-	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_reflectcall<>(SB);\
+TEXT NAME(SB), WRAPPER, $MAXSIZE-16;		\
+	NO_LOCAL_POINTERS;			\
 	/* copy arguments to stack */		\
 	MOVL	argptr+4(FP), SI;		\
 	MOVL	argsize+8(FP), CX;		\
@@ -421,33 +409,33 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-16;	\
 	REP;MOVSB;				\
 	RET
 
-CALLFN(runtime·call16, 16)
-CALLFN(runtime·call32, 32)
-CALLFN(runtime·call64, 64)
-CALLFN(runtime·call128, 128)
-CALLFN(runtime·call256, 256)
-CALLFN(runtime·call512, 512)
-CALLFN(runtime·call1024, 1024)
-CALLFN(runtime·call2048, 2048)
-CALLFN(runtime·call4096, 4096)
-CALLFN(runtime·call8192, 8192)
-CALLFN(runtime·call16384, 16384)
-CALLFN(runtime·call32768, 32768)
-CALLFN(runtime·call65536, 65536)
-CALLFN(runtime·call131072, 131072)
-CALLFN(runtime·call262144, 262144)
-CALLFN(runtime·call524288, 524288)
-CALLFN(runtime·call1048576, 1048576)
-CALLFN(runtime·call2097152, 2097152)
-CALLFN(runtime·call4194304, 4194304)
-CALLFN(runtime·call8388608, 8388608)
-CALLFN(runtime·call16777216, 16777216)
-CALLFN(runtime·call33554432, 33554432)
-CALLFN(runtime·call67108864, 67108864)
-CALLFN(runtime·call134217728, 134217728)
-CALLFN(runtime·call268435456, 268435456)
-CALLFN(runtime·call536870912, 536870912)
-CALLFN(runtime·call1073741824, 1073741824)
+CALLFN(·call16, 16)
+CALLFN(·call32, 32)
+CALLFN(·call64, 64)
+CALLFN(·call128, 128)
+CALLFN(·call256, 256)
+CALLFN(·call512, 512)
+CALLFN(·call1024, 1024)
+CALLFN(·call2048, 2048)
+CALLFN(·call4096, 4096)
+CALLFN(·call8192, 8192)
+CALLFN(·call16384, 16384)
+CALLFN(·call32768, 32768)
+CALLFN(·call65536, 65536)
+CALLFN(·call131072, 131072)
+CALLFN(·call262144, 262144)
+CALLFN(·call524288, 524288)
+CALLFN(·call1048576, 1048576)
+CALLFN(·call2097152, 2097152)
+CALLFN(·call4194304, 4194304)
+CALLFN(·call8388608, 8388608)
+CALLFN(·call16777216, 16777216)
+CALLFN(·call33554432, 33554432)
+CALLFN(·call67108864, 67108864)
+CALLFN(·call134217728, 134217728)
+CALLFN(·call268435456, 268435456)
+CALLFN(·call536870912, 536870912)
+CALLFN(·call1073741824, 1073741824)
 
 // bool cas(int32 *val, int32 old, int32 new)
 // Atomically:
@@ -479,6 +467,9 @@ TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $0-8
 TEXT runtime·atomicloaduint(SB), NOSPLIT, $0-8
 	JMP	runtime·atomicload(SB)
 
+TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-8
+	JMP	runtime·atomicstore(SB)
+
 // bool runtime·cas64(uint64 *val, uint64 old, uint64 new)
 // Atomically:
 //	if(*val == *old){
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index 3f7f60841..2ee331208 100644
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -339,11 +339,11 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
 #define DISPATCH(NAME,MAXSIZE)		\
 	CMPQ	CX, $MAXSIZE;		\
 	JA	3(PC);			\
-	MOVQ	$NAME(SB), AX;	\
+	MOVQ	$NAME(SB), AX;		\
 	JMP	AX
 // Note: can't just "JMP NAME(SB)" - bad inlining results.
 
-TEXT runtime·reflectcall(SB), NOSPLIT, $0-24
+TEXT ·reflectcall(SB), NOSPLIT, $0-24
 	MOVLQZX argsize+16(FP), CX
 	DISPATCH(runtime·call16, 16)
 	DISPATCH(runtime·call32, 32)
@@ -375,21 +375,9 @@ TEXT runtime·reflectcall(SB), NOSPLIT, $0-24
 	MOVQ	$runtime·badreflectcall(SB), AX
 	JMP	AX
 
-// Argument map for the callXX frames.  Each has one stack map.
-DATA gcargs_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gcargs_reflectcall<>+0x04(SB)/4, $6  // 3 words
-DATA gcargs_reflectcall<>+0x08(SB)/1, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsScalar<<4))
-GLOBL gcargs_reflectcall<>(SB),RODATA,$12
-
-// callXX frames have no locals
-DATA gclocals_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gclocals_reflectcall<>+0x04(SB)/4, $0  // 0 locals
-GLOBL gclocals_reflectcall<>(SB),RODATA,$8
-
 #define CALLFN(NAME,MAXSIZE)			\
 TEXT NAME(SB), WRAPPER, $MAXSIZE-24;		\
-	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_reflectcall<>(SB);	\
-	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_reflectcall<>(SB);\
+	NO_LOCAL_POINTERS;			\
 	/* copy arguments to stack */		\
 	MOVQ	argptr+8(FP), SI;		\
 	MOVLQZX argsize+16(FP), CX;		\
@@ -410,33 +398,33 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24;		\
 	REP;MOVSB;				\
 	RET
 
-CALLFN(runtime·call16, 16)
-CALLFN(runtime·call32, 32)
-CALLFN(runtime·call64, 64)
-CALLFN(runtime·call128, 128)
-CALLFN(runtime·call256, 256)
-CALLFN(runtime·call512, 512)
-CALLFN(runtime·call1024, 1024)
-CALLFN(runtime·call2048, 2048)
-CALLFN(runtime·call4096, 4096)
-CALLFN(runtime·call8192, 8192)
-CALLFN(runtime·call16384, 16384)
-CALLFN(runtime·call32768, 32768)
-CALLFN(runtime·call65536, 65536)
-CALLFN(runtime·call131072, 131072)
-CALLFN(runtime·call262144, 262144)
-CALLFN(runtime·call524288, 524288)
-CALLFN(runtime·call1048576, 1048576)
-CALLFN(runtime·call2097152, 2097152)
-CALLFN(runtime·call4194304, 4194304)
-CALLFN(runtime·call8388608, 8388608)
-CALLFN(runtime·call16777216, 16777216)
-CALLFN(runtime·call33554432, 33554432)
-CALLFN(runtime·call67108864, 67108864)
-CALLFN(runtime·call134217728, 134217728)
-CALLFN(runtime·call268435456, 268435456)
-CALLFN(runtime·call536870912, 536870912)
-CALLFN(runtime·call1073741824, 1073741824)
+CALLFN(·call16, 16)
+CALLFN(·call32, 32)
+CALLFN(·call64, 64)
+CALLFN(·call128, 128)
+CALLFN(·call256, 256)
+CALLFN(·call512, 512)
+CALLFN(·call1024, 1024)
+CALLFN(·call2048, 2048)
+CALLFN(·call4096, 4096)
+CALLFN(·call8192, 8192)
+CALLFN(·call16384, 16384)
+CALLFN(·call32768, 32768)
+CALLFN(·call65536, 65536)
+CALLFN(·call131072, 131072)
+CALLFN(·call262144, 262144)
+CALLFN(·call524288, 524288)
+CALLFN(·call1048576, 1048576)
+CALLFN(·call2097152, 2097152)
+CALLFN(·call4194304, 4194304)
+CALLFN(·call8388608, 8388608)
+CALLFN(·call16777216, 16777216)
+CALLFN(·call33554432, 33554432)
+CALLFN(·call67108864, 67108864)
+CALLFN(·call134217728, 134217728)
+CALLFN(·call268435456, 268435456)
+CALLFN(·call536870912, 536870912)
+CALLFN(·call1073741824, 1073741824)
 
 // bool cas(int32 *val, int32 old, int32 new)
 // Atomically:
@@ -491,6 +479,9 @@ TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $0-16
 TEXT runtime·atomicloaduint(SB), NOSPLIT, $0-16
 	JMP	runtime·atomicload64(SB)
 
+TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16
+	JMP	runtime·atomicstore64(SB)
+
 // bool casp(void **val, void *old, void *new)
 // Atomically:
 //	if(*val == old){
diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s
index 13a164256..e27f67e1e 100644
--- a/src/runtime/asm_amd64p32.s
+++ b/src/runtime/asm_amd64p32.s
@@ -310,11 +310,11 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
 #define DISPATCH(NAME,MAXSIZE)		\
 	CMPL	CX, $MAXSIZE;		\
 	JA	3(PC);			\
-	MOVL	$NAME(SB), AX;	\
+	MOVL	$NAME(SB), AX;		\
 	JMP	AX
 // Note: can't just "JMP NAME(SB)" - bad inlining results.
 
-TEXT runtime·reflectcall(SB), NOSPLIT, $0-16
+TEXT ·reflectcall(SB), NOSPLIT, $0-16
 	MOVLQZX argsize+8(FP), CX
 	DISPATCH(runtime·call16, 16)
 	DISPATCH(runtime·call32, 32)
@@ -346,22 +346,9 @@ TEXT runtime·reflectcall(SB), NOSPLIT, $0-16
 	MOVL	$runtime·badreflectcall(SB), AX
 	JMP	AX
 
-// Argument map for the callXX frames.  Each has one stack map.
-DATA gcargs_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gcargs_reflectcall<>+0x04(SB)/4, $10  // 5 words
-DATA gcargs_reflectcall<>+0x08(SB)/1, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsScalar<<4)+(const_BitsScalar<<6))
-DATA gcargs_reflectcall<>+0x09(SB)/1, $(const_BitsPointer)
-GLOBL gcargs_reflectcall<>(SB),RODATA,$12
-
-// callXX frames have no locals
-DATA gclocals_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gclocals_reflectcall<>+0x04(SB)/4, $0  // 0 locals
-GLOBL gclocals_reflectcall<>(SB),RODATA,$8
-
 #define CALLFN(NAME,MAXSIZE)			\
 TEXT NAME(SB), WRAPPER, $MAXSIZE-16;		\
-	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_reflectcall<>(SB);	\
-	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_reflectcall<>(SB);\
+	NO_LOCAL_POINTERS;			\
 	/* copy arguments to stack */		\
 	MOVL	argptr+4(FP), SI;		\
 	MOVL	argsize+8(FP), CX;		\
@@ -369,8 +356,8 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-16;		\
 	REP;MOVSB;				\
 	/* call function */			\
 	MOVL	f+0(FP), DX;			\
-	MOVL	(DX), AX;				\
-	CALL	AX; \
+	MOVL	(DX), AX;			\
+	CALL	AX;				\
 	/* copy return values back */		\
 	MOVL	argptr+4(FP), DI;		\
 	MOVL	argsize+8(FP), CX;		\
@@ -382,33 +369,33 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-16;		\
 	REP;MOVSB;				\
 	RET
 
-CALLFN(runtime·call16, 16)
-CALLFN(runtime·call32, 32)
-CALLFN(runtime·call64, 64)
-CALLFN(runtime·call128, 128)
-CALLFN(runtime·call256, 256)
-CALLFN(runtime·call512, 512)
-CALLFN(runtime·call1024, 1024)
-CALLFN(runtime·call2048, 2048)
-CALLFN(runtime·call4096, 4096)
-CALLFN(runtime·call8192, 8192)
-CALLFN(runtime·call16384, 16384)
-CALLFN(runtime·call32768, 32768)
-CALLFN(runtime·call65536, 65536)
-CALLFN(runtime·call131072, 131072)
-CALLFN(runtime·call262144, 262144)
-CALLFN(runtime·call524288, 524288)
-CALLFN(runtime·call1048576, 1048576)
-CALLFN(runtime·call2097152, 2097152)
-CALLFN(runtime·call4194304, 4194304)
-CALLFN(runtime·call8388608, 8388608)
-CALLFN(runtime·call16777216, 16777216)
-CALLFN(runtime·call33554432, 33554432)
-CALLFN(runtime·call67108864, 67108864)
-CALLFN(runtime·call134217728, 134217728)
-CALLFN(runtime·call268435456, 268435456)
-CALLFN(runtime·call536870912, 536870912)
-CALLFN(runtime·call1073741824, 1073741824)
+CALLFN(·call16, 16)
+CALLFN(·call32, 32)
+CALLFN(·call64, 64)
+CALLFN(·call128, 128)
+CALLFN(·call256, 256)
+CALLFN(·call512, 512)
+CALLFN(·call1024, 1024)
+CALLFN(·call2048, 2048)
+CALLFN(·call4096, 4096)
+CALLFN(·call8192, 8192)
+CALLFN(·call16384, 16384)
+CALLFN(·call32768, 32768)
+CALLFN(·call65536, 65536)
+CALLFN(·call131072, 131072)
+CALLFN(·call262144, 262144)
+CALLFN(·call524288, 524288)
+CALLFN(·call1048576, 1048576)
+CALLFN(·call2097152, 2097152)
+CALLFN(·call4194304, 4194304)
+CALLFN(·call8388608, 8388608)
+CALLFN(·call16777216, 16777216)
+CALLFN(·call33554432, 33554432)
+CALLFN(·call67108864, 67108864)
+CALLFN(·call134217728, 134217728)
+CALLFN(·call268435456, 268435456)
+CALLFN(·call536870912, 536870912)
+CALLFN(·call1073741824, 1073741824)
 
 // bool cas(int32 *val, int32 old, int32 new)
 // Atomically:
@@ -440,6 +427,9 @@ TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $0-12
 TEXT runtime·atomicloaduint(SB), NOSPLIT, $0-12
 	JMP	runtime·atomicload(SB)
 
+TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-12
+	JMP	runtime·atomicstore(SB)
+
 // bool	runtime·cas64(uint64 *val, uint64 old, uint64 new)
 // Atomically:
 //	if(*val == *old){
diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s
index 36fb022f9..b21441488 100644
--- a/src/runtime/asm_arm.s
+++ b/src/runtime/asm_arm.s
@@ -343,7 +343,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0
 	MOVW	$NAME(SB), R1;		\
 	B	(R1)
 
-TEXT runtime·reflectcall(SB),NOSPLIT,$-4-16
+TEXT ·reflectcall(SB),NOSPLIT,$-4-16
 	MOVW	argsize+8(FP), R0
 	DISPATCH(runtime·call16, 16)
 	DISPATCH(runtime·call32, 32)
@@ -375,21 +375,9 @@ TEXT runtime·reflectcall(SB),NOSPLIT,$-4-16
 	MOVW	$runtime·badreflectcall(SB), R1
 	B	(R1)
 
-// Argument map for the callXX frames.  Each has one stack map.
-DATA gcargs_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gcargs_reflectcall<>+0x04(SB)/4, $8  // 4 words
-DATA gcargs_reflectcall<>+0x08(SB)/1, $(const_BitsPointer+(const_BitsPointer<<2)+(const_BitsScalar<<4)+(const_BitsScalar<<6))
-GLOBL gcargs_reflectcall<>(SB),RODATA,$12
-
-// callXX frames have no locals
-DATA gclocals_reflectcall<>+0x00(SB)/4, $1  // 1 stackmap
-DATA gclocals_reflectcall<>+0x04(SB)/4, $0  // 0 locals
-GLOBL gclocals_reflectcall<>(SB),RODATA,$8
-
 #define CALLFN(NAME,MAXSIZE)			\
 TEXT NAME(SB), WRAPPER, $MAXSIZE-16;		\
-	FUNCDATA $FUNCDATA_ArgsPointerMaps,gcargs_reflectcall<>(SB);	\
-	FUNCDATA $FUNCDATA_LocalsPointerMaps,gclocals_reflectcall<>(SB);\
+	NO_LOCAL_POINTERS;			\
 	/* copy arguments to stack */		\
 	MOVW	argptr+4(FP), R0;		\
 	MOVW	argsize+8(FP), R2;		\
@@ -420,33 +408,33 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-16;		\
 	SUB	$1, R2, R2;			\
 	B	-5(PC)				\
 
-CALLFN(runtime·call16, 16)
-CALLFN(runtime·call32, 32)
-CALLFN(runtime·call64, 64)
-CALLFN(runtime·call128, 128)
-CALLFN(runtime·call256, 256)
-CALLFN(runtime·call512, 512)
-CALLFN(runtime·call1024, 1024)
-CALLFN(runtime·call2048, 2048)
-CALLFN(runtime·call4096, 4096)
-CALLFN(runtime·call8192, 8192)
-CALLFN(runtime·call16384, 16384)
-CALLFN(runtime·call32768, 32768)
-CALLFN(runtime·call65536, 65536)
-CALLFN(runtime·call131072, 131072)
-CALLFN(runtime·call262144, 262144)
-CALLFN(runtime·call524288, 524288)
-CALLFN(runtime·call1048576, 1048576)
-CALLFN(runtime·call2097152, 2097152)
-CALLFN(runtime·call4194304, 4194304)
-CALLFN(runtime·call8388608, 8388608)
-CALLFN(runtime·call16777216, 16777216)
-CALLFN(runtime·call33554432, 33554432)
-CALLFN(runtime·call67108864, 67108864)
-CALLFN(runtime·call134217728, 134217728)
-CALLFN(runtime·call268435456, 268435456)
-CALLFN(runtime·call536870912, 536870912)
-CALLFN(runtime·call1073741824, 1073741824)
+CALLFN(·call16, 16)
+CALLFN(·call32, 32)
+CALLFN(·call64, 64)
+CALLFN(·call128, 128)
+CALLFN(·call256, 256)
+CALLFN(·call512, 512)
+CALLFN(·call1024, 1024)
+CALLFN(·call2048, 2048)
+CALLFN(·call4096, 4096)
+CALLFN(·call8192, 8192)
+CALLFN(·call16384, 16384)
+CALLFN(·call32768, 32768)
+CALLFN(·call65536, 65536)
+CALLFN(·call131072, 131072)
+CALLFN(·call262144, 262144)
+CALLFN(·call524288, 524288)
+CALLFN(·call1048576, 1048576)
+CALLFN(·call2097152, 2097152)
+CALLFN(·call4194304, 4194304)
+CALLFN(·call8388608, 8388608)
+CALLFN(·call16777216, 16777216)
+CALLFN(·call33554432, 33554432)
+CALLFN(·call67108864, 67108864)
+CALLFN(·call134217728, 134217728)
+CALLFN(·call268435456, 268435456)
+CALLFN(·call536870912, 536870912)
+CALLFN(·call1073741824, 1073741824)
 
 // void jmpdefer(fn, sp);
 // called from deferreturn.
@@ -724,6 +712,9 @@ TEXT runtime·atomicloaduintptr(SB),NOSPLIT,$0-8
 TEXT runtime·atomicloaduint(SB),NOSPLIT,$0-8
 	B	runtime·atomicload(SB)
 
+TEXT runtime·atomicstoreuintptr(SB),NOSPLIT,$0-8
+	B	runtime·atomicstore(SB)
+
 // AES hashing not implemented for ARM
 TEXT runtime·aeshash(SB),NOSPLIT,$-4-0
 	MOVW	$0, R0
diff --git a/src/runtime/cgocallback.go b/src/runtime/cgocallback.go
index 1e1b57607..2c8914320 100644
--- a/src/runtime/cgocallback.go
+++ b/src/runtime/cgocallback.go
@@ -21,6 +21,9 @@ import "unsafe"
 // Either we need to add types or we need to stop using it.
 
 func _cgo_allocate_internal(len uintptr) unsafe.Pointer {
+	if len == 0 {
+		len = 1
+	}
 	ret := unsafe.Pointer(&make([]unsafe.Pointer, (len+ptrSize-1)/ptrSize)[0])
 	c := new(cgomal)
 	c.alloc = ret
diff --git a/src/runtime/chan.go b/src/runtime/chan.go
index 10503f4e1..004970182 100644
--- a/src/runtime/chan.go
+++ b/src/runtime/chan.go
@@ -174,6 +174,10 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
 		goparkunlock(&c.lock, "chan send")
 
 		// someone woke us up.
+		if mysg != gp.waiting {
+			gothrow("G waiting list is corrupted!")
+		}
+		gp.waiting = nil
 		if gp.param == nil {
 			if c.closed == 0 {
 				gothrow("chansend: spurious wakeup")
@@ -184,10 +188,6 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
 		if mysg.releasetime > 0 {
 			blockevent(int64(mysg.releasetime)-t0, 2)
 		}
-		if mysg != gp.waiting {
-			gothrow("G waiting list is corrupted!")
-		}
-		gp.waiting = nil
 		releaseSudog(mysg)
 		return true
 	}
@@ -410,6 +410,9 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
 		goparkunlock(&c.lock, "chan receive")
 
 		// someone woke us up
+		if mysg != gp.waiting {
+			gothrow("G waiting list is corrupted!")
+		}
 		gp.waiting = nil
 		if mysg.releasetime > 0 {
 			blockevent(mysg.releasetime-t0, 2)
diff --git a/src/runtime/defs_windows.go b/src/runtime/defs_windows.go
index cb0f54d8a..7ce679741 100644
--- a/src/runtime/defs_windows.go
+++ b/src/runtime/defs_windows.go
@@ -49,6 +49,7 @@ const (
 	CONTEXT_FULL    = C.CONTEXT_FULL
 
 	EXCEPTION_ACCESS_VIOLATION     = C.STATUS_ACCESS_VIOLATION
+	EXCEPTION_BREAKPOINT           = C.STATUS_BREAKPOINT
 	EXCEPTION_FLT_DENORMAL_OPERAND = C.STATUS_FLOAT_DENORMAL_OPERAND
 	EXCEPTION_FLT_DIVIDE_BY_ZERO   = C.STATUS_FLOAT_DIVIDE_BY_ZERO
 	EXCEPTION_FLT_INEXACT_RESULT   = C.STATUS_FLOAT_INEXACT_RESULT
@@ -59,6 +60,9 @@ const (
 
 	INFINITE     = C.INFINITE
 	WAIT_TIMEOUT = C.WAIT_TIMEOUT
+
+	EXCEPTION_CONTINUE_EXECUTION = C.EXCEPTION_CONTINUE_EXECUTION
+	EXCEPTION_CONTINUE_SEARCH    = C.EXCEPTION_CONTINUE_SEARCH
 )
 
 type SystemInfo C.SYSTEM_INFO
diff --git a/src/runtime/defs_windows_386.h b/src/runtime/defs_windows_386.h
index 295e422c6..2317c04f6 100644
--- a/src/runtime/defs_windows_386.h
+++ b/src/runtime/defs_windows_386.h
@@ -22,6 +22,7 @@ enum {
 	CONTEXT_FULL	= 0x10007,
 
 	EXCEPTION_ACCESS_VIOLATION	= 0xc0000005,
+	EXCEPTION_BREAKPOINT		= 0x80000003,
 	EXCEPTION_FLT_DENORMAL_OPERAND	= 0xc000008d,
 	EXCEPTION_FLT_DIVIDE_BY_ZERO	= 0xc000008e,
 	EXCEPTION_FLT_INEXACT_RESULT	= 0xc000008f,
@@ -32,6 +33,9 @@ enum {
 
 	INFINITE	= 0xffffffff,
 	WAIT_TIMEOUT	= 0x102,
+
+	EXCEPTION_CONTINUE_EXECUTION	= -0x1,
+	EXCEPTION_CONTINUE_SEARCH	= 0x0,
 };
 
 typedef struct SystemInfo SystemInfo;
diff --git a/src/runtime/defs_windows_amd64.h b/src/runtime/defs_windows_amd64.h
index 2516c8412..7f37a7a8c 100644
--- a/src/runtime/defs_windows_amd64.h
+++ b/src/runtime/defs_windows_amd64.h
@@ -22,6 +22,7 @@ enum {
 	CONTEXT_FULL	= 0x10000b,
 
 	EXCEPTION_ACCESS_VIOLATION	= 0xc0000005,
+	EXCEPTION_BREAKPOINT		= 0x80000003,
 	EXCEPTION_FLT_DENORMAL_OPERAND	= 0xc000008d,
 	EXCEPTION_FLT_DIVIDE_BY_ZERO	= 0xc000008e,
 	EXCEPTION_FLT_INEXACT_RESULT	= 0xc000008f,
@@ -32,6 +33,9 @@ enum {
 
 	INFINITE	= 0xffffffff,
 	WAIT_TIMEOUT	= 0x102,
+
+	EXCEPTION_CONTINUE_EXECUTION	= -0x1,
+	EXCEPTION_CONTINUE_SEARCH	= 0x0,
 };
 
 typedef struct SystemInfo SystemInfo;
diff --git a/src/runtime/heapdump.c b/src/runtime/heapdump.c
index 54b9666b5..71da419f1 100644
--- a/src/runtime/heapdump.c
+++ b/src/runtime/heapdump.c
@@ -7,7 +7,7 @@
 // finalizers, etc.) to a file.
 
 // The format of the dumped file is described at
-// http://code.google.com/p/go-wiki/wiki/heapdump13
+// http://code.google.com/p/go-wiki/wiki/heapdump14
 
 #include "runtime.h"
 #include "arch_GOARCH.h"
@@ -27,10 +27,8 @@ extern byte runtime·ebss[];
 enum {
 	FieldKindEol = 0,
 	FieldKindPtr = 1,
-	FieldKindString = 2,
-	FieldKindSlice = 3,
-	FieldKindIface = 4,
-	FieldKindEface = 5,
+	FieldKindIface = 2,
+	FieldKindEface = 3,
 
 	TagEOF = 0,
 	TagObject = 1,
@@ -200,7 +198,6 @@ dumptype(Type *t)
 		write(t->x->name->str, t->x->name->len);
 	}
 	dumpbool((t->kind & KindDirectIface) == 0 || (t->kind & KindNoPointers) == 0);
-	dumpfields((BitVector){0, nil});
 }
 
 // dump an object
@@ -210,9 +207,8 @@ dumpobj(byte *obj, uintptr size, BitVector bv)
 	dumpbvtypes(&bv, obj);
 	dumpint(TagObject);
 	dumpint((uintptr)obj);
-	dumpint(0); // Type*
-	dumpint(0); // kind
 	dumpmemrange(obj, size);
+	dumpfields(bv);
 }
 
 static void
@@ -255,6 +251,7 @@ dumpbv(BitVector *bv, uintptr offset)
 	for(i = 0; i < bv->n; i += BitsPerPointer) {
 		switch(bv->bytedata[i/8] >> i%8 & 3) {
 		case BitsDead:
+			return;
 		case BitsScalar:
 			break;
 		case BitsPointer:
@@ -489,16 +486,18 @@ dumproots(void)
 	byte *p;
 
 	// data segment
+	dumpbvtypes(&runtime·gcdatamask, runtime·data);
 	dumpint(TagData);
 	dumpint((uintptr)runtime·data);
 	dumpmemrange(runtime·data, runtime·edata - runtime·data);
 	dumpfields(runtime·gcdatamask);
 
 	// bss segment
+	dumpbvtypes(&runtime·gcbssmask, runtime·bss);
 	dumpint(TagBss);
 	dumpint((uintptr)runtime·bss);
 	dumpmemrange(runtime·bss, runtime·ebss - runtime·bss);
-	dumpfields(runtime·gcdatamask);
+	dumpfields(runtime·gcbssmask);
 
 	// MSpan.types
 	allspans = runtime·mheap.allspans;
@@ -578,10 +577,29 @@ itab_callback(Itab *tab)
 {
 	Type *t;
 
-	dumpint(TagItab);
-	dumpint((uintptr)tab);
 	t = tab->type;
-	dumpbool((t->kind & KindDirectIface) == 0 || (t->kind & KindNoPointers) == 0);
+	// Dump a map from itab* to the type of its data field.
+	// We want this map so we can deduce types of interface referents.
+	if((t->kind & KindDirectIface) == 0) {
+		// indirect - data slot is a pointer to t.
+		dumptype(t->ptrto);
+		dumpint(TagItab);
+		dumpint((uintptr)tab);
+		dumpint((uintptr)t->ptrto);
+	} else if((t->kind & KindNoPointers) == 0) {
+		// t is pointer-like - data slot is a t.
+		dumptype(t);
+		dumpint(TagItab);
+		dumpint((uintptr)tab);
+		dumpint((uintptr)t);
+	} else {
+		// Data slot is a scalar.  Dump type just for fun.
+		// With pointer-only interfaces, this shouldn't happen.
+		dumptype(t);
+		dumpint(TagItab);
+		dumpint((uintptr)tab);
+		dumpint((uintptr)t);
+	}
 }
 
 static void
@@ -726,7 +744,7 @@ mdump(void)
 	}
 
 	runtime·memclr((byte*)&typecache[0], sizeof(typecache));
-	hdr = (byte*)"go1.3 heap dump\n";
+	hdr = (byte*)"go1.4 heap dump\n";
 	write(hdr, runtime·findnull(hdr));
 	dumpparams();
 	dumpitabs();
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 99d14e314..9b4264f2b 100644
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -270,7 +270,7 @@ func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer {
 			}
 			ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte
 		} else {
-			ptrmask = (*uint8)(unsafe.Pointer(&typ.gc[0])) // embed mask
+			ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask
 		}
 		if size == 2*ptrSize {
 			*xbits = *ptrmask | bitBoundary
diff --git a/src/runtime/mgc0.c b/src/runtime/mgc0.c
index dabd38a60..8620f47af 100644
--- a/src/runtime/mgc0.c
+++ b/src/runtime/mgc0.c
@@ -383,25 +383,19 @@ greyobject(byte *obj, Markbits *mbits, Workbuf *wbuf)
 static Workbuf*
 scanobject(byte *b, uintptr n, byte *ptrmask, Workbuf *wbuf)
 {
-	byte *obj, *arena_start, *arena_used, *ptrbitp, bits, cshift, cached;
-	uintptr i;
-	intptr ncached;
+	byte *obj, *arena_start, *arena_used, *ptrbitp;
+	uintptr i, j;
+	int32 bits;
 	Markbits mbits;
 
 	arena_start = (byte*)runtime·mheap.arena_start;
 	arena_used = runtime·mheap.arena_used;
 	ptrbitp = nil;
-	cached = 0;
-	ncached = 0;
 
 	// Find bits of the beginning of the object.
 	if(ptrmask == nil) {
 		b = objectstart(b, &mbits);
 		ptrbitp = mbits.bitp; //arena_start - off/wordsPerBitmapByte - 1;
-		cshift = mbits.shift; //(off % wordsPerBitmapByte) * gcBits;
-		cached = *ptrbitp >> cshift;
-		cached &= ~bitBoundary;
-		ncached = (8 - cshift)/gcBits;
 	}
 	for(i = 0; i < n; i += PtrSize) {
 		// Find bits for this word.
@@ -414,26 +408,28 @@ scanobject(byte *b, uintptr n, byte *ptrmask, Workbuf *wbuf)
 				runtime·mheap.spans[(b-arena_start)>>PageShift] != runtime·mheap.spans[(b+i-arena_start)>>PageShift])
 				break;
 			// Consult GC bitmap.
-			if(ncached <= 0) {
-				// Refill cache.
-				cached = *--ptrbitp;
-				ncached = 2;
-			}
-			bits = cached;
-			cached >>= gcBits;
-			ncached--;
-			
-			if((bits&bitBoundary) != 0)
+			bits = *ptrbitp;
+			if(wordsPerBitmapByte != 2)
+				runtime·throw("alg doesn't work for wordsPerBitmapByte != 2");
+			j = ((uintptr)b+i)/PtrSize & 1;
+			bits >>= gcBits*j;
+			if(i == 0)
+				bits &= ~bitBoundary;
+			ptrbitp -= j;
+		
+			if((bits&bitBoundary) != 0 && i != 0)
 				break; // reached beginning of the next object
 			bits = (bits>>2)&BitsMask;
 			if(bits == BitsDead)
 				break; // reached no-scan part of the object
 		} 
 
-		if(bits == BitsScalar || bits == BitsDead)
+		if(bits <= BitsScalar) // Bits Scalar || BitsDead
 			continue;
-		if(bits != BitsPointer)
+		if(bits != BitsPointer) {
+			runtime·printf("gc bits=%x\n", bits);
 			runtime·throw("unexpected garbage collection bits");
+		}
 
 		obj = *(byte**)(b+i);
 		// At this point we have extracted the next potential pointer.
@@ -1608,6 +1604,8 @@ gc(struct gc_args *args)
 	if(runtime·work.nproc > 1)
 		runtime·notesleep(&runtime·work.alldone);
 
+	runtime·shrinkfinish();
+
 	cachestats();
 	// next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
 	// estimate what was live heap size after previous GC (for tracing only)
diff --git a/src/runtime/mgc0.go b/src/runtime/mgc0.go
index 0e17599c2..3a7204b54 100644
--- a/src/runtime/mgc0.go
+++ b/src/runtime/mgc0.go
@@ -110,6 +110,27 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
 }
 
 //go:nosplit
+func writebarrierfat2(dst *[2]uintptr, _ *byte, src [2]uintptr) {
+	dst[0] = src[0]
+	dst[1] = src[1]
+}
+
+//go:nosplit
+func writebarrierfat3(dst *[3]uintptr, _ *byte, src [3]uintptr) {
+	dst[0] = src[0]
+	dst[1] = src[1]
+	dst[2] = src[2]
+}
+
+//go:nosplit
+func writebarrierfat4(dst *[4]uintptr, _ *byte, src [4]uintptr) {
+	dst[0] = src[0]
+	dst[1] = src[1]
+	dst[2] = src[2]
+	dst[3] = src[3]
+}
+
+//go:nosplit
 func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
 	memmove(dst, src, typ.size)
 }
diff --git a/src/runtime/mgc0.h b/src/runtime/mgc0.h
index 10f24d009..64f818914 100644
--- a/src/runtime/mgc0.h
+++ b/src/runtime/mgc0.h
@@ -42,6 +42,8 @@ enum {
 	BitsMask	= (1<<BitsPerPointer)-1,
 	PointersPerByte	= 8/BitsPerPointer,
 
+	// If you change these, also change scanblock.
+	// scanblock does "if(bits == BitsScalar || bits == BitsDead)" as "if(bits <= BitsScalar)".
 	BitsDead	= 0,
 	BitsScalar	= 1,
 	BitsPointer	= 2,
diff --git a/src/runtime/os_windows.c b/src/runtime/os_windows.c
index 77f99062c..b8b8eda5f 100644
--- a/src/runtime/os_windows.c
+++ b/src/runtime/os_windows.c
@@ -34,6 +34,7 @@
 #pragma dynimport runtime·SetEvent SetEvent "kernel32.dll"
 #pragma dynimport runtime·SetProcessPriorityBoost SetProcessPriorityBoost "kernel32.dll"
 #pragma dynimport runtime·SetThreadPriority SetThreadPriority "kernel32.dll"
+#pragma dynimport runtime·SetUnhandledExceptionFilter SetUnhandledExceptionFilter "kernel32.dll"
 #pragma dynimport runtime·SetWaitableTimer SetWaitableTimer "kernel32.dll"
 #pragma dynimport runtime·Sleep Sleep "kernel32.dll"
 #pragma dynimport runtime·SuspendThread SuspendThread "kernel32.dll"
@@ -65,6 +66,7 @@ extern void *runtime·SetConsoleCtrlHandler;
 extern void *runtime·SetEvent;
 extern void *runtime·SetProcessPriorityBoost;
 extern void *runtime·SetThreadPriority;
+extern void *runtime·SetUnhandledExceptionFilter;
 extern void *runtime·SetWaitableTimer;
 extern void *runtime·Sleep;
 extern void *runtime·SuspendThread;
@@ -77,7 +79,9 @@ void *runtime·GetQueuedCompletionStatusEx;
 
 extern uintptr runtime·externalthreadhandlerp;
 void runtime·externalthreadhandler(void);
-void runtime·sigtramp(void);
+void runtime·exceptiontramp(void);
+void runtime·firstcontinuetramp(void);
+void runtime·lastcontinuetramp(void);
 
 #pragma textflag NOSPLIT
 uintptr
@@ -106,12 +110,30 @@ void
 runtime·osinit(void)
 {
 	void *kernel32;
+	void *addVectoredContinueHandler;
+
+	kernel32 = runtime·stdcall1(runtime·LoadLibraryA, (uintptr)"kernel32.dll");
 
 	runtime·externalthreadhandlerp = (uintptr)runtime·externalthreadhandler;
 
-	runtime·stdcall2(runtime·AddVectoredExceptionHandler, 1, (uintptr)runtime·sigtramp);
+	runtime·stdcall2(runtime·AddVectoredExceptionHandler, 1, (uintptr)runtime·exceptiontramp);
+	addVectoredContinueHandler = nil;
+	if(kernel32 != nil)
+		addVectoredContinueHandler = runtime·stdcall2(runtime·GetProcAddress, (uintptr)kernel32, (uintptr)"AddVectoredContinueHandler");
+	if(addVectoredContinueHandler == nil || sizeof(void*) == 4) {
+		// use SetUnhandledExceptionFilter for windows-386 or
+		// if VectoredContinueHandler is unavailable.
+		// note: SetUnhandledExceptionFilter handler won't be called, if debugging.
+		runtime·stdcall1(runtime·SetUnhandledExceptionFilter, (uintptr)runtime·lastcontinuetramp);
+	} else {
+		runtime·stdcall2(addVectoredContinueHandler, 1, (uintptr)runtime·firstcontinuetramp);
+		runtime·stdcall2(addVectoredContinueHandler, 0, (uintptr)runtime·lastcontinuetramp);
+	}
+
 	runtime·stdcall2(runtime·SetConsoleCtrlHandler, (uintptr)runtime·ctrlhandler, 1);
+
 	runtime·stdcall1(runtime·timeBeginPeriod, 1);
+
 	runtime·ncpu = getproccount();
 	
 	// Windows dynamic priority boosting assumes that a process has different types
@@ -120,7 +142,6 @@ runtime·osinit(void)
 	// In such context dynamic priority boosting does nothing but harm, so we turn it off.
 	runtime·stdcall2(runtime·SetProcessPriorityBoost, -1, 1);
 
-	kernel32 = runtime·stdcall1(runtime·LoadLibraryA, (uintptr)"kernel32.dll");
 	if(kernel32 != nil) {
 		runtime·GetQueuedCompletionStatusEx = runtime·stdcall2(runtime·GetProcAddress, (uintptr)kernel32, (uintptr)"GetQueuedCompletionStatusEx");
 	}
@@ -268,19 +289,19 @@ runtime·mpreinit(M *mp)
 void
 runtime·minit(void)
 {
-	void *thandle;
+	uintptr thandle;
 
 	// -1 = current process, -2 = current thread
 	runtime·stdcall7(runtime·DuplicateHandle, -1, -2, -1, (uintptr)&thandle, 0, 0, DUPLICATE_SAME_ACCESS);
-	runtime·atomicstorep(&g->m->thread, thandle);
+	runtime·atomicstoreuintptr(&g->m->thread, thandle);
 }
 
 // Called from dropm to undo the effect of an minit.
 void
 runtime·unminit(void)
 {
-	runtime·stdcall1(runtime·CloseHandle, (uintptr)g->m->thread);
-	g->m->thread = nil;
+	runtime·stdcall1(runtime·CloseHandle, g->m->thread);
+	g->m->thread = 0;
 }
 
 // Described in http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/
@@ -467,6 +488,7 @@ runtime·issigpanic(uint32 code)
 	case EXCEPTION_FLT_INEXACT_RESULT:
 	case EXCEPTION_FLT_OVERFLOW:
 	case EXCEPTION_FLT_UNDERFLOW:
+	case EXCEPTION_BREAKPOINT:
 		return 1;
 	}
 	return 0;
@@ -475,10 +497,14 @@ runtime·issigpanic(uint32 code)
 void
 runtime·initsig(void)
 {
-	// following line keeps sigtramp alive at link stage
+	// following line keeps these functions alive at link stage
 	// if there's a better way please write it here
-	void *p = runtime·sigtramp;
-	USED(p);
+	void *e = runtime·exceptiontramp;
+	void *f = runtime·firstcontinuetramp;
+	void *l = runtime·lastcontinuetramp;
+	USED(e);
+	USED(f);
+	USED(l);
 }
 
 uint32
@@ -532,7 +558,7 @@ void
 runtime·profileloop1(void)
 {
 	M *mp, *allm;
-	void *thread;
+	uintptr thread;
 
 	runtime·stdcall2(runtime·SetThreadPriority, -2, THREAD_PRIORITY_HIGHEST);
 
@@ -540,11 +566,11 @@ runtime·profileloop1(void)
 		runtime·stdcall2(runtime·WaitForSingleObject, (uintptr)profiletimer, -1);
 		allm = runtime·atomicloadp(&runtime·allm);
 		for(mp = allm; mp != nil; mp = mp->alllink) {
-			thread = runtime·atomicloadp(&mp->thread);
+			thread = runtime·atomicloaduintptr(&mp->thread);
 			// Do not profile threads blocked on Notes,
 			// this includes idle worker threads,
 			// idle timer thread, idle heap scavenger, etc.
-			if(thread == nil || mp->profilehz == 0 || mp->blocked)
+			if(thread == 0 || mp->profilehz == 0 || mp->blocked)
 				continue;
 			runtime·stdcall1(runtime·SuspendThread, (uintptr)thread);
 			if(mp->profilehz != 0 && !mp->blocked)
diff --git a/src/runtime/os_windows_386.c b/src/runtime/os_windows_386.c
index e2ae8db27..213582799 100644
--- a/src/runtime/os_windows_386.c
+++ b/src/runtime/os_windows_386.c
@@ -24,45 +24,63 @@ runtime·dumpregs(Context *r)
 	runtime·printf("gs      %x\n", r->SegGs);
 }
 
-// Called by sigtramp from Windows VEH handler.
-// Return value signals whether the exception has been handled (-1)
-// or should be made available to other handlers in the chain (0).
-uint32
-runtime·sighandler(ExceptionRecord *info, Context *r, G *gp)
+bool
+runtime·isgoexception(ExceptionRecord *info, Context *r)
 {
-	bool crash;
-	uintptr *sp;
 	extern byte runtime·text[], runtime·etext[];
 
 	// Only handle exception if executing instructions in Go binary
 	// (not Windows library code). 
 	if(r->Eip < (uint32)runtime·text || (uint32)runtime·etext < r->Eip)
-		return 0;
-
-	if(gp != nil && runtime·issigpanic(info->ExceptionCode)) {
-		// Make it look like a call to the signal func.
-		// Have to pass arguments out of band since
-		// augmenting the stack frame would break
-		// the unwinding code.
-		gp->sig = info->ExceptionCode;
-		gp->sigcode0 = info->ExceptionInformation[0];
-		gp->sigcode1 = info->ExceptionInformation[1];
-		gp->sigpc = r->Eip;
-
-		// Only push runtime·sigpanic if r->eip != 0.
-		// If r->eip == 0, probably panicked because of a
-		// call to a nil func.  Not pushing that onto sp will
-		// make the trace look like a call to runtime·sigpanic instead.
-		// (Otherwise the trace will end at runtime·sigpanic and we
-		// won't get to see who faulted.)
-		if(r->Eip != 0) {
-			sp = (uintptr*)r->Esp;
-			*--sp = r->Eip;
-			r->Esp = (uintptr)sp;
-		}
-		r->Eip = (uintptr)runtime·sigpanic;
-		return -1;
+		return false;
+
+	if(!runtime·issigpanic(info->ExceptionCode))
+		return false;
+
+	return true;
+}
+
+// Called by sigtramp from Windows VEH handler.
+// Return value signals whether the exception has been handled (EXCEPTION_CONTINUE_EXECUTION)
+// or should be made available to other handlers in the chain (EXCEPTION_CONTINUE_SEARCH).
+uint32
+runtime·exceptionhandler(ExceptionRecord *info, Context *r, G *gp)
+{
+	uintptr *sp;
+
+	if(!runtime·isgoexception(info, r))
+		return EXCEPTION_CONTINUE_SEARCH;
+
+	// Make it look like a call to the signal func.
+	// Have to pass arguments out of band since
+	// augmenting the stack frame would break
+	// the unwinding code.
+	gp->sig = info->ExceptionCode;
+	gp->sigcode0 = info->ExceptionInformation[0];
+	gp->sigcode1 = info->ExceptionInformation[1];
+	gp->sigpc = r->Eip;
+
+	// Only push runtime·sigpanic if r->eip != 0.
+	// If r->eip == 0, probably panicked because of a
+	// call to a nil func.  Not pushing that onto sp will
+	// make the trace look like a call to runtime·sigpanic instead.
+	// (Otherwise the trace will end at runtime·sigpanic and we
+	// won't get to see who faulted.)
+	if(r->Eip != 0) {
+		sp = (uintptr*)r->Esp;
+		*--sp = r->Eip;
+		r->Esp = (uintptr)sp;
 	}
+	r->Eip = (uintptr)runtime·sigpanic;
+	return EXCEPTION_CONTINUE_EXECUTION;
+}
+
+// lastcontinuehandler is reached, because runtime cannot handle
+// current exception. lastcontinuehandler will print crash info and exit.
+uint32
+runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp)
+{
+	bool crash;
 
 	if(runtime·panicking)	// traceback already printed
 		runtime·exit(2);
@@ -88,7 +106,7 @@ runtime·sighandler(ExceptionRecord *info, Context *r, G *gp)
 		runtime·crash();
 
 	runtime·exit(2);
-	return -1; // not reached
+	return 0; // not reached
 }
 
 void
diff --git a/src/runtime/os_windows_amd64.c b/src/runtime/os_windows_amd64.c
index 261880d45..b96cf70d1 100644
--- a/src/runtime/os_windows_amd64.c
+++ b/src/runtime/os_windows_amd64.c
@@ -32,45 +32,76 @@ runtime·dumpregs(Context *r)
 	runtime·printf("gs      %X\n", (uint64)r->SegGs);
 }
 
-// Called by sigtramp from Windows VEH handler.
-// Return value signals whether the exception has been handled (-1)
-// or should be made available to other handlers in the chain (0).
-uint32
-runtime·sighandler(ExceptionRecord *info, Context *r, G *gp)
+bool
+runtime·isgoexception(ExceptionRecord *info, Context *r)
 {
-	bool crash;
-	uintptr *sp;
 	extern byte runtime·text[], runtime·etext[];
 
 	// Only handle exception if executing instructions in Go binary
 	// (not Windows library code). 
 	if(r->Rip < (uint64)runtime·text || (uint64)runtime·etext < r->Rip)
-		return 0;
-
-	if(gp != nil && runtime·issigpanic(info->ExceptionCode)) {
-		// Make it look like a call to the signal func.
-		// Have to pass arguments out of band since
-		// augmenting the stack frame would break
-		// the unwinding code.
-		gp->sig = info->ExceptionCode;
-		gp->sigcode0 = info->ExceptionInformation[0];
-		gp->sigcode1 = info->ExceptionInformation[1];
-		gp->sigpc = r->Rip;
-
-		// Only push runtime·sigpanic if r->rip != 0.
-		// If r->rip == 0, probably panicked because of a
-		// call to a nil func.  Not pushing that onto sp will
-		// make the trace look like a call to runtime·sigpanic instead.
-		// (Otherwise the trace will end at runtime·sigpanic and we
-		// won't get to see who faulted.)
-		if(r->Rip != 0) {
-			sp = (uintptr*)r->Rsp;
-			*--sp = r->Rip;
-			r->Rsp = (uintptr)sp;
-		}
-		r->Rip = (uintptr)runtime·sigpanic;
-		return -1;
+		return false;
+
+	if(!runtime·issigpanic(info->ExceptionCode))
+		return false;
+
+	return true;
+}
+
+// Called by sigtramp from Windows VEH handler.
+// Return value signals whether the exception has been handled (EXCEPTION_CONTINUE_EXECUTION)
+// or should be made available to other handlers in the chain (EXCEPTION_CONTINUE_SEARCH).
+uint32
+runtime·exceptionhandler(ExceptionRecord *info, Context *r, G *gp)
+{
+	uintptr *sp;
+
+	if(!runtime·isgoexception(info, r))
+		return EXCEPTION_CONTINUE_SEARCH;
+
+	// Make it look like a call to the signal func.
+	// Have to pass arguments out of band since
+	// augmenting the stack frame would break
+	// the unwinding code.
+	gp->sig = info->ExceptionCode;
+	gp->sigcode0 = info->ExceptionInformation[0];
+	gp->sigcode1 = info->ExceptionInformation[1];
+	gp->sigpc = r->Rip;
+
+	// Only push runtime·sigpanic if r->rip != 0.
+	// If r->rip == 0, probably panicked because of a
+	// call to a nil func.  Not pushing that onto sp will
+	// make the trace look like a call to runtime·sigpanic instead.
+	// (Otherwise the trace will end at runtime·sigpanic and we
+	// won't get to see who faulted.)
+	if(r->Rip != 0) {
+		sp = (uintptr*)r->Rsp;
+		*--sp = r->Rip;
+		r->Rsp = (uintptr)sp;
 	}
+	r->Rip = (uintptr)runtime·sigpanic;
+	return EXCEPTION_CONTINUE_EXECUTION;
+}
+
+// It seems Windows searches ContinueHandler's list even
+// if ExceptionHandler returns EXCEPTION_CONTINUE_EXECUTION.
+// firstcontinuehandler will stop that search,
+// if exceptionhandler did the same earlier.
+uint32
+runtime·firstcontinuehandler(ExceptionRecord *info, Context *r, G *gp)
+{
+	USED(gp);
+	if(!runtime·isgoexception(info, r))
+		return EXCEPTION_CONTINUE_SEARCH;
+	return EXCEPTION_CONTINUE_EXECUTION;
+}
+
+// lastcontinuehandler is reached, because runtime cannot handle
+// current exception. lastcontinuehandler will print crash info and exit.
+uint32
+runtime·lastcontinuehandler(ExceptionRecord *info, Context *r, G *gp)
+{
+	bool crash;
 
 	if(runtime·panicking)	// traceback already printed
 		runtime·exit(2);
@@ -97,7 +128,7 @@ runtime·sighandler(ExceptionRecord *info, Context *r, G *gp)
 		runtime·crash();
 
 	runtime·exit(2);
-	return -1; // not reached
+	return 0; // not reached
 }
 
 void
diff --git a/src/runtime/panic.c b/src/runtime/panic.c
index 1cd0aa865..24eb6dbfe 100644
--- a/src/runtime/panic.c
+++ b/src/runtime/panic.c
@@ -31,8 +31,11 @@ runtime·deferproc_m(void)
 	argp = g->m->scalararg[1];
 	callerpc = g->m->scalararg[2];
 	g->m->ptrarg[0] = nil;
+	g->m->scalararg[1] = 0;
 
 	d = runtime·newdefer(siz);
+	if(d->panic != nil)
+		runtime·throw("deferproc: d->panic != nil after newdefer");
 	d->fn = fn;
 	d->pc = callerpc;
 	d->argp = argp;
@@ -131,6 +134,7 @@ runtime·dopanic_m(void)
 	g->m->ptrarg[0] = nil;
 	pc = g->m->scalararg[0];
 	sp = g->m->scalararg[1];
+	g->m->scalararg[1] = 0;
 	if(gp->sig != 0)
 		runtime·printf("[signal %x code=%p addr=%p pc=%p]\n",
 			gp->sig, gp->sigcode0, gp->sigcode1, gp->sigpc);
diff --git a/src/runtime/panic.go b/src/runtime/panic.go
index 7eb2d6055..685ff5ca0 100644
--- a/src/runtime/panic.go
+++ b/src/runtime/panic.go
@@ -188,6 +188,12 @@ func newdefer(siz int32) *_defer {
 // The defer cannot be used after this call.
 //go:nosplit
 func freedefer(d *_defer) {
+	if d._panic != nil {
+		freedeferpanic()
+	}
+	if d.fn != nil {
+		freedeferfn()
+	}
 	sc := deferclass(uintptr(d.siz))
 	if sc < uintptr(len(p{}.deferpool)) {
 		mp := acquirem()
@@ -199,6 +205,18 @@ func freedefer(d *_defer) {
 	}
 }
 
+// Separate function so that it can split stack.
+// Windows otherwise runs out of stack space.
+func freedeferpanic() {
+	// _panic must be cleared before d is unlinked from gp.
+	gothrow("freedefer with d._panic != nil")
+}
+
+func freedeferfn() {
+	// fn must be cleared before d is unlinked from gp.
+	gothrow("freedefer with d.fn != nil")
+}
+
 // Run a deferred function if there is one.
 // The compiler inserts a call to this at the end of any
 // function which calls defer.
@@ -231,6 +249,7 @@ func deferreturn(arg0 uintptr) {
 	mp := acquirem()
 	memmove(unsafe.Pointer(argp), deferArgs(d), uintptr(d.siz))
 	fn := d.fn
+	d.fn = nil
 	gp._defer = d.link
 	freedefer(d)
 	releasem(mp)
@@ -258,7 +277,9 @@ func Goexit() {
 		if d.started {
 			if d._panic != nil {
 				d._panic.aborted = true
+				d._panic = nil
 			}
+			d.fn = nil
 			gp._defer = d.link
 			freedefer(d)
 			continue
@@ -268,6 +289,8 @@ func Goexit() {
 		if gp._defer != d {
 			gothrow("bad defer entry in Goexit")
 		}
+		d._panic = nil
+		d.fn = nil
 		gp._defer = d.link
 		freedefer(d)
 		// Note: we ignore recovers here because Goexit isn't a panic
@@ -343,6 +366,8 @@ func gopanic(e interface{}) {
 			if d._panic != nil {
 				d._panic.aborted = true
 			}
+			d._panic = nil
+			d.fn = nil
 			gp._defer = d.link
 			freedefer(d)
 			continue
@@ -366,6 +391,8 @@ func gopanic(e interface{}) {
 		if gp._defer != d {
 			gothrow("bad defer entry in panic")
 		}
+		d._panic = nil
+		d.fn = nil
 		gp._defer = d.link
 
 		// trigger shrinkage to test stack copy.  See stack_test.go:TestStackPanic
diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h
index 609ae9406..bea773799 100644
--- a/src/runtime/runtime.h
+++ b/src/runtime/runtime.h
@@ -372,7 +372,7 @@ struct	M
 	uintptr scalararg[4];	// scalar argument/return for mcall
 	void*   ptrarg[4];	// pointer argument/return for mcall
 #ifdef GOOS_windows
-	void*	thread;		// thread handle
+	uintptr	thread;		// thread handle
 	// these are here because they are too large to be on the stack
 	// of low-level NOSPLIT functions.
 	LibCall	libcall;
@@ -865,6 +865,7 @@ void	runtime·stackinit(void);
 Stack	runtime·stackalloc(uint32);
 void	runtime·stackfree(Stack);
 void	runtime·shrinkstack(G*);
+void	runtime·shrinkfinish(void);
 MCache*	runtime·allocmcache(void);
 void	runtime·freemcache(MCache*);
 void	runtime·mallocinit(void);
@@ -898,7 +899,9 @@ void	runtime·atomicstore(uint32 volatile*, uint32);
 void	runtime·atomicstore64(uint64 volatile*, uint64);
 uint64	runtime·atomicload64(uint64 volatile*);
 void*	runtime·atomicloadp(void* volatile*);
+uintptr	runtime·atomicloaduintptr(uintptr volatile*);
 void	runtime·atomicstorep(void* volatile*, void*);
+void	runtime·atomicstoreuintptr(uintptr volatile*, uintptr);
 void	runtime·atomicor8(byte volatile*, byte);
 
 void	runtime·setg(G*);
diff --git a/src/runtime/stack.c b/src/runtime/stack.c
index 8562b9407..e402691f4 100644
--- a/src/runtime/stack.c
+++ b/src/runtime/stack.c
@@ -36,6 +36,8 @@ MSpan runtime·stackpool[NumStackOrders];
 Mutex runtime·stackpoolmu;
 // TODO: one lock per order?
 
+static Stack stackfreequeue;
+
 void
 runtime·stackinit(void)
 {
@@ -656,7 +658,24 @@ copystack(G *gp, uintptr newsize)
 		while(p < ep)
 			*p++ = 0xfc;
 	}
-	runtime·stackfree(old);
+	if(newsize > old.hi-old.lo) {
+		// growing, free stack immediately
+		runtime·stackfree(old);
+	} else {
+		// shrinking, queue up free operation.  We can't actually free the stack
+		// just yet because we might run into the following situation:
+		// 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
+		// 2) The stack that pointer points to is shrunk
+		// 3) The old stack is freed
+		// 4) The containing span is marked free
+		// 5) GC attempts to mark the SudoG.elem pointer.  The marking fails because
+		//    the pointer looks like a pointer into a free span.
+		// By not freeing, we prevent step #4 until GC is done.
+		runtime·lock(&runtime·stackpoolmu);
+		*(Stack*)old.lo = stackfreequeue;
+		stackfreequeue = old;
+		runtime·unlock(&runtime·stackpoolmu);
+	}
 }
 
 // round x up to a power of 2.
@@ -706,6 +725,7 @@ runtime·newstack(void)
 	g->m->morebuf.pc = (uintptr)nil;
 	g->m->morebuf.lr = (uintptr)nil;
 	g->m->morebuf.sp = (uintptr)nil;
+	g->m->morebuf.g = (G*)nil;
 
 	runtime·casgstatus(gp, Grunning, Gwaiting);
 	gp->waitreason = runtime·gostringnocopy((byte*)"stack growth");
@@ -841,6 +861,23 @@ runtime·shrinkstack(G *gp)
 	copystack(gp, newsize);
 }
 
+// Do any delayed stack freeing that was queued up during GC.
+void
+runtime·shrinkfinish(void)
+{
+	Stack s, t;
+
+	runtime·lock(&runtime·stackpoolmu);
+	s = stackfreequeue;
+	stackfreequeue = (Stack){0,0};
+	runtime·unlock(&runtime·stackpoolmu);
+	while(s.lo != 0) {
+		t = *(Stack*)s.lo;
+		runtime·stackfree(s);
+		s = t;
+	}
+}
+
 static void badc(void);
 
 #pragma textflag NOSPLIT
diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go
index c6a9cf9f5..6561094ff 100644
--- a/src/runtime/stubs.go
+++ b/src/runtime/stubs.go
@@ -252,3 +252,33 @@ func return0()
 
 // thunk to call time.now.
 func timenow() (sec int64, nsec int32)
+
+// in asm_*.s
+// not called directly; definitions here supply type information for traceback.
+func call16(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call32(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call64(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call128(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call256(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call512(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call1024(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call2048(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call4096(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call8192(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call16384(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call32768(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call65536(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call131072(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call262144(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call524288(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call1048576(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call2097152(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call4194304(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call8388608(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call16777216(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call33554432(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call67108864(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call134217728(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call268435456(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call536870912(fn, arg unsafe.Pointer, n, retoffset uint32)
+func call1073741824(fn, arg unsafe.Pointer, n, retoffset uint32)
diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s
index 1bf4d062a..932fe9dd2 100644
--- a/src/runtime/sys_windows_386.s
+++ b/src/runtime/sys_windows_386.s
@@ -73,6 +73,7 @@ TEXT runtime·setlasterror(SB),NOSPLIT,$0
 // Called by Windows as a Vectored Exception Handler (VEH).
 // First argument is pointer to struct containing
 // exception record and context pointers.
+// Handler function is stored in AX.
 // Return 0 for 'not handled', -1 for handled.
 TEXT runtime·sigtramp(SB),NOSPLIT,$0-0
 	MOVL	ptrs+0(FP), CX
@@ -84,6 +85,8 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0
 	MOVL	SI, 20(SP)
 	MOVL	DI, 24(SP)
 
+	MOVL	AX, SI	// save handler address
+
 	// find g
 	get_tls(DX)
 	CMPL	DX, $0
@@ -123,11 +126,10 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0
 sigtramp_g0:
 	MOVL	0(CX), BX // ExceptionRecord*
 	MOVL	4(CX), CX // Context*
-	// call sighandler(ExceptionRecord*, Context*, G*)
 	MOVL	BX, 0(SP)
 	MOVL	CX, 4(SP)
 	MOVL	DX, 8(SP)
-	CALL	runtime·sighandler(SB)
+	CALL	SI	// call handler
 	// AX is set to report result back to Windows
 	MOVL	12(SP), AX
 
@@ -149,6 +151,18 @@ done:
 	// RET 4 (return and pop 4 bytes parameters)
 	BYTE $0xC2; WORD $4
 	RET // unreached; make assembler happy
+ 
+TEXT runtime·exceptiontramp(SB),NOSPLIT,$0
+	MOVL	$runtime·exceptionhandler(SB), AX
+	JMP	runtime·sigtramp(SB)
+
+TEXT runtime·firstcontinuetramp(SB),NOSPLIT,$0-0
+	// is never called
+	INT	$3
+
+TEXT runtime·lastcontinuetramp(SB),NOSPLIT,$0-0
+	MOVL	$runtime·lastcontinuehandler(SB), AX
+	JMP	runtime·sigtramp(SB)
 
 TEXT runtime·ctrlhandler(SB),NOSPLIT,$0
 	PUSHL	$runtime·ctrlhandler1(SB)
diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s
index 05750398e..e6190ce68 100644
--- a/src/runtime/sys_windows_amd64.s
+++ b/src/runtime/sys_windows_amd64.s
@@ -99,6 +99,7 @@ TEXT runtime·setlasterror(SB),NOSPLIT,$0
 // Called by Windows as a Vectored Exception Handler (VEH).
 // First argument is pointer to struct containing
 // exception record and context pointers.
+// Handler function is stored in AX.
 // Return 0 for 'not handled', -1 for handled.
 TEXT runtime·sigtramp(SB),NOSPLIT,$0-0
 	// CX: PEXCEPTION_POINTERS ExceptionInfo
@@ -116,6 +117,8 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0
 	MOVQ	R14, 32(SP)
 	MOVQ	R15, 88(SP)
 
+	MOVQ	AX, R15	// save handler address
+
 	// find g
 	get_tls(DX)
 	CMPQ	DX, $0
@@ -157,11 +160,10 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0-0
 sigtramp_g0:
 	MOVQ	0(CX), BX // ExceptionRecord*
 	MOVQ	8(CX), CX // Context*
-	// call sighandler(ExceptionRecord*, Context*, G*)
 	MOVQ	BX, 0(SP)
 	MOVQ	CX, 8(SP)
 	MOVQ	DX, 16(SP)
-	CALL	runtime·sighandler(SB)
+	CALL	R15	// call handler
 	// AX is set to report result back to Windows
 	MOVL	24(SP), AX
 
@@ -187,6 +189,18 @@ done:
 
 	RET
 
+TEXT runtime·exceptiontramp(SB),NOSPLIT,$0
+	MOVQ	$runtime·exceptionhandler(SB), AX
+	JMP	runtime·sigtramp(SB)
+
+TEXT runtime·firstcontinuetramp(SB),NOSPLIT,$0-0
+	MOVQ	$runtime·firstcontinuehandler(SB), AX
+	JMP	runtime·sigtramp(SB)
+
+TEXT runtime·lastcontinuetramp(SB),NOSPLIT,$0-0
+	MOVQ	$runtime·lastcontinuehandler(SB), AX
+	JMP	runtime·sigtramp(SB)
+
 TEXT runtime·ctrlhandler(SB),NOSPLIT,$8
 	MOVQ	CX, 16(SP)		// spill
 	MOVQ	$runtime·ctrlhandler1(SB), CX
diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go
index 9ed016ccc..ce8a9ec1b 100644
--- a/src/runtime/syscall_windows_test.go
+++ b/src/runtime/syscall_windows_test.go
@@ -494,3 +494,42 @@ func TestOutputDebugString(t *testing.T) {
 	p := syscall.StringToUTF16Ptr("testing OutputDebugString")
 	d.Proc("OutputDebugStringW").Call(uintptr(unsafe.Pointer(p)))
 }
+
+func TestRaiseException(t *testing.T) {
+	o := executeTest(t, raiseExceptionSource, nil)
+	if strings.Contains(o, "RaiseException should not return") {
+		t.Fatalf("RaiseException did not crash program: %v", o)
+	}
+	if !strings.Contains(o, "Exception 0xbad") {
+		t.Fatalf("No stack trace: %v", o)
+	}
+}
+
+const raiseExceptionSource = `
+package main
+import "syscall"
+func main() {
+	const EXCEPTION_NONCONTINUABLE = 1
+	mod := syscall.MustLoadDLL("kernel32.dll")
+	proc := mod.MustFindProc("RaiseException")
+	proc.Call(0xbad, EXCEPTION_NONCONTINUABLE, 0, 0)
+	println("RaiseException should not return")
+}
+`
+
+func TestZeroDivisionException(t *testing.T) {
+	o := executeTest(t, zeroDivisionExceptionSource, nil)
+	if !strings.Contains(o, "panic: runtime error: integer divide by zero") {
+		t.Fatalf("No stack trace: %v", o)
+	}
+}
+
+const zeroDivisionExceptionSource = `
+package main
+func main() {
+	x := 1
+	y := 0
+	z := x / y
+	println(z)
+}
+`
diff --git a/src/runtime/type.h b/src/runtime/type.h
index de82e886f..f5b4f9d13 100644
--- a/src/runtime/type.h
+++ b/src/runtime/type.h
@@ -23,7 +23,7 @@ struct Type
 	uint8 kind;
 	void* alg;
 	// gc stores type info required for garbage collector.
-	// If (kind&KindGCProg)==0, then gc directly contains sparse GC bitmap
+	// If (kind&KindGCProg)==0, then gc[0] points at sparse GC bitmap
 	// (no indirection), 4 bits per word.
 	// If (kind&KindGCProg)!=0, then gc[1] points to a compiler-generated
 	// read-only GC program; and gc[0] points to BSS space for sparse GC bitmap.