1 files changed, 282 insertions, 11 deletions

diff --git a/libgo/runtime/malloc.goc b/libgo/runtime/malloc.goc
index eb163469076..1b6828fd480 100644
--- a/libgo/runtime/malloc.goc
+++ b/libgo/runtime/malloc.goc
@@ -17,12 +17,13 @@ package runtime
 #include "go-string.h"
 #include "interface.h"
 #include "go-type.h"
+#include "race.h"
 
 MHeap runtime_mheap;
 
 extern MStats mstats;	// defined in extern.go
 
-extern volatile int32 runtime_MemProfileRate
+extern volatile intgo runtime_MemProfileRate
   __asm__ ("runtime.MemProfileRate");
 
 // Allocate an object of at least size bytes.
@@ -33,7 +34,8 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 {
 	M *m;
 	G *g;
-	int32 sizeclass, rate;
+	int32 sizeclass;
+	intgo rate;
 	MCache *c;
 	uintptr npages;
 	MSpan *s;
@@ -53,6 +55,9 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 	if(size == 0)
 		size = 1;
 
+	if(DebugTypeAtBlockEnd)
+		size += sizeof(uintptr);
+
 	c = m->mcache;
 	c->local_nmalloc++;
 	if(size <= MaxSmallSize) {
@@ -72,7 +77,7 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		npages = size >> PageShift;
 		if((size & PageMask) != 0)
 			npages++;
-		s = runtime_MHeap_Alloc(&runtime_mheap, npages, 0, 1);
+		s = runtime_MHeap_Alloc(&runtime_mheap, npages, 0, 1, zeroed);
 		if(s == nil)
 			runtime_throw("out of memory");
 		size = npages<<PageShift;
@@ -83,9 +88,20 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		// setup for mark sweep
 		runtime_markspan(v, 0, 0, true);
 	}
+
+	if (sizeof(void*) == 4 && c->local_total_alloc >= (1<<30)) {
+		// purge cache stats to prevent overflow
+		runtime_lock(&runtime_mheap);
+		runtime_purgecachedstats(c);
+		runtime_unlock(&runtime_mheap);
+	}
+
 	if(!(flag & FlagNoGC))
 		runtime_markallocated(v, size, (flag&FlagNoPointers) != 0);
 
+	if(DebugTypeAtBlockEnd)
+		*(uintptr*)((uintptr)v+size-sizeof(uintptr)) = 0;
+
 	m->mallocing = 0;
 
 	if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) {
@@ -107,6 +123,11 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 
 	if(dogc && mstats.heap_alloc >= mstats.next_gc)
 		runtime_gc(0);
+
+	if(raceenabled) {
+		runtime_racemalloc(v, size, m->racepc);
+		m->racepc = nil;
+	}
 	return v;
 }
 
@@ -144,6 +165,9 @@ __go_free(void *v)
 	}
 	prof = runtime_blockspecial(v);
 
+	if(raceenabled)
+		runtime_racefree(v);
+
 	// Find size class for v.
 	sizeclass = s->sizeclass;
 	c = m->mcache;
@@ -178,11 +202,21 @@ __go_free(void *v)
 int32
 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
 {
+	M *m;
 	uintptr n, i;
 	byte *p;
 	MSpan *s;
 
-	runtime_m()->mcache->local_nlookup++;
+	m = runtime_m();
+
+	m->mcache->local_nlookup++;
+	if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) {
+		// purge cache stats to prevent overflow
+		runtime_lock(&runtime_mheap);
+		runtime_purgecachedstats(m->mcache);
+		runtime_unlock(&runtime_mheap);
+	}
+
 	s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
 	if(sp)
 		*sp = s;
@@ -210,7 +244,7 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
 		return 0;
 	}
 
-	n = runtime_class_to_size[s->sizeclass];
+	n = s->elemsize;
 	if(base) {
 		i = ((byte*)v - p)/n;
 		*base = p + i*n;
@@ -224,7 +258,7 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
 MCache*
 runtime_allocmcache(void)
 {
-	int32 rate;
+	intgo rate;
 	MCache *c;
 
 	runtime_lock(&runtime_mheap);
@@ -232,6 +266,7 @@ runtime_allocmcache(void)
 	mstats.mcache_inuse = runtime_mheap.cachealloc.inuse;
 	mstats.mcache_sys = runtime_mheap.cachealloc.sys;
 	runtime_unlock(&runtime_mheap);
+	runtime_memclr((byte*)c, sizeof(*c));
 
 	// Set first allocation sample size.
 	rate = runtime_MemProfileRate;
@@ -244,12 +279,19 @@ runtime_allocmcache(void)
 }
 
 void
-runtime_purgecachedstats(M* m)
+runtime_freemcache(MCache *c)
 {
-	MCache *c;
+	runtime_MCache_ReleaseAll(c);
+	runtime_lock(&runtime_mheap);
+	runtime_purgecachedstats(c);
+	runtime_FixAlloc_Free(&runtime_mheap.cachealloc, c);
+	runtime_unlock(&runtime_mheap);
+}
 
+void
+runtime_purgecachedstats(MCache *c)
+{
 	// Protected by either heap or GC lock.
-	c = m->mcache;
 	mstats.heap_alloc += c->local_cachealloc;
 	c->local_cachealloc = 0;
 	mstats.heap_objects += c->local_objects;
@@ -445,6 +487,220 @@ runtime_MHeap_SysAlloc(MHeap *h, uintptr n)
 	return p;
 }
 
+static Lock settype_lock;
+
+void
+runtime_settype_flush(M *m, bool sysalloc)
+{
+	uintptr *buf, *endbuf;
+	uintptr size, ofs, j, t;
+	uintptr ntypes, nbytes2, nbytes3;
+	uintptr *data2;
+	byte *data3;
+	bool sysalloc3;
+	void *v;
+	uintptr typ, p;
+	MSpan *s;
+
+	buf = m->settype_buf;
+	endbuf = buf + m->settype_bufsize;
+
+	runtime_lock(&settype_lock);
+	while(buf < endbuf) {
+		v = (void*)*buf;
+		*buf = 0;
+		buf++;
+		typ = *buf;
+		buf++;
+
+		// (Manually inlined copy of runtime_MHeap_Lookup)
+		p = (uintptr)v>>PageShift;
+		if(sizeof(void*) == 8)
+			p -= (uintptr)runtime_mheap.arena_start >> PageShift;
+		s = runtime_mheap.map[p];
+
+		if(s->sizeclass == 0) {
+			s->types.compression = MTypes_Single;
+			s->types.data = typ;
+			continue;
+		}
+
+		size = s->elemsize;
+		ofs = ((uintptr)v - (s->start<<PageShift)) / size;
+
+		switch(s->types.compression) {
+		case MTypes_Empty:
+			ntypes = (s->npages << PageShift) / size;
+			nbytes3 = 8*sizeof(uintptr) + 1*ntypes;
+
+			if(!sysalloc) {
+				data3 = runtime_mallocgc(nbytes3, FlagNoPointers, 0, 1);
+			} else {
+				data3 = runtime_SysAlloc(nbytes3);
+				if(0) runtime_printf("settype(0->3): SysAlloc(%x) --> %p\n", (uint32)nbytes3, data3);
+			}
+
+			s->types.compression = MTypes_Bytes;
+			s->types.sysalloc = sysalloc;
+			s->types.data = (uintptr)data3;
+
+			((uintptr*)data3)[1] = typ;
+			data3[8*sizeof(uintptr) + ofs] = 1;
+			break;
+
+		case MTypes_Words:
+			((uintptr*)s->types.data)[ofs] = typ;
+			break;
+
+		case MTypes_Bytes:
+			data3 = (byte*)s->types.data;
+			for(j=1; j<8; j++) {
+				if(((uintptr*)data3)[j] == typ) {
+					break;
+				}
+				if(((uintptr*)data3)[j] == 0) {
+					((uintptr*)data3)[j] = typ;
+					break;
+				}
+			}
+			if(j < 8) {
+				data3[8*sizeof(uintptr) + ofs] = j;
+			} else {
+				ntypes = (s->npages << PageShift) / size;
+				nbytes2 = ntypes * sizeof(uintptr);
+
+				if(!sysalloc) {
+					data2 = runtime_mallocgc(nbytes2, FlagNoPointers, 0, 1);
+				} else {
+					data2 = runtime_SysAlloc(nbytes2);
+					if(0) runtime_printf("settype.(3->2): SysAlloc(%x) --> %p\n", (uint32)nbytes2, data2);
+				}
+
+				sysalloc3 = s->types.sysalloc;
+
+				s->types.compression = MTypes_Words;
+				s->types.sysalloc = sysalloc;
+				s->types.data = (uintptr)data2;
+
+				// Move the contents of data3 to data2. Then deallocate data3.
+				for(j=0; j<ntypes; j++) {
+					t = data3[8*sizeof(uintptr) + j];
+					t = ((uintptr*)data3)[t];
+					data2[j] = t;
+				}
+				if(sysalloc3) {
+					nbytes3 = 8*sizeof(uintptr) + 1*ntypes;
+					if(0) runtime_printf("settype.(3->2): SysFree(%p,%x)\n", data3, (uint32)nbytes3);
+					runtime_SysFree(data3, nbytes3);
+				}
+
+				data2[ofs] = typ;
+			}
+			break;
+		}
+	}
+	runtime_unlock(&settype_lock);
+
+	m->settype_bufsize = 0;
+}
+
+// It is forbidden to use this function if it is possible that
+// explicit deallocation via calling runtime_free(v) may happen.
+void
+runtime_settype(void *v, uintptr t)
+{
+	M *m1;
+	uintptr *buf;
+	uintptr i;
+	MSpan *s;
+
+	if(t == 0)
+		runtime_throw("settype: zero type");
+
+	m1 = runtime_m();
+	buf = m1->settype_buf;
+	i = m1->settype_bufsize;
+	buf[i+0] = (uintptr)v;
+	buf[i+1] = t;
+	i += 2;
+	m1->settype_bufsize = i;
+
+	if(i == nelem(m1->settype_buf)) {
+		runtime_settype_flush(m1, false);
+	}
+
+	if(DebugTypeAtBlockEnd) {
+		s = runtime_MHeap_Lookup(&runtime_mheap, v);
+		*(uintptr*)((uintptr)v+s->elemsize-sizeof(uintptr)) = t;
+	}
+}
+
+void
+runtime_settype_sysfree(MSpan *s)
+{
+	uintptr ntypes, nbytes;
+
+	if(!s->types.sysalloc)
+		return;
+
+	nbytes = (uintptr)-1;
+
+	switch (s->types.compression) {
+	case MTypes_Words:
+		ntypes = (s->npages << PageShift) / s->elemsize;
+		nbytes = ntypes * sizeof(uintptr);
+		break;
+	case MTypes_Bytes:
+		ntypes = (s->npages << PageShift) / s->elemsize;
+		nbytes = 8*sizeof(uintptr) + 1*ntypes;
+		break;
+	}
+
+	if(nbytes != (uintptr)-1) {
+		if(0) runtime_printf("settype: SysFree(%p,%x)\n", (void*)s->types.data, (uint32)nbytes);
+		runtime_SysFree((void*)s->types.data, nbytes);
+	}
+}
+
+uintptr
+runtime_gettype(void *v)
+{
+	MSpan *s;
+	uintptr t, ofs;
+	byte *data;
+
+	s = runtime_MHeap_LookupMaybe(&runtime_mheap, v);
+	if(s != nil) {
+		t = 0;
+		switch(s->types.compression) {
+		case MTypes_Empty:
+			break;
+		case MTypes_Single:
+			t = s->types.data;
+			break;
+		case MTypes_Words:
+			ofs = (uintptr)v - (s->start<<PageShift);
+			t = ((uintptr*)s->types.data)[ofs/s->elemsize];
+			break;
+		case MTypes_Bytes:
+			ofs = (uintptr)v - (s->start<<PageShift);
+			data = (byte*)s->types.data;
+			t = data[8*sizeof(uintptr) + ofs/s->elemsize];
+			t = ((uintptr*)data)[t];
+			break;
+		default:
+			runtime_throw("runtime_gettype: invalid compression kind");
+		}
+		if(0) {
+			runtime_lock(&settype_lock);
+			runtime_printf("%p -> %d,%X\n", v, (int32)s->types.compression, (int64)t);
+			runtime_unlock(&settype_lock);
+		}
+		return t;
+	}
+	return 0;
+}
+
 // Runtime stubs.
 
 void*
@@ -453,9 +709,24 @@ runtime_mal(uintptr n)
 	return runtime_mallocgc(n, 0, 1, 1);
 }
 
-func new(typ *Type) (ret *uint8) {
-	uint32 flag = typ->__code&GO_NO_POINTERS ? FlagNoPointers : 0;
+void *
+runtime_new(Type *typ)
+{
+	void *ret;
+	uint32 flag;
+
+	runtime_m()->racepc = runtime_getcallerpc(&typ);
+	flag = typ->__code&GO_NO_POINTERS ? FlagNoPointers : 0;
 	ret = runtime_mallocgc(typ->__size, flag, 1, 1);
+
+	if(UseSpanType && !flag) {
+		if(false) {
+			runtime_printf("new %S: %p\n", *typ->__reflection, ret);
+		}
+		runtime_settype(ret, (uintptr)typ | TypeInfo_SingleObject);
+	}
+
+	return ret;
 }
 
 func GC() {