diff options
Diffstat (limited to 'libgo/runtime')
-rw-r--r-- | libgo/runtime/go-go.c | 29 | ||||
-rw-r--r-- | libgo/runtime/go-gomaxprocs.c | 14 | ||||
-rw-r--r-- | libgo/runtime/go-rand.c | 18 | ||||
-rw-r--r-- | libgo/runtime/go-reflect-chan.c | 18 | ||||
-rw-r--r-- | libgo/runtime/go-reflect-map.c | 37 | ||||
-rw-r--r-- | libgo/runtime/go-select.c | 4 | ||||
-rw-r--r-- | libgo/runtime/go-semacquire.c | 4 | ||||
-rw-r--r-- | libgo/runtime/goc2c.c | 106 | ||||
-rw-r--r-- | libgo/runtime/malloc.goc | 71 | ||||
-rw-r--r-- | libgo/runtime/malloc.h | 20 | ||||
-rw-r--r-- | libgo/runtime/mcache.c | 4 | ||||
-rw-r--r-- | libgo/runtime/mgc0.c | 42 | ||||
-rw-r--r-- | libgo/runtime/mheap.c | 10 | ||||
-rw-r--r-- | libgo/runtime/mprof.goc | 2 | ||||
-rw-r--r-- | libgo/runtime/runtime.h | 8 |
15 files changed, 254 insertions, 133 deletions
diff --git a/libgo/runtime/go-go.c b/libgo/runtime/go-go.c index 3d8e9e62908..139162056b8 100644 --- a/libgo/runtime/go-go.c +++ b/libgo/runtime/go-go.c @@ -73,7 +73,7 @@ static sigset_t __go_thread_wait_sigset; /* Remove the current thread from the list of threads. */ static void -remove_current_thread (void) +remove_current_thread (void *dummy __attribute__ ((unused))) { struct __go_thread_id *list_entry; MCache *mcache; @@ -92,7 +92,7 @@ remove_current_thread (void) if (list_entry->next != NULL) list_entry->next->prev = list_entry->prev; - /* This will look runtime_mheap as needed. */ + /* This will lock runtime_mheap as needed. */ runtime_MCache_ReleaseAll (mcache); /* This should never deadlock--there shouldn't be any code that @@ -139,6 +139,8 @@ start_go_thread (void *thread_arg) m = newm; + pthread_cleanup_push (remove_current_thread, NULL); + list_entry = newm->list_entry; pfn = list_entry->pfn; @@ -166,7 +168,7 @@ start_go_thread (void *thread_arg) (*pfn) (arg); - remove_current_thread (); + pthread_cleanup_pop (1); return NULL; } @@ -178,11 +180,14 @@ void Goexit (void) asm ("libgo_runtime.runtime.Goexit"); void Goexit (void) { - remove_current_thread (); pthread_exit (NULL); abort (); } +/* Count of threads created. */ + +static volatile int mcount; + /* Implement the go statement. */ void @@ -224,6 +229,9 @@ __go_go (void (*pfn) (void*), void *arg) newm->list_entry = list_entry; + newm->id = __sync_fetch_and_add (&mcount, 1); + newm->fastrand = 0x49f6428aUL + newm->id; + newm->mcache = runtime_allocmcache (); /* Add the thread to the list of all threads, marked as tentative @@ -536,12 +544,17 @@ __go_cachestats (void) for (p = __go_all_thread_ids; p != NULL; p = p->next) { MCache *c; + int i; + runtime_purgecachedstats(p->m); c = p->m->mcache; - mstats.heap_alloc += c->local_alloc; - c->local_alloc = 0; - mstats.heap_objects += c->local_objects; - c->local_objects = 0; + for (i = 0; i < NumSizeClasses; ++i) + { + mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc; + c->local_by_size[i].nmalloc = 0; + mstats.by_size[i].nfree += c->local_by_size[i].nfree; + c->local_by_size[i].nfree = 0; + } } } diff --git a/libgo/runtime/go-gomaxprocs.c b/libgo/runtime/go-gomaxprocs.c index 04dc448b854..65146c50120 100644 --- a/libgo/runtime/go-gomaxprocs.c +++ b/libgo/runtime/go-gomaxprocs.c @@ -7,9 +7,17 @@ /* This is the runtime.GOMAXPROCS function. This currently does nothing, since each goroutine runs in a separate thread anyhow. */ -void GOMAXPROCS (int) asm ("libgo_runtime.runtime.GOMAXPROCS"); +extern int GOMAXPROCS (int) asm ("libgo_runtime.runtime.GOMAXPROCS"); -void -GOMAXPROCS (int n __attribute__ ((unused))) +static int set = 1; + +int +GOMAXPROCS (int n) { + int ret; + + ret = set; + if (n > 0) + set = n; + return ret; } diff --git a/libgo/runtime/go-rand.c b/libgo/runtime/go-rand.c new file mode 100644 index 00000000000..9632efc09cd --- /dev/null +++ b/libgo/runtime/go-rand.c @@ -0,0 +1,18 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" + +uint32 +runtime_fastrand1(void) +{ + uint32 x; + + x = m->fastrand; + x += x; + if(x & 0x80000000L) + x ^= 0x88888eefUL; + m->fastrand = x; + return x; +} diff --git a/libgo/runtime/go-reflect-chan.c b/libgo/runtime/go-reflect-chan.c index 61e360212f0..e8b4366dc57 100644 --- a/libgo/runtime/go-reflect-chan.c +++ b/libgo/runtime/go-reflect-chan.c @@ -33,16 +33,21 @@ makechan (const struct __go_type_descriptor *typ, uint32_t size) return (uintptr_t) ret; } -extern _Bool chansend (uintptr_t, uintptr_t, _Bool) +extern _Bool chansend (struct __go_channel_type *, uintptr_t, uintptr_t, _Bool) asm ("libgo_reflect.reflect.chansend"); _Bool -chansend (uintptr_t ch, uintptr_t val_i, _Bool nb) +chansend (struct __go_channel_type *ct, uintptr_t ch, uintptr_t val_i, + _Bool nb) { struct __go_channel *channel = (struct __go_channel *) ch; uintptr_t element_size; void *pv; + __go_assert (ct->__common.__code == GO_CHAN); + __go_assert (__go_type_descriptors_equal (ct->__element_type, + channel->element_type)); + if (channel == NULL) __go_panic_msg ("send to nil channel"); @@ -94,17 +99,22 @@ struct chanrecv_ret _Bool received; }; -extern struct chanrecv_ret chanrecv (uintptr_t, _Bool) +extern struct chanrecv_ret chanrecv (struct __go_channel_type *, uintptr_t, + _Bool) asm ("libgo_reflect.reflect.chanrecv"); struct chanrecv_ret -chanrecv (uintptr_t ch, _Bool nb) +chanrecv (struct __go_channel_type *ct, uintptr_t ch, _Bool nb) { struct __go_channel *channel = (struct __go_channel *) ch; void *pv; uintptr_t element_size; struct chanrecv_ret ret; + __go_assert (ct->__common.__code == GO_CHAN); + __go_assert (__go_type_descriptors_equal (ct->__element_type, + channel->element_type)); + element_size = channel->element_type->__size; if (__go_is_pointer_type (channel->element_type)) diff --git a/libgo/runtime/go-reflect-map.c b/libgo/runtime/go-reflect-map.c index 5559f6eadaf..eb0590602e0 100644 --- a/libgo/runtime/go-reflect-map.c +++ b/libgo/runtime/go-reflect-map.c @@ -8,6 +8,7 @@ #include <stdint.h> #include "go-alloc.h" +#include "go-assert.h" #include "go-panic.h" #include "go-type.h" #include "map.h" @@ -21,11 +22,12 @@ struct mapaccess_ret _Bool pres; }; -extern struct mapaccess_ret mapaccess (uintptr_t, uintptr_t) +extern struct mapaccess_ret mapaccess (struct __go_map_type *, uintptr_t, + uintptr_t) asm ("libgo_reflect.reflect.mapaccess"); struct mapaccess_ret -mapaccess (uintptr_t m, uintptr_t key_i) +mapaccess (struct __go_map_type *mt, uintptr_t m, uintptr_t key_i) { struct __go_map *map = (struct __go_map *) m; void *key; @@ -36,18 +38,20 @@ mapaccess (uintptr_t m, uintptr_t key_i) void *val; void *pv; - if (map == NULL) - __go_panic_msg ("lookup in nil map"); + __go_assert (mt->__common.__code == GO_MAP); - key_descriptor = map->__descriptor->__map_descriptor->__key_type; + key_descriptor = mt->__key_type; if (__go_is_pointer_type (key_descriptor)) key = &key_i; else key = (void *) key_i; - p = __go_map_index (map, key, 0); + if (map == NULL) + p = NULL; + else + p = __go_map_index (map, key, 0); - val_descriptor = map->__descriptor->__map_descriptor->__val_type; + val_descriptor = mt->__val_type; if (__go_is_pointer_type (val_descriptor)) { val = NULL; @@ -71,20 +75,24 @@ mapaccess (uintptr_t m, uintptr_t key_i) return ret; } -extern void mapassign (uintptr_t, uintptr_t, uintptr_t, _Bool) +extern void mapassign (struct __go_map_type *, uintptr_t, uintptr_t, + uintptr_t, _Bool) asm ("libgo_reflect.reflect.mapassign"); void -mapassign (uintptr_t m, uintptr_t key_i, uintptr_t val_i, _Bool pres) +mapassign (struct __go_map_type *mt, uintptr_t m, uintptr_t key_i, + uintptr_t val_i, _Bool pres) { struct __go_map *map = (struct __go_map *) m; const struct __go_type_descriptor *key_descriptor; void *key; + __go_assert (mt->__common.__code == GO_MAP); + if (map == NULL) - __go_panic_msg ("lookup in nil map"); + __go_panic_msg ("assignment to entry in nil map"); - key_descriptor = map->__descriptor->__map_descriptor->__key_type; + key_descriptor = mt->__key_type; if (__go_is_pointer_type (key_descriptor)) key = &key_i; else @@ -100,7 +108,7 @@ mapassign (uintptr_t m, uintptr_t key_i, uintptr_t val_i, _Bool pres) p = __go_map_index (map, key, 1); - val_descriptor = map->__descriptor->__map_descriptor->__val_type; + val_descriptor = mt->__val_type; if (__go_is_pointer_type (val_descriptor)) pv = &val_i; else @@ -122,14 +130,15 @@ maplen (uintptr_t m) return (int32_t) map->__element_count; } -extern unsigned char *mapiterinit (uintptr_t) +extern unsigned char *mapiterinit (struct __go_map_type *, uintptr_t) asm ("libgo_reflect.reflect.mapiterinit"); unsigned char * -mapiterinit (uintptr_t m) +mapiterinit (struct __go_map_type *mt, uintptr_t m) { struct __go_hash_iter *it; + __go_assert (mt->__common.__code == GO_MAP); it = __go_alloc (sizeof (struct __go_hash_iter)); __go_mapiterinit ((struct __go_map *) m, it); return (unsigned char *) it; diff --git a/libgo/runtime/go-select.c b/libgo/runtime/go-select.c index 5ea521d423d..e425aae24c7 100644 --- a/libgo/runtime/go-select.c +++ b/libgo/runtime/go-select.c @@ -533,7 +533,9 @@ mark_all_channels_waiting (struct select_channel* channels, uintptr_t count, uintptr_t j; /* A channel may be selected for both read and write. */ - if (channels[channels[i].dup_index].is_send != is_send) + if (channels[channels[i].dup_index].is_send == is_send) + continue; + else { for (j = channels[i].dup_index + 1; j < i; ++j) { diff --git a/libgo/runtime/go-semacquire.c b/libgo/runtime/go-semacquire.c index 24c6a7388f6..40fe2af7864 100644 --- a/libgo/runtime/go-semacquire.c +++ b/libgo/runtime/go-semacquire.c @@ -44,7 +44,7 @@ acquire (uint32 *addr) and it remains nonnegative. */ void -semacquire (uint32 *addr) +runtime_semacquire (uint32 *addr) { while (1) { @@ -86,7 +86,7 @@ semacquire (uint32 *addr) process. */ void -semrelease (uint32 *addr) +runtime_semrelease (uint32 *addr) { int32_t val; diff --git a/libgo/runtime/goc2c.c b/libgo/runtime/goc2c.c index bf7483309bf..32fbceba1f5 100644 --- a/libgo/runtime/goc2c.c +++ b/libgo/runtime/goc2c.c @@ -2,22 +2,27 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -/* Translate a .goc file into a .c file. A .goc file is a combination - of a limited form of Go with C. */ +/* + * Translate a .goc file into a .c file. A .goc file is a combination + * of a limited form of Go with C. + */ /* - package PACKAGENAME - {# line} - func NAME([NAME TYPE { , NAME TYPE }]) [(NAME TYPE { , NAME TYPE })] \{ - C code with proper brace nesting - \} + package PACKAGENAME + {# line} + func NAME([NAME TYPE { , NAME TYPE }]) [(NAME TYPE { , NAME TYPE })] \{ + C code with proper brace nesting + \} */ -/* We generate C code which implements the function such that it can - be called from Go and executes the C code. */ +/* + * We generate C code which implements the function such that it can + * be called from Go and executes the C code. + */ #include <assert.h> #include <ctype.h> +#include <stdarg.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -87,20 +92,34 @@ static struct { /* Fixed structure alignment (non-gcc only) */ int structround = 4; +char *argv0; + +static void +sysfatal(char *fmt, ...) +{ + char buf[256]; + va_list arg; + + va_start(arg, fmt); + vsnprintf(buf, sizeof buf, fmt, arg); + va_end(arg); + + fprintf(stderr, "%s: %s\n", argv0 ? argv0 : "<prog>", buf); + exit(1); +} + /* Unexpected EOF. */ static void bad_eof(void) { - fprintf(stderr, "%s:%u: unexpected EOF\n", file, lineno); - exit(1); + sysfatal("%s:%ud: unexpected EOF\n", file, lineno); } /* Out of memory. */ static void bad_mem(void) { - fprintf(stderr, "%s:%u: out of memory\n", file, lineno); - exit(1); + sysfatal("%s:%ud: out of memory\n", file, lineno); } /* Allocate memory without fail. */ @@ -199,8 +218,10 @@ getchar_skipping_comments(void) } } -/* Read and return a token. Tokens are delimited by whitespace or by - [(),{}]. The latter are all returned as single characters. */ +/* + * Read and return a token. Tokens are delimited by whitespace or by + * [(),{}]. The latter are all returned as single characters. + */ static char * read_token(void) { @@ -262,11 +283,11 @@ read_package(void) char *token; token = read_token_no_eof(); + if (token == NULL) + sysfatal("%s:%ud: no token\n", file, lineno); if (strcmp(token, "package") != 0) { - fprintf(stderr, - "%s:%u: expected \"package\", got \"%s\"\n", + sysfatal("%s:%ud: expected \"package\", got \"%s\"\n", file, lineno, token); - exit(1); } return read_token_no_eof(); } @@ -293,8 +314,10 @@ read_preprocessor_lines(void) } } -/* Read a type in Go syntax and return a type in C syntax. We only - permit basic types and pointers. */ +/* + * Read a type in Go syntax and return a type in C syntax. We only + * permit basic types and pointers. + */ static char * read_type(void) { @@ -337,14 +360,15 @@ type_size(char *p) if(strcmp(type_table[i].name, p) == 0) return type_table[i].size; if(!gcc) { - fprintf(stderr, "%s:%u: unknown type %s\n", file, lineno, p); - exit(1); + sysfatal("%s:%ud: unknown type %s\n", file, lineno, p); } return 1; } -/* Read a list of parameters. Each parameter is a name and a type. - The list ends with a ')'. We have already read the '('. */ +/* + * Read a list of parameters. Each parameter is a name and a type. + * The list ends with a ')'. We have already read the '('. + */ static struct params * read_params(int *poffset) { @@ -380,17 +404,18 @@ read_params(int *poffset) } } if (strcmp(token, ")") != 0) { - fprintf(stderr, "%s:%u: expected '('\n", + sysfatal("%s:%ud: expected '('\n", file, lineno); - exit(1); } if (poffset != NULL) *poffset = offset; return ret; } -/* Read a function header. This reads up to and including the initial - '{' character. Returns 1 if it read a header, 0 at EOF. */ +/* + * Read a function header. This reads up to and including the initial + * '{' character. Returns 1 if it read a header, 0 at EOF. + */ static int read_func_header(char **name, struct params **params, int *paramwid, struct params **rets) { @@ -421,9 +446,8 @@ read_func_header(char **name, struct params **params, int *paramwid, struct para token = read_token(); if (token == NULL || strcmp(token, "(") != 0) { - fprintf(stderr, "%s:%u: expected \"(\"\n", + sysfatal("%s:%ud: expected \"(\"\n", file, lineno); - exit(1); } *params = read_params(paramwid); @@ -435,9 +459,8 @@ read_func_header(char **name, struct params **params, int *paramwid, struct para token = read_token(); } if (token == NULL || strcmp(token, "{") != 0) { - fprintf(stderr, "%s:%u: expected \"{\"\n", + sysfatal("%s:%ud: expected \"{\"\n", file, lineno); - exit(1); } return 1; } @@ -589,8 +612,10 @@ write_func_trailer(char *package, char *name, write_6g_func_trailer(rets); } -/* Read and write the body of the function, ending in an unnested } - (which is read but not written). */ +/* + * Read and write the body of the function, ending in an unnested } + * (which is read but not written). + */ static void copy_body(void) { @@ -677,15 +702,15 @@ process_file(void) static void usage(void) { - fprintf(stderr, "Usage: goc2c [--6g | --gc] [--go-prefix PREFIX] [file]\n"); - exit(1); + sysfatal("Usage: goc2c [--6g | --gc] [--go-prefix PREFIX] [file]\n"); } -int +void main(int argc, char **argv) { char *goarch; + argv0 = argv[0]; while(argc > 1 && argv[1][0] == '-') { if(strcmp(argv[1], "-") == 0) break; @@ -706,7 +731,7 @@ main(int argc, char **argv) if(argc <= 1 || strcmp(argv[1], "-") == 0) { file = "<stdin>"; process_file(); - return 0; + exit(0); } if(argc > 2) @@ -714,8 +739,7 @@ main(int argc, char **argv) file = argv[1]; if(freopen(file, "r", stdin) == 0) { - fprintf(stderr, "open %s: %s\n", file, strerror(errno)); - exit(1); + sysfatal("open %s: %r\n", file); } if(!gcc) { @@ -731,5 +755,5 @@ main(int argc, char **argv) } process_file(); - return 0; + exit(0); } diff --git a/libgo/runtime/malloc.goc b/libgo/runtime/malloc.goc index b46995ae0b6..2ea69ee795b 100644 --- a/libgo/runtime/malloc.goc +++ b/libgo/runtime/malloc.goc @@ -26,21 +26,6 @@ extern MStats mstats; // defined in extern.go extern volatile int32 runtime_MemProfileRate __asm__ ("libgo_runtime.runtime.MemProfileRate"); -// Same algorithm from chan.c, but a different -// instance of the static uint32 x. -// Not protected by a lock - let the threads use -// the same random number if they like. -static uint32 -fastrand1(void) -{ - static uint32 x = 0x49f6428aUL; - - x += x; - if(x & 0x80000000L) - x ^= 0x88888eefUL; - return x; -} - // Allocate an object of at least size bytes. // Small objects are allocated from the per-thread cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. @@ -58,18 +43,18 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) if(size == 0) size = 1; - mstats.nmalloc++; + c = m->mcache; + c->local_nmalloc++; if(size <= MaxSmallSize) { // Allocate from mcache free lists. sizeclass = runtime_SizeToClass(size); size = runtime_class_to_size[sizeclass]; - c = m->mcache; v = runtime_MCache_Alloc(c, sizeclass, size, zeroed); if(v == nil) runtime_throw("out of memory"); - mstats.alloc += size; - mstats.total_alloc += size; - mstats.by_size[sizeclass].nmalloc++; + c->local_alloc += size; + c->local_total_alloc += size; + c->local_by_size[sizeclass].nmalloc++; } else { // TODO(rsc): Report tracebacks for very large allocations. @@ -81,8 +66,8 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) if(s == nil) runtime_throw("out of memory"); size = npages<<PageShift; - mstats.alloc += size; - mstats.total_alloc += size; + c->local_alloc += size; + c->local_total_alloc += size; v = (void*)(s->start << PageShift); // setup for mark sweep @@ -113,7 +98,7 @@ runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) // pick next profile time if(rate > 0x3fffffff) // make 2*rate not overflow rate = 0x3fffffff; - m->mcache->next_sample = fastrand1() % (2*rate); + m->mcache->next_sample = runtime_fastrand1() % (2*rate); profile: runtime_setblockspecial(v); runtime_MProf_Malloc(v, size); @@ -158,6 +143,7 @@ __go_free(void *v) // Find size class for v. sizeclass = s->sizeclass; + c = m->mcache; if(sizeclass == 0) { // Large object. size = s->npages<<PageShift; @@ -169,18 +155,17 @@ __go_free(void *v) runtime_MHeap_Free(&runtime_mheap, s, 1); } else { // Small object. - c = m->mcache; size = runtime_class_to_size[sizeclass]; - if(size > (int32)sizeof(uintptr)) + if(size > sizeof(uintptr)) ((uintptr*)v)[1] = 1; // mark as "needs to be zeroed" // Must mark v freed before calling MCache_Free: // it might coalesce v and other blocks into a bigger span // and change the bitmap further. runtime_markfreed(v, size); - mstats.by_size[sizeclass].nfree++; + c->local_by_size[sizeclass].nfree++; runtime_MCache_Free(c, v, sizeclass, size); } - mstats.alloc -= size; + c->local_alloc -= size; if(prof) runtime_MProf_Free(v, size); @@ -197,7 +182,7 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp) byte *p; MSpan *s; - mstats.nlookup++; + m->mcache->local_nlookup++; s = runtime_MHeap_LookupMaybe(&runtime_mheap, v); if(sp) *sp = s; @@ -226,9 +211,10 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp) } n = runtime_class_to_size[s->sizeclass]; - i = ((byte*)v - p)/n; - if(base) + if(base) { + i = ((byte*)v - p)/n; *base = p + i*n; + } if(size) *size = n; @@ -260,7 +246,30 @@ runtime_allocmcache(void) return c; } -extern int32 runtime_sizeof_C_MStats +void +runtime_purgecachedstats(M* m) +{ + MCache *c; + + // Protected by either heap or GC lock. + c = m->mcache; + mstats.heap_alloc += c->local_cachealloc; + c->local_cachealloc = 0; + mstats.heap_objects += c->local_objects; + c->local_objects = 0; + mstats.nmalloc += c->local_nmalloc; + c->local_nmalloc = 0; + mstats.nfree += c->local_nfree; + c->local_nfree = 0; + mstats.nlookup += c->local_nlookup; + c->local_nlookup = 0; + mstats.alloc += c->local_alloc; + c->local_alloc= 0; + mstats.total_alloc += c->local_total_alloc; + c->local_total_alloc= 0; +} + +extern uintptr runtime_sizeof_C_MStats __asm__ ("libgo_runtime.runtime.Sizeof_C_MStats"); #define MaxArena32 (2U<<30) diff --git a/libgo/runtime/malloc.h b/libgo/runtime/malloc.h index 8131e964e49..3e813bbde8c 100644 --- a/libgo/runtime/malloc.h +++ b/libgo/runtime/malloc.h @@ -80,7 +80,6 @@ // This C code was written with an eye toward translating to Go // in the future. Methods have the form Type_Method(Type *t, ...). -typedef struct FixAlloc FixAlloc; typedef struct MCentral MCentral; typedef struct MHeap MHeap; typedef struct MSpan MSpan; @@ -186,10 +185,10 @@ void runtime_FixAlloc_Free(FixAlloc *f, void *p); // Shared with Go: if you edit this structure, also edit extern.go. struct MStats { - // General statistics. No locking; approximate. + // General statistics. uint64 alloc; // bytes allocated and still in use uint64 total_alloc; // bytes allocated (even if freed) - uint64 sys; // bytes obtained from system (should be sum of xxx_sys below) + uint64 sys; // bytes obtained from system (should be sum of xxx_sys below, no locking, approximate) uint64 nlookup; // number of pointer lookups uint64 nmalloc; // number of mallocs uint64 nfree; // number of frees @@ -222,7 +221,6 @@ struct MStats bool debuggc; // Statistics about allocation size classes. - // No locking; approximate. struct { uint32 size; uint64 nmalloc; @@ -268,9 +266,20 @@ struct MCache { MCacheList list[NumSizeClasses]; uint64 size; + int64 local_cachealloc; // bytes allocated (or freed) from cache since last lock of heap + int64 local_objects; // objects allocated (or freed) from cache since last lock of heap int64 local_alloc; // bytes allocated (or freed) since last lock of heap - int64 local_objects; // objects allocated (or freed) since last lock of heap + int64 local_total_alloc; // bytes allocated (even if freed) since last lock of heap + int64 local_nmalloc; // number of mallocs since last lock of heap + int64 local_nfree; // number of frees since last lock of heap + int64 local_nlookup; // number of pointer lookups since last lock of heap int32 next_sample; // trigger heap sample after allocating this many bytes + // Statistics about allocation size classes since last lock of heap + struct { + int64 nmalloc; + int64 nfree; + } local_by_size[NumSizeClasses]; + }; void* runtime_MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed); @@ -379,6 +388,7 @@ void runtime_markspan(void *v, uintptr size, uintptr n, bool leftover); void runtime_unmarkspan(void *v, uintptr size); bool runtime_blockspecial(void*); void runtime_setblockspecial(void*); +void runtime_purgecachedstats(M*); enum { diff --git a/libgo/runtime/mcache.c b/libgo/runtime/mcache.c index 65d849c16ea..191b0d1c3f2 100644 --- a/libgo/runtime/mcache.c +++ b/libgo/runtime/mcache.c @@ -48,7 +48,7 @@ runtime_MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed) v->next = nil; } } - c->local_alloc += size; + c->local_cachealloc += size; c->local_objects++; return v; } @@ -90,7 +90,7 @@ runtime_MCache_Free(MCache *c, void *v, int32 sizeclass, uintptr size) l->list = p; l->nlist++; c->size += size; - c->local_alloc -= size; + c->local_cachealloc -= size; c->local_objects--; if(l->nlist >= MaxMCacheListLen) { diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index 0f28f5f6bd8..900ebde687c 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -443,6 +443,7 @@ sweep(void) // Mark freed; restore block boundary bit. *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); + c = m->mcache; if(s->sizeclass == 0) { // Free large span. runtime_unmarkspan(p, 1<<PageShift); @@ -450,14 +451,13 @@ sweep(void) runtime_MHeap_Free(&runtime_mheap, s, 1); } else { // Free small object. - c = m->mcache; if(size > sizeof(uintptr)) ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed" - mstats.by_size[s->sizeclass].nfree++; + c->local_by_size[s->sizeclass].nfree++; runtime_MCache_Free(c, p, s->sizeclass, size); } - mstats.alloc -= size; - mstats.nfree++; + c->local_alloc -= size; + c->local_nfree++; } } } @@ -537,6 +537,7 @@ runtime_gc(int32 force __attribute__ ((unused))) sweep(); t2 = runtime_nanotime(); __go_stealcache(); + __go_cachestats(); mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; m->gcing = 0; @@ -584,6 +585,25 @@ runtime_gc(int32 force __attribute__ ((unused))) runtime_gc(1); } +void runtime_UpdateMemStats(void) + __asm__("libgo_runtime.runtime.UpdateMemStats"); + +void +runtime_UpdateMemStats(void) +{ + // Have to acquire gcsema to stop the world, + // because stoptheworld can only be used by + // one goroutine at a time, and there might be + // a pending garbage collection already calling it. + pthread_mutex_lock(&gcsema); + m->gcing = 1; + runtime_stoptheworld(); + __go_cachestats(); + m->gcing = 0; + pthread_mutex_unlock(&gcsema); + runtime_starttheworld(); +} + static void runfinq(void* dummy) { @@ -617,7 +637,7 @@ runfinq(void* dummy) } } -#define runtime_gomaxprocs 2 +#define runtime_singleproc 0 // mark the block at v of size n as allocated. // If noptr is true, mark it as having no pointers. @@ -641,11 +661,11 @@ runtime_markallocated(void *v, uintptr n, bool noptr) bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); if(noptr) bits |= bitNoPointers<<shift; - if(runtime_gomaxprocs == 1) { + if(runtime_singleproc) { *b = bits; break; } else { - // gomaxprocs > 1: use atomic op + // more than one goroutine is potentially running: use atomic op if(runtime_casp((void**)b, (void*)obits, (void*)bits)) break; } @@ -671,11 +691,11 @@ runtime_markfreed(void *v, uintptr n) for(;;) { obits = *b; bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); - if(runtime_gomaxprocs == 1) { + if(runtime_singleproc) { *b = bits; break; } else { - // gomaxprocs > 1: use atomic op + // more than one goroutine is potentially running: use atomic op if(runtime_casp((void**)b, (void*)obits, (void*)bits)) break; } @@ -782,11 +802,11 @@ runtime_setblockspecial(void *v) for(;;) { obits = *b; bits = obits | (bitSpecial<<shift); - if(runtime_gomaxprocs == 1) { + if(runtime_singleproc) { *b = bits; break; } else { - // gomaxprocs > 1: use atomic op + // more than one goroutine is potentially running: use atomic op if(runtime_casp((void**)b, (void*)obits, (void*)bits)) break; } diff --git a/libgo/runtime/mheap.c b/libgo/runtime/mheap.c index cc6b3aff423..cacac7d6037 100644 --- a/libgo/runtime/mheap.c +++ b/libgo/runtime/mheap.c @@ -58,10 +58,7 @@ runtime_MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct) MSpan *s; runtime_lock(h); - mstats.heap_alloc += m->mcache->local_alloc; - m->mcache->local_alloc = 0; - mstats.heap_objects += m->mcache->local_objects; - m->mcache->local_objects = 0; + runtime_purgecachedstats(m); s = MHeap_AllocLocked(h, npage, sizeclass); if(s != nil) { mstats.heap_inuse += npage<<PageShift; @@ -259,10 +256,7 @@ void runtime_MHeap_Free(MHeap *h, MSpan *s, int32 acct) { runtime_lock(h); - mstats.heap_alloc += m->mcache->local_alloc; - m->mcache->local_alloc = 0; - mstats.heap_objects += m->mcache->local_objects; - m->mcache->local_objects = 0; + runtime_purgecachedstats(m); mstats.heap_inuse -= s->npages<<PageShift; if(acct) { mstats.heap_alloc -= s->npages<<PageShift; diff --git a/libgo/runtime/mprof.goc b/libgo/runtime/mprof.goc index 2e147edda02..d87be429d85 100644 --- a/libgo/runtime/mprof.goc +++ b/libgo/runtime/mprof.goc @@ -115,7 +115,7 @@ static uintptr addrmem; // hashMultiplier is the bottom 32 bits of int((sqrt(5)-1)/2 * (1<<32)). // This is a good multiplier as suggested in CLR, Knuth. The hash // value is taken to be the top AddrHashBits bits of the bottom 32 bits -// of the muliplied value. +// of the multiplied value. enum { HashMultiplier = 2654435769U }; diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index bf07c1219ba..ddc99eb6fb7 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -50,6 +50,7 @@ typedef uint8 bool; typedef uint8 byte; typedef struct M M; typedef struct MCache MCache; +typedef struct FixAlloc FixAlloc; typedef struct Lock Lock; /* We use mutexes for locks. 6g uses futexes directly, and perhaps @@ -95,6 +96,7 @@ enum struct M { + int32 id; int32 mallocing; int32 gcing; int32 locks; @@ -103,6 +105,7 @@ struct M int32 holds_finlock; int32 gcing_for_finlock; int32 profilehz; + uint32 fastrand; MCache *mcache; /* For the list of all threads. */ @@ -152,8 +155,8 @@ void runtime_lock(Lock*); void runtime_unlock(Lock*); void runtime_destroylock(Lock*); -void semacquire (uint32 *) asm ("libgo_runtime.runtime.Semacquire"); -void semrelease (uint32 *) asm ("libgo_runtime.runtime.Semrelease"); +void runtime_semacquire (uint32 *) asm ("libgo_runtime.runtime.Semacquire"); +void runtime_semrelease (uint32 *) asm ("libgo_runtime.runtime.Semrelease"); /* * sleep and wakeup on one-time events. @@ -192,6 +195,7 @@ void runtime_sigprof(uint8 *pc, uint8 *sp, uint8 *lr); void runtime_cpuprofinit(void); void runtime_resetcpuprofiler(int32); void runtime_setcpuprofilerate(void(*)(uintptr*, int32), int32); +uint32 runtime_fastrand1(void); struct __go_func_type; void reflect_call(const struct __go_func_type *, const void *, _Bool, _Bool, |