diff options
Diffstat (limited to 'libgo/go/runtime/pprof/internal/protopprof')
4 files changed, 463 insertions, 0 deletions
diff --git a/libgo/go/runtime/pprof/internal/protopprof/protomemprofile.go b/libgo/go/runtime/pprof/internal/protopprof/protomemprofile.go new file mode 100644 index 00000000000..c2ab5b57025 --- /dev/null +++ b/libgo/go/runtime/pprof/internal/protopprof/protomemprofile.go @@ -0,0 +1,83 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package protopprof + +import ( + "internal/pprof/profile" + "math" + "runtime" + "time" +) + +// EncodeMemProfile converts MemProfileRecords to a Profile. +func EncodeMemProfile(mr []runtime.MemProfileRecord, rate int64, t time.Time) *profile.Profile { + p := &profile.Profile{ + Period: rate, + PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"}, + SampleType: []*profile.ValueType{ + {Type: "alloc_objects", Unit: "count"}, + {Type: "alloc_space", Unit: "bytes"}, + {Type: "inuse_objects", Unit: "count"}, + {Type: "inuse_space", Unit: "bytes"}, + }, + TimeNanos: int64(t.UnixNano()), + } + + locs := make(map[uintptr]*profile.Location) + for _, r := range mr { + stack := r.Stack() + sloc := make([]*profile.Location, len(stack)) + for i, addr := range stack { + loc := locs[addr] + if loc == nil { + loc = &profile.Location{ + ID: uint64(len(p.Location) + 1), + Address: uint64(addr), + } + locs[addr] = loc + p.Location = append(p.Location, loc) + } + sloc[i] = loc + } + + ao, ab := scaleHeapSample(r.AllocObjects, r.AllocBytes, rate) + uo, ub := scaleHeapSample(r.InUseObjects(), r.InUseBytes(), rate) + + p.Sample = append(p.Sample, &profile.Sample{ + Value: []int64{ao, ab, uo, ub}, + Location: sloc, + }) + } + if runtime.GOOS == "linux" { + addMappings(p) + } + return p +} + +// scaleHeapSample adjusts the data from a heap Sample to +// account for its probability of appearing in the collected +// data. heap profiles are a sampling of the memory allocations +// requests in a program. We estimate the unsampled value by dividing +// each collected sample by its probability of appearing in the +// profile. heap profiles rely on a poisson process to determine +// which samples to collect, based on the desired average collection +// rate R. The probability of a sample of size S to appear in that +// profile is 1-exp(-S/R). +func scaleHeapSample(count, size, rate int64) (int64, int64) { + if count == 0 || size == 0 { + return 0, 0 + } + + if rate <= 1 { + // if rate==1 all samples were collected so no adjustment is needed. + // if rate<1 treat as unknown and skip scaling. + return count, size + } + + avgSize := float64(size) / float64(count) + scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) + + return int64(float64(count) * scale), int64(float64(size) * scale) +} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protomemprofile_test.go b/libgo/go/runtime/pprof/internal/protopprof/protomemprofile_test.go new file mode 100644 index 00000000000..a10fe772ccf --- /dev/null +++ b/libgo/go/runtime/pprof/internal/protopprof/protomemprofile_test.go @@ -0,0 +1,104 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package protopprof + +import ( + "bytes" + "internal/pprof/profile" + "io/ioutil" + "reflect" + "runtime" + "testing" + "time" +) + +// TestSampledHeapAllocProfile tests encoding of a memory profile from +// runtime.MemProfileRecord data. +func TestSampledHeapAllocProfile(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Test requires a system with /proc/self/maps") + } + + // Figure out two addresses from /proc/self/maps. + mmap, err := ioutil.ReadFile("/proc/self/maps") + if err != nil { + t.Fatal("Cannot read /proc/self/maps") + } + rd := bytes.NewReader(mmap) + mprof := &profile.Profile{} + if err = mprof.ParseMemoryMap(rd); err != nil { + t.Fatalf("Cannot parse /proc/self/maps") + } + if len(mprof.Mapping) < 2 { + // It is possible for a binary to only have 1 executable + // region of memory. + t.Skipf("need 2 or more mappings, got %v", len(mprof.Mapping)) + } + address1 := mprof.Mapping[0].Start + address2 := mprof.Mapping[1].Start + + var buf bytes.Buffer + + rec, rate := testMemRecords(address1, address2) + p := EncodeMemProfile(rec, rate, time.Now()) + if err := p.Write(&buf); err != nil { + t.Fatalf("Failed to write profile: %v", err) + } + + p, err = profile.Parse(&buf) + if err != nil { + t.Fatalf("Could not parse Profile profile: %v", err) + } + + // Expected PeriodType, SampleType and Sample. + expectedPeriodType := &profile.ValueType{Type: "space", Unit: "bytes"} + expectedSampleType := []*profile.ValueType{ + {Type: "alloc_objects", Unit: "count"}, + {Type: "alloc_space", Unit: "bytes"}, + {Type: "inuse_objects", Unit: "count"}, + {Type: "inuse_space", Unit: "bytes"}, + } + // Expected samples, with values unsampled according to the profiling rate. + expectedSample := []*profile.Sample{ + {Value: []int64{2050, 2099200, 1537, 1574400}, Location: []*profile.Location{ + {ID: 1, Mapping: mprof.Mapping[0], Address: address1}, + {ID: 2, Mapping: mprof.Mapping[1], Address: address2}, + }}, + {Value: []int64{1, 829411, 1, 829411}, Location: []*profile.Location{ + {ID: 3, Mapping: mprof.Mapping[1], Address: address2 + 1}, + {ID: 4, Mapping: mprof.Mapping[1], Address: address2 + 2}, + }}, + {Value: []int64{1, 829411, 0, 0}, Location: []*profile.Location{ + {ID: 5, Mapping: mprof.Mapping[0], Address: address1 + 1}, + {ID: 6, Mapping: mprof.Mapping[0], Address: address1 + 2}, + {ID: 7, Mapping: mprof.Mapping[1], Address: address2 + 3}, + }}, + } + + if p.Period != 512*1024 { + t.Fatalf("Sampling periods do not match") + } + if !reflect.DeepEqual(p.PeriodType, expectedPeriodType) { + t.Fatalf("Period types do not match") + } + if !reflect.DeepEqual(p.SampleType, expectedSampleType) { + t.Fatalf("Sample types do not match") + } + if !reflect.DeepEqual(p.Sample, expectedSample) { + t.Fatalf("Samples do not match: Expected: %v, Got:%v", getSampleAsString(expectedSample), + getSampleAsString(p.Sample)) + } +} + +func testMemRecords(a1, a2 uint64) ([]runtime.MemProfileRecord, int64) { + addr1, addr2 := uintptr(a1), uintptr(a2) + rate := int64(512 * 1024) + rec := []runtime.MemProfileRecord{ + {4096, 1024, 4, 1, [32]uintptr{addr1, addr2}}, + {512 * 1024, 0, 1, 0, [32]uintptr{addr2 + 1, addr2 + 2}}, + {512 * 1024, 512 * 1024, 1, 1, [32]uintptr{addr1 + 1, addr1 + 2, addr2 + 3}}, + } + return rec, rate +} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protopprof.go b/libgo/go/runtime/pprof/internal/protopprof/protopprof.go new file mode 100644 index 00000000000..5d269c4f652 --- /dev/null +++ b/libgo/go/runtime/pprof/internal/protopprof/protopprof.go @@ -0,0 +1,105 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package protopprof converts the runtime's raw profile logs +// to Profile structs containing a representation of the pprof +// protocol buffer profile format. +package protopprof + +import ( + "fmt" + "os" + "runtime" + "time" + "unsafe" + + "internal/pprof/profile" +) + +// TranslateCPUProfile parses binary CPU profiling stack trace data +// generated by runtime.CPUProfile() into a profile struct. +func TranslateCPUProfile(b []byte, startTime time.Time) (*profile.Profile, error) { + const wordSize = unsafe.Sizeof(uintptr(0)) + const minRawProfile = 5 * wordSize // Need a minimum of 5 words. + if uintptr(len(b)) < minRawProfile { + return nil, fmt.Errorf("truncated profile") + } + n := int(uintptr(len(b)) / wordSize) + data := ((*[1 << 28]uintptr)(unsafe.Pointer(&b[0])))[:n:n] + period := data[3] + data = data[5:] // skip header + + // profile initialization taken from pprof tool + p := &profile.Profile{ + Period: int64(period) * 1000, + PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}, + SampleType: []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + }, + TimeNanos: int64(startTime.UnixNano()), + DurationNanos: time.Since(startTime).Nanoseconds(), + } + // Parse CPU samples from the profile. + locs := make(map[uint64]*profile.Location) + for len(b) > 0 { + if len(data) < 2 || uintptr(len(data)) < 2+data[1] { + return nil, fmt.Errorf("truncated profile") + } + count := data[0] + nstk := data[1] + if uintptr(len(data)) < 2+nstk { + return nil, fmt.Errorf("truncated profile") + } + stk := data[2 : 2+nstk] + data = data[2+nstk:] + + if count == 0 && nstk == 1 && stk[0] == 0 { + // end of data marker + break + } + + sloc := make([]*profile.Location, len(stk)) + for i, addr := range stk { + addr := uint64(addr) + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call + // (except for the leaf, which is not a call). + if i > 0 { + addr-- + } + loc := locs[addr] + if loc == nil { + loc = &profile.Location{ + ID: uint64(len(p.Location) + 1), + Address: addr, + } + locs[addr] = loc + p.Location = append(p.Location, loc) + } + sloc[i] = loc + } + p.Sample = append(p.Sample, &profile.Sample{ + Value: []int64{int64(count), int64(count) * int64(p.Period)}, + Location: sloc, + }) + } + + if runtime.GOOS == "linux" { + if err := addMappings(p); err != nil { + return nil, err + } + } + return p, nil +} + +func addMappings(p *profile.Profile) error { + // Parse memory map from /proc/self/maps + f, err := os.Open("/proc/self/maps") + if err != nil { + return err + } + defer f.Close() + return p.ParseMemoryMap(f) +} diff --git a/libgo/go/runtime/pprof/internal/protopprof/protopprof_test.go b/libgo/go/runtime/pprof/internal/protopprof/protopprof_test.go new file mode 100644 index 00000000000..f1937b5bd06 --- /dev/null +++ b/libgo/go/runtime/pprof/internal/protopprof/protopprof_test.go @@ -0,0 +1,171 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package protopprof + +import ( + "bytes" + "fmt" + "internal/pprof/profile" + "io/ioutil" + "reflect" + "runtime" + "testing" + "time" + "unsafe" +) + +// Helper function to initialize empty cpu profile with sampling period provided. +func createEmptyProfileWithPeriod(t *testing.T, periodMs uint64) bytes.Buffer { + // Mock the sample header produced by cpu profiler. Write a sample + // period of 2000 microseconds, followed by no samples. + buf := new(bytes.Buffer) + // Profile header is as follows: + // The first, third and fifth words are 0. The second word is 3. + // The fourth word is the period. + // EOD marker: + // The sixth word -- count is initialized to 0 above. + // The code below sets the seventh word -- nstk to 1 + // The eighth word -- addr is initialized to 0 above. + words := []int{0, 3, 0, int(periodMs), 0, 0, 1, 0} + n := int(unsafe.Sizeof(0)) * len(words) + data := ((*[1 << 29]byte)(unsafe.Pointer(&words[0])))[:n:n] + if _, err := buf.Write(data); err != nil { + t.Fatalf("createEmptyProfileWithPeriod failed: %v", err) + } + return *buf +} + +// Helper function to initialize cpu profile with two sample values. +func createProfileWithTwoSamples(t *testing.T, periodMs uintptr, count1 uintptr, count2 uintptr, + address1 uintptr, address2 uintptr) bytes.Buffer { + // Mock the sample header produced by cpu profiler. Write a sample + // period of 2000 microseconds, followed by no samples. + buf := new(bytes.Buffer) + words := []uintptr{0, 3, 0, uintptr(periodMs), 0, uintptr(count1), 2, + uintptr(address1), uintptr(address1 + 2), + uintptr(count2), 2, uintptr(address2), uintptr(address2 + 2), + 0, 1, 0} + for _, n := range words { + var err error + switch unsafe.Sizeof(int(0)) { + case 8: + _, err = buf.Write((*[8]byte)(unsafe.Pointer(&n))[:8:8]) + case 4: + _, err = buf.Write((*[4]byte)(unsafe.Pointer(&n))[:4:4]) + } + if err != nil { + t.Fatalf("createProfileWithTwoSamples failed: %v", err) + } + } + return *buf +} + +// Tests TranslateCPUProfile parses correct sampling period in an otherwise empty cpu profile. +func TestTranlateCPUProfileSamplingPeriod(t *testing.T) { + // A test server with mock cpu profile data. + var buf bytes.Buffer + + startTime := time.Now() + b := createEmptyProfileWithPeriod(t, 2000) + p, err := TranslateCPUProfile(b.Bytes(), startTime) + if err != nil { + t.Fatalf("translate failed: %v", err) + } + if err := p.Write(&buf); err != nil { + t.Fatalf("write failed: %v", err) + } + + p, err = profile.Parse(&buf) + if err != nil { + t.Fatalf("Could not parse Profile profile: %v", err) + } + + // Expected PeriodType and SampleType. + expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"} + expectedSampleType := []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + } + if p.Period != 2000*1000 || !reflect.DeepEqual(p.PeriodType, expectedPeriodType) || + !reflect.DeepEqual(p.SampleType, expectedSampleType) || p.Sample != nil { + t.Fatalf("Unexpected Profile fields") + } +} + +func getSampleAsString(sample []*profile.Sample) string { + var str string + for _, x := range sample { + for _, y := range x.Location { + if y.Mapping != nil { + str += fmt.Sprintf("Mapping:%v\n", *y.Mapping) + } + str += fmt.Sprintf("Location:%v\n", y) + } + str += fmt.Sprintf("Sample:%v\n", *x) + } + return str +} + +// Tests TranslateCPUProfile parses a cpu profile with sample values present. +func TestTranslateCPUProfileWithSamples(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("test requires a system with /proc/self/maps") + } + // Figure out two addresses from /proc/self/maps. + mmap, err := ioutil.ReadFile("/proc/self/maps") + if err != nil { + t.Fatal("Cannot read /proc/self/maps") + } + rd := bytes.NewReader(mmap) + mprof := &profile.Profile{} + if err = mprof.ParseMemoryMap(rd); err != nil { + t.Fatalf("Cannot parse /proc/self/maps") + } + if len(mprof.Mapping) < 2 { + // It is possible for a binary to only have 1 executable + // region of memory. + t.Skipf("need 2 or more mappings, got %v", len(mprof.Mapping)) + } + address1 := mprof.Mapping[0].Start + address2 := mprof.Mapping[1].Start + // A test server with mock cpu profile data. + + startTime := time.Now() + b := createProfileWithTwoSamples(t, 2000, 20, 40, uintptr(address1), uintptr(address2)) + p, err := TranslateCPUProfile(b.Bytes(), startTime) + + if err != nil { + t.Fatalf("Could not parse Profile profile: %v", err) + } + // Expected PeriodType, SampleType and Sample. + expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"} + expectedSampleType := []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + } + expectedSample := []*profile.Sample{ + {Value: []int64{20, 20 * 2000 * 1000}, Location: []*profile.Location{ + {ID: 1, Mapping: mprof.Mapping[0], Address: address1}, + {ID: 2, Mapping: mprof.Mapping[0], Address: address1 + 1}, + }}, + {Value: []int64{40, 40 * 2000 * 1000}, Location: []*profile.Location{ + {ID: 3, Mapping: mprof.Mapping[1], Address: address2}, + {ID: 4, Mapping: mprof.Mapping[1], Address: address2 + 1}, + }}, + } + if p.Period != 2000*1000 { + t.Fatalf("Sampling periods do not match") + } + if !reflect.DeepEqual(p.PeriodType, expectedPeriodType) { + t.Fatalf("Period types do not match") + } + if !reflect.DeepEqual(p.SampleType, expectedSampleType) { + t.Fatalf("Sample types do not match") + } + if !reflect.DeepEqual(p.Sample, expectedSample) { + t.Fatalf("Samples do not match: Expected: %v, Got:%v", getSampleAsString(expectedSample), + getSampleAsString(p.Sample)) + } +} |