diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 7e6539fb26..7ac781a0a7 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -91,9 +91,9 @@ var Flags = []cli.Flag{ //vmoduleFlag, //backtraceAtFlag, //debugFlag, - //pprofFlag, - //pprofAddrFlag, - //pprofPortFlag, + pprofFlag, + pprofAddrFlag, + pprofPortFlag, //memprofilerateFlag, //blockprofilerateFlag, //cpuprofileFlag, diff --git a/metrics/exp/exp.go b/metrics/exp/exp.go index e25026d6ef..77d96e4a3f 100644 --- a/metrics/exp/exp.go +++ b/metrics/exp/exp.go @@ -8,6 +8,8 @@ import ( "net/http" "sync" + "github.com/tomochain/tomochain/metrics/prometheus" + "github.com/tomochain/tomochain/metrics" ) @@ -42,6 +44,7 @@ func Exp(r metrics.Registry) { // http.HandleFunc("/debug/vars", e.expHandler) // haven't found an elegant way, so just use a different endpoint http.Handle("/debug/metrics", h) + http.Handle("/debug/metrics/prometheus", prometheus.Handler(r)) } // ExpHandler will return an expvar powered metrics handler. @@ -134,6 +137,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) { exp.getFloat(name + ".mean-rate").Set(t.RateMean()) } +func (exp *exp) publishResettingTimer(name string, metric metrics.ResettingTimer) { + t := metric.Snapshot() + ps := t.Percentiles([]float64{50, 75, 95, 99}) + exp.getInt(name + ".count").Set(int64(len(t.Values()))) + exp.getFloat(name + ".mean").Set(t.Mean()) + exp.getInt(name + ".50-percentile").Set(ps[0]) + exp.getInt(name + ".75-percentile").Set(ps[1]) + exp.getInt(name + ".95-percentile").Set(ps[2]) + exp.getInt(name + ".99-percentile").Set(ps[3]) +} + func (exp *exp) syncToExpvar() { exp.registry.Each(func(name string, i interface{}) { switch i.(type) { @@ -149,6 +163,8 @@ func (exp *exp) syncToExpvar() { exp.publishMeter(name, i.(metrics.Meter)) case metrics.Timer: exp.publishTimer(name, i.(metrics.Timer)) + case metrics.ResettingTimer: + exp.publishResettingTimer(name, i.(metrics.ResettingTimer)) default: panic(fmt.Sprintf("unsupported type for '%s': %T", name, i)) } diff --git a/metrics/prometheus/collector.go b/metrics/prometheus/collector.go new file mode 100644 index 0000000000..1ee5316cca --- /dev/null +++ b/metrics/prometheus/collector.go @@ -0,0 +1,115 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package prometheus + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + "github.com/tomochain/tomochain/metrics" +) + +var ( + typeGaugeTpl = "# TYPE %s gauge\n" + typeCounterTpl = "# TYPE %s counter\n" + typeSummaryTpl = "# TYPE %s summary\n" + keyValueTpl = "%s %v\n\n" + keyQuantileTagValueTpl = "%s {quantile=\"%s\"} %v\n\n" +) + +// collector is a collection of byte buffers that aggregate Prometheus reports +// for different metric types. +type collector struct { + buff *bytes.Buffer +} + +// newCollector createa a new Prometheus metric aggregator. +func newCollector() *collector { + return &collector{ + buff: &bytes.Buffer{}, + } +} + +func (c *collector) addCounter(name string, m metrics.Counter) { + c.writeGaugeCounter(name, m.Count()) +} + +func (c *collector) addGauge(name string, m metrics.Gauge) { + c.writeGaugeCounter(name, m.Value()) +} + +func (c *collector) addGaugeFloat64(name string, m metrics.GaugeFloat64) { + c.writeGaugeCounter(name, m.Value()) +} + +func (c *collector) addHistogram(name string, m metrics.Histogram) { + pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + ps := m.Percentiles(pv) + c.writeSummaryCounter(name, m.Count()) + for i := range pv { + c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) + } +} + +func (c *collector) addMeter(name string, m metrics.Meter) { + c.writeGaugeCounter(name, m.Count()) +} + +func (c *collector) addTimer(name string, m metrics.Timer) { + pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + ps := m.Percentiles(pv) + c.writeSummaryCounter(name, m.Count()) + for i := range pv { + c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) + } +} + +func (c *collector) addResettingTimer(name string, m metrics.ResettingTimer) { + if len(m.Values()) <= 0 { + return + } + ps := m.Percentiles([]float64{50, 95, 99}) + val := m.Values() + c.writeSummaryCounter(name, len(val)) + c.writeSummaryPercentile(name, "0.50", ps[0]) + c.writeSummaryPercentile(name, "0.95", ps[1]) + c.writeSummaryPercentile(name, "0.99", ps[2]) +} + +func (c *collector) writeGaugeCounter(name string, value interface{}) { + name = mutateKey(name) + c.buff.WriteString(fmt.Sprintf(typeGaugeTpl, name)) + c.buff.WriteString(fmt.Sprintf(keyValueTpl, name, value)) +} + +func (c *collector) writeSummaryCounter(name string, value interface{}) { + name = mutateKey(name + "_count") + c.buff.WriteString(fmt.Sprintf(typeCounterTpl, name)) + c.buff.WriteString(fmt.Sprintf(keyValueTpl, name, value)) +} + +func (c *collector) writeSummaryPercentile(name, p string, value interface{}) { + name = mutateKey(name) + c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, name)) + c.buff.WriteString(fmt.Sprintf(keyQuantileTagValueTpl, name, p, value)) +} + +func mutateKey(key string) string { + return strings.Replace(key, "/", "_", -1) +} diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go new file mode 100644 index 0000000000..1a6cb24c22 --- /dev/null +++ b/metrics/prometheus/prometheus.go @@ -0,0 +1,68 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package prometheus exposes go-metrics into a Prometheus format. +package prometheus + +import ( + "fmt" + "net/http" + "sort" + + "github.com/tomochain/tomochain/log" + "github.com/tomochain/tomochain/metrics" +) + +// Handler returns an HTTP handler which dump metrics in Prometheus format. +func Handler(reg metrics.Registry) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Gather and pre-sort the metrics to avoid random listings + var names []string + reg.Each(func(name string, i interface{}) { + names = append(names, name) + }) + sort.Strings(names) + + // Aggregate all the metris into a Prometheus collector + c := newCollector() + + for _, name := range names { + i := reg.Get(name) + + switch m := i.(type) { + case metrics.Counter: + c.addCounter(name, m.Snapshot()) + case metrics.Gauge: + c.addGauge(name, m.Snapshot()) + case metrics.GaugeFloat64: + c.addGaugeFloat64(name, m.Snapshot()) + case metrics.Histogram: + c.addHistogram(name, m.Snapshot()) + case metrics.Meter: + c.addMeter(name, m.Snapshot()) + case metrics.Timer: + c.addTimer(name, m.Snapshot()) + case metrics.ResettingTimer: + c.addResettingTimer(name, m.Snapshot()) + default: + log.Warn("Unknown Prometheus metric type", "type", fmt.Sprintf("%T", i)) + } + } + w.Header().Add("Content-Type", "text/plain") + w.Header().Add("Content-Length", fmt.Sprint(c.buff.Len())) + w.Write(c.buff.Bytes()) + }) +} diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 57bcb31343..e5327d3bd3 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -58,7 +58,11 @@ type NilResettingTimer struct { func (NilResettingTimer) Values() []int64 { return nil } // Snapshot is a no-op. -func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} } +func (NilResettingTimer) Snapshot() ResettingTimer { + return &ResettingTimerSnapshot{ + values: []int64{}, + } +} // Time is a no-op. func (NilResettingTimer) Time(func()) {} @@ -210,7 +214,7 @@ func (t *ResettingTimerSnapshot) calc(percentiles []float64) { // poor man's math.Round(x): // math.Floor(x + 0.5) indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5)) - if pct >= 0 { + if pct >= 0 && indexOfPerc > 0 { indexOfPerc -= 1 // index offset=0 } thresholdBoundary = t.values[indexOfPerc] diff --git a/metrics/resetting_timer_test.go b/metrics/resetting_timer_test.go index 58fd47f352..77c49dc386 100644 --- a/metrics/resetting_timer_test.go +++ b/metrics/resetting_timer_test.go @@ -104,3 +104,113 @@ func TestResettingTimer(t *testing.T) { } } } + +func TestResettingTimerWithFivePercentiles(t *testing.T) { + tests := []struct { + values []int64 + start int + end int + wantP05 int64 + wantP20 int64 + wantP50 int64 + wantP95 int64 + wantP99 int64 + wantMean float64 + wantMin int64 + wantMax int64 + }{ + { + values: []int64{}, + start: 1, + end: 11, + wantP05: 1, wantP20: 2, wantP50: 5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + { + values: []int64{}, + start: 1, + end: 101, + wantP05: 5, wantP20: 20, wantP50: 50, wantP95: 95, wantP99: 99, + wantMin: 1, wantMax: 100, wantMean: 50.5, + }, + { + values: []int64{1}, + start: 0, + end: 0, + wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 1, wantP99: 1, + wantMin: 1, wantMax: 1, wantMean: 1, + }, + { + values: []int64{0}, + start: 0, + end: 0, + wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{}, + start: 0, + end: 0, + wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{1, 10}, + start: 0, + end: 0, + wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + } + for ind, tt := range tests { + timer := NewResettingTimer() + + for i := tt.start; i < tt.end; i++ { + tt.values = append(tt.values, int64(i)) + } + + for _, v := range tt.values { + timer.Update(time.Duration(v)) + } + + snap := timer.Snapshot() + + ps := snap.Percentiles([]float64{5, 20, 50, 95, 99}) + + val := snap.Values() + + if len(val) > 0 { + if tt.wantMin != val[0] { + t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin) + } + + if tt.wantMax != val[len(val)-1] { + t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax) + } + } + + if tt.wantMean != snap.Mean() { + t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + } + + if tt.wantP05 != ps[0] { + t.Fatalf("%d: p05: got %d, want %d", ind, ps[0], tt.wantP05) + } + + if tt.wantP20 != ps[1] { + t.Fatalf("%d: p20: got %d, want %d", ind, ps[1], tt.wantP20) + } + + if tt.wantP50 != ps[2] { + t.Fatalf("%d: p50: got %d, want %d", ind, ps[2], tt.wantP50) + } + + if tt.wantP95 != ps[3] { + t.Fatalf("%d: p95: got %d, want %d", ind, ps[3], tt.wantP95) + } + + if tt.wantP99 != ps[4] { + t.Fatalf("%d: p99: got %d, want %d", ind, ps[4], tt.wantP99) + } + } +} diff --git a/node/api.go b/node/api.go index 23edbe2b3f..b2a5e95967 100644 --- a/node/api.go +++ b/node/api.go @@ -337,7 +337,20 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) { "95": metric.Percentile(0.95), }, } - + case metrics.ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{5, 20, 50, 80, 95}) + root[name] = map[string]interface{}{ + "Measurements": len(t.Values()), + "Mean": time.Duration(t.Mean()).String(), + "Percentiles": map[string]interface{}{ + "5": time.Duration(ps[0]).String(), + "20": time.Duration(ps[1]).String(), + "50": time.Duration(ps[2]).String(), + "80": time.Duration(ps[3]).String(), + "95": time.Duration(ps[4]).String(), + }, + } default: root[name] = "Unknown metric type" } @@ -372,7 +385,20 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) { "95": time.Duration(metric.Percentile(0.95)).String(), }, } - + case metrics.ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{5, 20, 50, 80, 95}) + root[name] = map[string]interface{}{ + "Measurements": len(t.Values()), + "Mean": time.Duration(t.Mean()).String(), + "Percentiles": map[string]interface{}{ + "5": time.Duration(ps[0]).String(), + "20": time.Duration(ps[1]).String(), + "50": time.Duration(ps[2]).String(), + "80": time.Duration(ps[3]).String(), + "95": time.Duration(ps[4]).String(), + }, + } default: root[name] = "Unknown metric type" }