Merge pull request #6 from istreamlabs/sample-rate

Add sample rate functionality to all implementations

Author: danielgtaylor

.travis.yml

@@ -2,6 +2,7 @@ language: go
 go:
 - 1.8
 - 1.9
+- '1.10'
 install:
 - go get -u github.com/golang/dep/cmd/dep
 - dep ensure

CHANGELOG.md

@@ -5,8 +5,28 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- Put unreleased items here.
 
-- Add unreleased items here.
+## [1.3.0] - 2018-03-19
+
+- Add `WithRate(float64)` to the metrics interface and to all clients that implement
+  the interface. All metrics calls support sample rates.
+  - The `LoggerClient`:
+    - Applies the sample rate when printing log messages. If the rate is `0.1` and you call `Incr()` ten times, expect about one message to have been printed out.
+    - Displays the sample rate for counts if it is not `1.0`, e.g: `Count foo:0.2 (2 * 0.1) [tag1 tag2]`. This shows the sampled value, the passed value, and the sample rate.
+    - Gauges, timings, and histograms will show the sample rate, but he value is left unmodified just like the DataDog implementation.
+  - The `RecorderClient`:
+    - Records all sample rates for metrics calls in `MetricCall.Rate`. No calls are excluded from the call list based on the sample rate, and the value recorded is the full value before multiplying by the sample rate.
+    - Adds a `Rate(float64)` query method to filter by sampled metrics.
+    - The following should work:
+
+        ```go
+        recorder := metrics.NewRecorderClient().WithTest(t)
+        recorder.WithRate(0.1).Count("foo", 5)
+        recorder.Expect("foo").Rate(0.1).Value(5)
+        ```
+- Add `Colorized()` method to `LoggerClient`, and automatically detect a TTY and enable color when `nil` is passed to the `NewLoggerClient` constructor.
+- Test with Go 1.10.x.
 
 ## [1.2.0] - 2018-03-01
 

Gopkg.lock

@@ -49,6 +49,15 @@ client.WithTags(map[string]string{
 
 The above code would result in `myprefix.requests.count` with a value of `1` showing up in DataDog if you have [`dogstatsd`](https://docs.datadoghq.com/guides/dogstatsd/) running locally and an environment variable `env` set to `prod`, otherwise it will print metrics to standard out. See the [`Client`](https://godoc.org/github.com/istreamlabs/go-metrics/metrics/#Client) interface for a list of available metrics methods.
 
+Sometimes you wouldn't want to send a metric every single time a piece of code is executed. This is supported by setting a sample rate:
+
+```go
+// Sample rate for high-throughput applications
+client.WithRate(0.01).Incr("requests.count")
+```
+
+Sample rates apply to metrics but not events. Any count-type metric (`Incr`, `Decr`, `Count`, and timing/histogram counts) will get multiplied to the full value, while gauges are sent unmodified. For example, when emitting a 10% sampled timing metric that takes an average of `200ms` to DataDog, you would see `1 call * (1/0.1 sample rate) = 10 calls` added to the histogram count while the average value remains `200ms` in the DataDog UI.
+
 Also provided are useful clients for testing. For example, the following asserts that a metric with the given name, value, and tag was emitted during a test:
 
 ```go

README.md

@@ -18,6 +18,9 @@
 //     "tag": "value"
 //   }).Incr("requests.count")
 //
+//   // Sample rate for high-throughput applications
+//   client.WithRate(0.01).Incr("requests.count")
+//
 // Also provided are useful clients for testing, both for when you want
 // to assert that certain metrics are emitted and a `NullClient` for when
 // you want to ignore them.
@@ -49,6 +52,9 @@ type Client interface {
 	// WithTags returns a new client with the given tags.
 	WithTags(tags map[string]string) Client
 
+	// WithRate returns a new client with the given sample rate.
+	WithRate(rate float64) Client
+
 	// Count/Incr/Decr set a numeric integer value.
 	Count(name string, value int64)
 	Incr(name string)

metrics/client.go

@@ -1,11 +1,26 @@
 package metrics
 
 import (
+	"fmt"
 	"log"
+	"math/rand"
 	"os"
+	"sort"
 	"time"
 
 	"github.com/DataDog/datadog-go/statsd"
+	"github.com/mattn/go-isatty"
+	"github.com/mgutz/ansi"
+)
+
+var (
+	// Colors are from the ANSI 256 color pallette.
+	// https://en.wikipedia.org/wiki/ANSI_escape_code#8-bit
+	cname    = ansi.ColorFunc("208")
+	cvalue   = ansi.ColorFunc("32")
+	crate    = ansi.ColorFunc("106")
+	csampled = ansi.ColorFunc("43")
+	ctag     = ansi.ColorFunc("133")
 )
 
 // InfoLogger provides a method for logging info messages and is implemented
@@ -18,20 +33,46 @@ type InfoLogger interface {
 // locally for testing. Can be used with multiple different logging systems.
 type LoggerClient struct {
 	logger InfoLogger
+	colors bool
+	rate   float64
 	tagMap map[string]string
 }
 
 // NewLoggerClient creates a new logging client. If `logger` is `nil` then it
-// defaults to stdout using the built-in `log` package. It is equivalent to:
+// defaults to stdout using the built-in `log` package. It is equivalent to
+// the following with added auto-detection for colorized output:
 //
 //   metrics.NewLoggerClient(log.New(os.Stdout, "", 0))
+//
+// You can use your own logger and enable colorized output manually via:
+//
+//   metrics.NewLoggerClient(myLog).Colorized()
 func NewLoggerClient(logger InfoLogger) *LoggerClient {
+	colors := false
 	if logger == nil {
 		logger = log.New(os.Stdout, "", 0)
+
+		if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
+			colors = true
+		}
 	}
 
-	return &LoggerClient{
+	client := &LoggerClient{
 		logger: logger,
+		colors: colors,
+		rate:   1.0,
+	}
+
+	return client
+}
+
+// Colorized enables colored terminal output.
+func (c *LoggerClient) Colorized() *LoggerClient {
+	return &LoggerClient{
+		logger: c.logger,
+		rate:   c.rate,
+		colors: true,
+		tagMap: combine(map[string]string{}, c.tagMap),
 	}
 }
 
@@ -40,13 +81,76 @@ func NewLoggerClient(logger InfoLogger) *LoggerClient {
 func (c *LoggerClient) WithTags(tags map[string]string) Client {
 	return &LoggerClient{
 		logger: c.logger,
+		rate:   c.rate,
+		colors: c.colors,
 		tagMap: combine(c.tagMap, tags),
 	}
 }
 
+// WithRate clones this client with a given sample rate. Subsequent calls
+// will be limited to logging metrics at this rate.
+func (c *LoggerClient) WithRate(rate float64) Client {
+	return &LoggerClient{
+		logger: c.logger,
+		rate:   rate,
+		colors: c.colors,
+		tagMap: combine(map[string]string{}, c.tagMap),
+	}
+}
+
+// print out the metric call, taking into account sample rate.
+func (c *LoggerClient) print(t string, name string, value interface{}, sampled interface{}) {
+	r := fmt.Sprintf("%v", c.rate)
+	v := value
+	s := sampled
+
+	if c.colors {
+		name = cname(name)
+		r = crate(r)
+		v = cvalue(fmt.Sprintf("%v", value))
+		s = csampled(fmt.Sprintf("%v", sampled))
+	}
+
+	if c.rate == 1.0 {
+		c.logger.Printf("%s %s:%v %v", t, name, v, c.getTags())
+		return
+	}
+
+	if rand.Float64() < c.rate {
+		if value == sampled {
+			c.logger.Printf("%s %s:%v (%v) %v", t, name, v, r, c.getTags())
+		} else {
+			c.logger.Printf("%s %s:%v (%v * %v) %v", t, name, s, v, r, c.getTags())
+		}
+	}
+}
+
+func (c *LoggerClient) getTags() string {
+	if !c.colors {
+		return fmt.Sprintf("%v", c.tagMap)
+	}
+
+	keys := make([]string, 0, len(c.tagMap))
+	for k := range c.tagMap {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	tags := ""
+	for _, key := range keys {
+		if tags != "" {
+			tags += " "
+		}
+
+		tags += fmt.Sprintf("%s:%s", ctag(key), c.tagMap[key])
+	}
+
+	return "map[" + tags + "]"
+}
+
 // Count adds some value to a metric.
 func (c *LoggerClient) Count(name string, value int64) {
-	c.logger.Printf("Count %s:%d %v", name, value, c.tagMap)
+	c.print("Count", name, value, float64(value)*c.rate)
 }
 
 // Incr adds one to a metric.
@@ -61,7 +165,7 @@ func (c *LoggerClient) Decr(name string) {
 
 // Gauge sets a numeric value.
 func (c *LoggerClient) Gauge(name string, value float64) {
-	c.logger.Printf("Gauge %s:%v %v", name, value, c.tagMap)
+	c.print("Gauge", name, value, value)
 }
 
 // Event tracks an event that may be relevant to other metrics.
@@ -71,10 +175,10 @@ func (c *LoggerClient) Event(e *statsd.Event) {
 
 // Timing tracks a duration.
 func (c *LoggerClient) Timing(name string, value time.Duration) {
-	c.logger.Printf("Timing %s:%s %v", name, value, c.tagMap)
+	c.print("Timing", name, value, value)
 }
 
 // Histogram sets a numeric value while tracking min/max/avg/p95/etc.
 func (c *LoggerClient) Histogram(name string, value float64) {
-	c.logger.Printf("Histogram %s:%v %v", name, value, c.tagMap)
+	c.print("Histogram", name, value, value)
 }

metrics/logger.go

@@ -53,4 +53,18 @@ func TestLoggerClient(t *testing.T) {
 	ExpectEqual(t, "Count one:-1 map[]", recorder.messages[3])
 	ExpectEqual(t, "Gauge memory:1024 map[]", recorder.messages[4])
 	ExpectEqual(t, "Histogram histo:123 map[]", recorder.messages[5])
+
+	// Make sure the call works, but since it is randomly sampled we have no
+	// assertion to make.
+	sampled := client.WithRate(0.8)
+	sampled.Incr("sampled")
+	sampled.Incr("sampled")
+	sampled.Gauge("sampled-gauge", 123)
+
+	// Test colorized output
+	client.(*metrics.LoggerClient).Colorized().WithTags(map[string]string{
+		"tag1": "val1",
+		"tag2": "val2",
+	}).Incr("colored")
+	ExpectEqual(t, "Count \x1b[38;5;208mcolored\x1b[0m:\x1b[38;5;32m1\x1b[0m map[\x1b[38;5;133mtag1\x1b[0m:val1 \x1b[38;5;133mtag2\x1b[0m:val2]", recorder.messages[len(recorder.messages)-1])
 }

metrics/logger_test.go

@@ -22,6 +22,11 @@ func (c *NullClient) WithTags(tags map[string]string) Client {
 	return &NullClient{}
 }
 
+// WithRate clones this client with a given sample rate.
+func (c *NullClient) WithRate(rate float64) Client {
+	return &NullClient{}
+}
+
 // Count adds some value to a metric.
 func (c *NullClient) Count(name string, value int64) {
 }

metrics/null.go

@@ -32,4 +32,6 @@ func TestNullClientMethods(t *testing.T) {
 	client.Timing("timing", time.Duration(123))
 
 	client.Event(&statsd.Event{})
+
+	client.WithRate(1.2).Incr("rated")
 }

metrics/null_test.go

@@ -65,6 +65,9 @@ type Query interface {
 	// TagName filters out any metric or event that does not contain a given
 	// tag with name `name`. The value does not matter.
 	TagName(name string) Query
+
+	// Rate filters out any metric that does not have the given sample rate.
+	Rate(rate float64) Query
 }
 
 // query is an implementation of the `Query` interface.
@@ -266,3 +269,23 @@ func (q *query) TagName(name string) Query {
 
 	return q
 }
+
+// Rate expects a value with the given rate to exist.
+func (q *query) Rate(rate float64) Query {
+	q.history = fmt.Sprintf("%s rate(%f)", q.history, rate)
+	q.filter(func(call Call) bool {
+		switch t := call.(type) {
+		case *MetricCall:
+			return t.Rate == rate
+		case *EventCall:
+			return false
+		}
+		return false
+	})
+
+	if q.checkMin && len(q.calls) < q.minCalls {
+		q.fatalf("Expected rate '%f'", rate)
+	}
+
+	return q
+}