diff --git a/common/stats/aggregator.go b/common/stats/aggregator.go index fa5e4bc..6a71987 100644 --- a/common/stats/aggregator.go +++ b/common/stats/aggregator.go @@ -15,6 +15,7 @@ package stats import ( + "strings" "sync" "time" ) @@ -50,9 +51,8 @@ func newCollectedStatUnescaped(name string) *collectedStat { // previously. Useful for reporters that have low throughput ie stathat. type Aggregator struct { // Holds all of our stats based on stat.Name - sl sync.RWMutex - stats map[string]*statHolder - + sl sync.RWMutex + stats map[string]*statHolder reporters []reporter } @@ -167,22 +167,30 @@ func (a *Aggregator) report(st []*collectedStat) { } } -func (r *Aggregator) Inc(component string, stat string, value int64, rate float32) { - r.add(component, stat, counterKind, value) +func (r *Aggregator) Inc(value int64, stat ...string) { + component := stat[0] + newstat := strings.Join(stat[1:], ".") + r.add(component, newstat, counterKind, value) } -func (r *Aggregator) Gauge(component string, stat string, value int64, rate float32) { - r.add(component, stat, gaugeKind, value) +func (r *Aggregator) Gauge(value int64, stat ...string) { + component := stat[0] + newstat := strings.Join(stat[1:], ".") + r.add(component, newstat, gaugeKind, value) } -func (r *Aggregator) Measure(component string, stat string, value int64, rate float32) { - r.add(component, stat, valueKind, value) +func (r *Aggregator) Measure(value int64, stat ...string) { + component := stat[0] + newstat := strings.Join(stat[1:], ".") + r.add(component, newstat, valueKind, value) } -func (r *Aggregator) Time(component string, stat string, value time.Duration, rate float32) { - r.add(component, stat, durationKind, value) +func (r *Aggregator) Time(value time.Duration, stat ...string) { + component := stat[0] + newstat := strings.Join(stat[1:], ".") + r.add(component, newstat, durationKind, value) } -func (r *Aggregator) NewTimer(component string, stat string, rate float32) *Timer { - return newTimer(r, component, stat, rate) +func (r *Aggregator) NewTimer(stat ...string) *Timer { + return newTimer(r, stat...) } diff --git a/common/stats/mem.go b/common/stats/mem.go index 69ad09b..789cb31 100644 --- a/common/stats/mem.go +++ b/common/stats/mem.go @@ -29,12 +29,12 @@ func StartReportingMemoryAndGC(reporter Statter, d time.Duration) { prefix := "runtime" - reporter.Measure(prefix, "allocated", int64(ms.Alloc), 1.0) - reporter.Measure(prefix, "allocated.heap", int64(ms.HeapAlloc), 1.0) - reporter.Time(prefix, "gc.pause", time.Duration(ms.PauseNs[(ms.NumGC+255)%256]), 1.0) + reporter.Measure(int64(ms.Alloc), prefix, "allocated") + reporter.Measure(int64(ms.HeapAlloc), prefix, "allocated.heap") + reporter.Time(time.Duration(ms.PauseNs[(ms.NumGC+255)%256]), prefix, "gc.pause") // GC CPU percentage. - reporter.Measure(prefix, "gc.cpufraction", int64(ms.GCCPUFraction*100), 1.0) + reporter.Measure(int64(ms.GCCPUFraction*100), prefix, "gc.cpufraction") } } } diff --git a/common/stats/metricsstatter.go b/common/stats/metricsstatter.go new file mode 100644 index 0000000..cd51702 --- /dev/null +++ b/common/stats/metricsstatter.go @@ -0,0 +1,103 @@ +package stats + +import ( + "os" + "path" + "strings" + "time" + + "github.com/armon/go-metrics" + "gopkg.in/inconshreveable/log15.v2" +) + +func NewMetricsStatter(config Config) Statter { + inm := metrics.NewInmemSink(10*time.Second, time.Minute) + metrics.DefaultInmemSignal(inm) + + var fanout metrics.FanoutSink + + if config.Statsd != nil && config.Statsd.StatsdUdpTarget != "" { + sink, err := metrics.NewStatsdSink(config.Statsd.StatsdUdpTarget) + if err != nil { + log15.Error("Couldn't create statsd reporter", "err", err) + } + fanout = append(fanout, sink) + } + prefix := "iron" + if config.Statsd != nil && len(config.Statsd.Prefix) > 0 { + prefix = config.Statsd.Prefix + ".iron" + } + metricsConfig := metrics.DefaultConfig(prefix) + metricsConfig.EnableRuntimeMetrics = false // Use another metrics instance for runtime stats reporting + metricsConfig.EnableHostname = false + + m := new(MetricsStatter) + m.hostname = whoami() + m.servicePrefix = path.Base(os.Args[0]) + + if len(fanout) > 0 { + if !config.NoHostname { + metricsConfig.ServiceName = strings.Join([]string{prefix, m.servicePrefix, m.hostname}, ".") + } + fanout = append(fanout, inm) + metrics.NewGlobal(metricsConfig, fanout) + } else { + metrics.NewGlobal(metricsConfig, inm) + } + + if config.GCStats >= 0 { + if config.GCStats == 0 { + config.GCStats = 1 + } + metricsConfig.EnableRuntimeMetrics = true + metricsConfig.ProfileInterval = time.Duration(config.Interval * float64(time.Second)) + if len(fanout) > 0 { + metricsConfig.ServiceName = strings.Join([]string{prefix, m.servicePrefix, m.hostname}, ".") + metrics.New(metricsConfig, fanout) + } else { + metrics.New(metricsConfig, inm) + } + } + + log15.Info("Statter configured", "servicePrefix", m.servicePrefix) + + return m +} + +type MetricsStatter struct { + hostname string + servicePrefix string +} + +func (m *MetricsStatter) template(stat ...string) []string { + newstat := []string{m.servicePrefix} + for _, token := range stat { + switch token { + case "@hostname": + newstat = append(newstat, m.hostname) + default: + newstat = append(newstat, token) + } + } + return newstat +} + +func (m *MetricsStatter) Inc(value int64, stat ...string) { + newstat := m.template(stat...) + metrics.IncrCounter(newstat, float32(value)) +} +func (m *MetricsStatter) Gauge(value int64, stat ...string) { + newstat := m.template(stat...) + metrics.SetGauge(newstat, float32(value)) +} +func (m *MetricsStatter) Measure(value int64, stat ...string) { + newstat := m.template(stat...) + metrics.AddSample(newstat, float32(value)) +} +func (m *MetricsStatter) Time(value time.Duration, stat ...string) { + newstat := m.template(stat...) + metrics.AddSample(newstat, float32(value.Nanoseconds()/1e6)) //Report ms to statsd +} +func (m *MetricsStatter) NewTimer(stat ...string) *Timer { + return newTimer(m, m.template(stat...)...) +} diff --git a/common/stats/multistatter.go b/common/stats/multistatter.go new file mode 100644 index 0000000..5795bdf --- /dev/null +++ b/common/stats/multistatter.go @@ -0,0 +1,90 @@ +package stats + +import ( + "time" + + "gopkg.in/inconshreveable/log15.v2" +) + +type MultiStatter struct { + statters []Statter +} + +func NewMultiStatter(config Config) Statter { + s := new(MultiStatter) + + var reporters []reporter + if config.StatHat != nil && config.StatHat.Email != "" { + reporters = append(reporters, config.StatHat) + } + + if config.NewRelic != nil && config.NewRelic.LicenseKey != "" { + // NR wants version? + // can get it out of the namespace? roll it here? + reporters = append(reporters, NewNewRelicReporter("1.0", config.NewRelic.LicenseKey)) + } + + if config.Log15 != nil { + reporters = append(reporters, NewLogReporter()) + } + + if len(reporters) > 0 { + ag := newAggregator(reporters) + s.statters = append(s.statters, ag) + go func() { + for range time.Tick(time.Duration(config.Interval * float64(time.Second))) { + ag.report(nil) + } + }() + } + + if config.Statsd != nil && config.Statsd.StatsdUdpTarget != "" { + std, err := NewStatsd(config.Statsd) + if err == nil { + s.statters = append(s.statters, std) + } else { + log15.Error("Couldn't create statsd reporter", "err", err) + } + } + + if len(reporters) == 0 && config.Statsd == nil && config.History == 0 { + return &NilStatter{} + } + + if config.GCStats >= 0 { + if config.GCStats == 0 { + config.GCStats = 1 + } + go StartReportingMemoryAndGC(s, time.Duration(config.GCStats)*time.Second) + } + + return s +} + +func (s *MultiStatter) Inc(value int64, stat ...string) { + for _, st := range s.statters { + st.Inc(value, stat...) + } +} + +func (s *MultiStatter) Gauge(value int64, stat ...string) { + for _, st := range s.statters { + st.Gauge(value, stat...) + } +} + +func (s *MultiStatter) Measure(value int64, stat ...string) { + for _, st := range s.statters { + st.Measure(value, stat...) + } +} + +func (s *MultiStatter) Time(value time.Duration, stat ...string) { + for _, st := range s.statters { + st.Time(value, stat...) + } +} + +func (s *MultiStatter) NewTimer(stat ...string) *Timer { + return newTimer(s, stat...) +} diff --git a/common/stats/stats.go b/common/stats/stats.go index babb1f1..5a10fbe 100644 --- a/common/stats/stats.go +++ b/common/stats/stats.go @@ -1,27 +1,13 @@ -// Copyright 2016 Iron.io -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package stats import ( + "bytes" "encoding/json" - "errors" + "fmt" + "net" "net/http" "strings" "time" - - "github.com/Sirupsen/logrus" ) type HTTPSubHandler interface { @@ -29,111 +15,45 @@ type HTTPSubHandler interface { } type Config struct { - Interval float64 `json:"interval" envconfig:"STATS_INTERVAL"` // seconds + Interval float64 `json:"interval"` // seconds History int // minutes - Log string `json:"log" envconfig:"STATS_LOG"` - StatHat *StatHatReporterConfig - NewRelic *NewRelicReporterConfig - Statsd *StatsdConfig - GCStats int `json:"gc_stats" envconfig:"GC_STATS"` // seconds -} - -type Statter interface { - Inc(component string, stat string, value int64, rate float32) - Gauge(component string, stat string, value int64, rate float32) - Measure(component string, stat string, value int64, rate float32) - Time(component string, stat string, value time.Duration, rate float32) - NewTimer(component string, stat string, rate float32) *Timer -} - -type MultiStatter struct { - statters []Statter -} - -func (s *MultiStatter) Inc(component string, stat string, value int64, rate float32) { - for _, st := range s.statters { - st.Inc(component, stat, value, rate) - } -} - -func (s *MultiStatter) Gauge(component string, stat string, value int64, rate float32) { - for _, st := range s.statters { - st.Gauge(component, stat, value, rate) - } + Log15 []string `json:"log15"` + StatHat *StatHatReporterConfig + NewRelic *NewRelicReporterConfig + Statsd *StatsdConfig + GCStats int `json:"gc_stats" envconfig:"GC_STATS"` // seconds + NewStatter bool `json:"new_statter" envconfig:"NEW_STATTER"` // Statter using armon/go-metrics + NoHostname bool `json:"no_hostname" envconfig:"NO_HOSTNAME"` //Don't include hostname by default } -func (s *MultiStatter) Measure(component string, stat string, value int64, rate float32) { - for _, st := range s.statters { - st.Measure(component, stat, value, rate) - } +type StatsdConfig struct { + StatsdUdpTarget string `json:"target" mapstructure:"target" envconfig:"STATSD_TARGET"` + Interval int64 `json:"interval" envconfig:"STATSD_INTERVAL"` + Prefix string `json:"prefix" envconfig:"STATSD_PREFIX"` } -func (s *MultiStatter) Time(component string, stat string, value time.Duration, rate float32) { - for _, st := range s.statters { - st.Time(component, stat, value, rate) - } +type Statter interface { + Inc(value int64, stat ...string) + Gauge(value int64, stat ...string) + Measure(value int64, stat ...string) + Time(value time.Duration, stat ...string) + NewTimer(stat ...string) *Timer } -func (s *MultiStatter) NewTimer(component string, stat string, rate float32) *Timer { - return newTimer(s, component, stat, rate) +func AsStatField(input string) string { + return strings.Replace(strings.ToLower(strings.TrimSpace(input)), ".", "_", -1) } -var badDecode error = errors.New("bad stats decode") - func New(config Config) Statter { - s := new(MultiStatter) - if config.Interval == 0.0 { - config.Interval = 10.0 // convenience - } - - var reporters []reporter - if config.StatHat != nil && config.StatHat.Email != "" { - reporters = append(reporters, config.StatHat) + config.Interval = 60.0 // convenience } - if config.NewRelic != nil && config.NewRelic.LicenseKey != "" { - // NR wants version? - // can get it out of the namespace? roll it here? - reporters = append(reporters, NewNewRelicReporter("1.0", config.NewRelic.LicenseKey)) + if !config.NewStatter { + return NewMultiStatter(config) } - - if config.Log != "" { - reporters = append(reporters, NewLogReporter()) - } - - if len(reporters) > 0 { - ag := newAggregator(reporters) - s.statters = append(s.statters, ag) - go func() { - for range time.Tick(time.Duration(config.Interval * float64(time.Second))) { - ag.report(nil) - } - }() - } - - if config.Statsd != nil && config.Statsd.StatsdUdpTarget != "" { - std, err := NewStatsd(config.Statsd) - if err == nil { - s.statters = append(s.statters, std) - } else { - logrus.WithError(err).Error("Couldn't create statsd reporter") - } - } - - if len(reporters) == 0 && config.Statsd == nil && config.History == 0 { - return &NilStatter{} - } - - if config.GCStats >= 0 { - if config.GCStats == 0 { - config.GCStats = 1 - } - go StartReportingMemoryAndGC(s, time.Duration(config.GCStats)*time.Second) - } - - return s + return NewMetricsStatter(config) } func HTTPReturnJson(w http.ResponseWriter, result interface{}) { @@ -146,25 +66,16 @@ func HTTPReturnJson(w http.ResponseWriter, result interface{}) { } } -// Convert a string to a stat name by replacing '.' with '_', lowercasing the -// string and trimming it. Doesn't do any validation, so do try this out -// locally before sending stats. -func AsStatField(input string) string { - return strings.Replace(strings.ToLower(strings.TrimSpace(input)), ".", "_", -1) -} - // statsd like API on top of the map manipulation API. type Timer struct { - statter Statter - component string - stat string - start time.Time - rate float32 - measured bool + statter Statter + stat []string + start time.Time + measured bool } -func newTimer(st Statter, component, stat string, rate float32) *Timer { - return &Timer{st, component, stat, time.Now(), rate, false} +func newTimer(st Statter, stat ...string) *Timer { + return &Timer{st, stat, time.Now(), false} } func (timer *Timer) Measure() { @@ -173,15 +84,29 @@ func (timer *Timer) Measure() { } timer.measured = true - timer.statter.Time(timer.component, timer.stat, time.Since(timer.start), timer.rate) + timer.statter.Time(time.Since(timer.start), timer.stat...) } type NilStatter struct{} -func (n *NilStatter) Inc(component string, stat string, value int64, rate float32) {} -func (n *NilStatter) Gauge(component string, stat string, value int64, rate float32) {} -func (n *NilStatter) Measure(component string, stat string, value int64, rate float32) {} -func (n *NilStatter) Time(component string, stat string, value time.Duration, rate float32) {} -func (r *NilStatter) NewTimer(component string, stat string, rate float32) *Timer { - return newTimer(r, component, stat, rate) +func (n *NilStatter) Inc(value int64, stat ...string) {} +func (n *NilStatter) Gauge(value int64, stat ...string) {} +func (n *NilStatter) Measure(value int64, stat ...string) {} +func (n *NilStatter) Time(value time.Duration, stat ...string) {} +func (r *NilStatter) NewTimer(stat ...string) *Timer { + return newTimer(r, stat...) +} + +func whoami() string { + a, _ := net.InterfaceAddrs() + for i := range a { + // is a textual representation of an IPv4 address + z, _, err := net.ParseCIDR(a[i].String()) + if a[i].Network() == "ip+net" && err == nil && z.To4() != nil { + if !bytes.Equal(z, net.ParseIP("127.0.0.1")) { + return strings.Replace(fmt.Sprintf("%v", z), ".", "_", -1) + } + } + } + return "127_0_0_1" // shrug } diff --git a/common/stats/statsd.go b/common/stats/statsd.go index 1ff1c22..e1e81fa 100644 --- a/common/stats/statsd.go +++ b/common/stats/statsd.go @@ -15,21 +15,12 @@ package stats import ( - "bytes" - "fmt" - "net" "strings" "time" "github.com/cactus/go-statsd-client/statsd" ) -type StatsdConfig struct { - StatsdUdpTarget string `json:"target" mapstructure:"target" envconfig:"STATSD_TARGET"` - Interval int64 `json:"interval" envconfig:"STATSD_INTERVAL"` - Prefix string `json:"prefix" envconfig:"STATSD_PREFIX"` -} - type keyCreator interface { // The return value of Key *MUST* never have a '.' at the end. Key(stat string) string @@ -63,20 +54,6 @@ func (pkc *prefixKeyCreator) Key(stat string) string { return prefix + "." + stat } -func whoami() string { - a, _ := net.InterfaceAddrs() - for i := range a { - // is a textual representation of an IPv4 address - z, _, err := net.ParseCIDR(a[i].String()) - if a[i].Network() == "ip+net" && err == nil && z.To4() != nil { - if !bytes.Equal(z, net.ParseIP("127.0.0.1")) { - return strings.Replace(fmt.Sprintf("%v", z), ".", "_", -1) - } - } - } - return "127_0_0_1" // shrug -} - // The config.Prefix is sent before each message and can be used to set API // keys. The prefix is used as the key prefix. // If config is nil, creates a noop reporter. @@ -100,27 +77,22 @@ func NewStatsd(config *StatsdConfig) (*theStatsdReporter, error) { return &theStatsdReporter{keyCreator: &prefixKeyCreator{}, client: client}, nil } -func (sr *theStatsdReporter) Inc(component, stat string, value int64, rate float32) { - sr.client.Inc(sr.keyCreator.Key(component+"."+stat), value, rate) +func (sr *theStatsdReporter) Inc(value int64, stat ...string) { + sr.client.Inc(sr.keyCreator.Key(strings.Join(stat, ".")), value, 1) } -func (sr *theStatsdReporter) Measure(component, stat string, delta int64, rate float32) { - sr.client.Timing(sr.keyCreator.Key(component+"."+stat), delta, rate) +func (sr *theStatsdReporter) Measure(delta int64, stat ...string) { + sr.client.Timing(sr.keyCreator.Key(strings.Join(stat, ".")), delta, 1) } -func (sr *theStatsdReporter) Time(component, stat string, delta time.Duration, rate float32) { - sr.client.TimingDuration(sr.keyCreator.Key(component+"."+stat), delta, rate) +func (sr *theStatsdReporter) Time(delta time.Duration, stat ...string) { + sr.client.TimingDuration(sr.keyCreator.Key(strings.Join(stat, ".")), delta, 1) } -func (sr *theStatsdReporter) Gauge(component, stat string, value int64, rate float32) { - sr.client.Gauge(sr.keyCreator.Key(component+"."+stat), value, rate) +func (sr *theStatsdReporter) Gauge(value int64, stat ...string) { + sr.client.Gauge(sr.keyCreator.Key(strings.Join(stat, ".")), value, 1) } -func (sr *theStatsdReporter) NewTimer(component string, stat string, rate float32) *Timer { - return newTimer(sr, component, stat, rate) +func (sr *theStatsdReporter) NewTimer(stat ...string) *Timer { + return newTimer(sr, strings.Join(stat, ".")) } - -// We need some kind of all-or-nothing sampler where multiple stats can be -// given the same rate and they are either all logged on that run or none of -// them are. The statsd library we use ends up doing its own rate calculation -// which is going to impede doing something like this. diff --git a/drivers/docker/docker.go b/drivers/docker/docker.go index 58ddcb0..4b5a413 100644 --- a/drivers/docker/docker.go +++ b/drivers/docker/docker.go @@ -33,7 +33,6 @@ import ( "github.com/fsouza/go-dockerclient" "github.com/heroku/docker-registry-client/registry" "github.com/iron-io/runner/common" - "github.com/iron-io/runner/common/stats" "github.com/iron-io/runner/drivers" ) @@ -278,7 +277,7 @@ func (drv *DockerDriver) Prepare(ctx context.Context, task drivers.ContainerTask return nil, err } - createTimer := drv.NewTimer("docker", "create_container", 1.0) + createTimer := drv.NewTimer("docker", "create_container") _, err = drv.docker.CreateContainer(container) createTimer.Measure() if err != nil { @@ -313,7 +312,7 @@ func (c *cookie) Run(ctx context.Context) (drivers.RunResult, error) { } func (drv *DockerDriver) removeContainer(container string) error { - removeTimer := drv.NewTimer("docker", "remove_container", 1.0) + removeTimer := drv.NewTimer("docker", "remove_container") defer removeTimer.Measure() err := drv.docker.RemoveContainer(docker.RemoveContainerOptions{ ID: container, Force: true, RemoveVolumes: true}) @@ -358,10 +357,10 @@ func (drv *DockerDriver) pullImage(ctx context.Context, task drivers.ContainerTa reg, repo, tag := drivers.ParseImage(task.Image()) globalRepo := path.Join(reg, repo) - pullTimer := drv.NewTimer("docker", "pull_image", 1.0) + pullTimer := drv.NewTimer("docker", "pull_image") defer pullTimer.Measure() - drv.Inc("docker", "pull_image_count."+stats.AsStatField(task.Image()), 1, 1) + drv.Inc(1, "docker", "pull_image_count") if reg != "" { config.ServerAddress = reg @@ -377,7 +376,7 @@ func (drv *DockerDriver) pullImage(ctx context.Context, task drivers.ContainerTa err = drv.docker.PullImage(docker.PullImageOptions{Repository: globalRepo, Tag: tag, Context: ctx}, config) if err != nil { - drv.Inc("task", "error.pull."+stats.AsStatField(task.Image()), 1, 1) + drv.Inc(1, "docker", "error", "pull", "@hostname") log.WithFields(logrus.Fields{"registry": config.ServerAddress, "username": config.Username, "image": task.Image()}).WithError(err).Error("Failed to pull image") // TODO need to inspect for hub or network errors and pick. @@ -413,7 +412,7 @@ func (drv *DockerDriver) run(ctx context.Context, container string, task drivers mwOut, mwErr := task.Logger() - timer := drv.NewTimer("docker", "attach_container", 1) + timer := drv.NewTimer("docker", "attach_container") waiter, err := drv.docker.AttachToContainerNonBlocking(docker.AttachToContainerOptions{ Container: container, OutputStream: mwOut, ErrorStream: mwErr, Stream: true, Logs: true, Stdout: true, Stderr: true, @@ -428,7 +427,7 @@ func (drv *DockerDriver) run(ctx context.Context, container string, task drivers return nil, err } - taskTimer := drv.NewTimer("docker", "container_runtime", 1) + taskTimer := drv.NewTimer("docker", "container_runtime") // can discard error, inspect will tell us about the task and wait will retry under the hood drv.docker.WaitContainerWithContext(container, ctx) @@ -462,7 +461,7 @@ func (drv *DockerDriver) nanny(ctx context.Context, container string) { } func (drv *DockerDriver) cancel(container string) { - stopTimer := drv.NewTimer("docker", "stop_container", 1.0) + stopTimer := drv.NewTimer("docker", "stop_container") err := drv.docker.StopContainer(container, 30) stopTimer.Measure() if err != nil { @@ -565,7 +564,7 @@ func newContainerID(task drivers.ContainerTask) string { func (drv *DockerDriver) startTask(ctx context.Context, container string) error { log := common.Logger(ctx) - startTimer := drv.NewTimer("docker", "start_container", 1.0) + startTimer := drv.NewTimer("docker", "start_container") log.WithFields(logrus.Fields{"container": container}).Debug("Starting container execution") err := drv.docker.StartContainerWithContext(container, nil, ctx) startTimer.Measure() @@ -623,14 +622,14 @@ func (drv *DockerDriver) status(ctx context.Context, container string) (status s case 0: return drivers.StatusSuccess, nil case 137: // OOM - drv.Inc("docker", "oom", 1, 1) + drv.Inc(1, "docker", "oom", "@hostname") if !cinfo.State.OOMKilled { // It is possible that the host itself is running out of memory and // the host kernel killed one of the container processes. // See: https://github.com/docker/docker/issues/15621 // TODO reed: isn't an OOM an OOM? this is wasting space imo log.WithFields(logrus.Fields{"container": container}).Info("Setting task as OOM killed, but docker disagreed.") - drv.Inc("docker", "possible_oom_false_alarm", 1, 1.0) + drv.Inc(1, "docker", "possible_oom_false_alarm", "@hostname") } return drivers.StatusKilled, drivers.ErrOutOfMemory diff --git a/drivers/docker/docker_client.go b/drivers/docker/docker_client.go index d831406..170b476 100644 --- a/drivers/docker/docker_client.go +++ b/drivers/docker/docker_client.go @@ -115,7 +115,7 @@ func (d *dockerWrap) retry(ctx context.Context, f func() error) error { for { select { case <-ctx.Done(): - d.Inc("task", "fail.docker", 1, 1) + d.Inc(1, "@hostname", "task", "fail", "docker") logrus.WithError(ctx.Err()).Warnf("retrying on docker errors timed out, restart docker or rotate this instance?") return ctx.Err() default: @@ -125,11 +125,11 @@ func (d *dockerWrap) retry(ctx context.Context, f func() error) error { if common.IsTemporary(err) || isDocker50x(err) { logrus.WithError(err).Warn("docker temporary error, retrying") b.Sleep() - d.Inc("task", "error.docker", 1, 1) + d.Inc(1, "task", "error", "docker", "@hostname") continue } if err != nil { - d.Inc("task", "error.docker", 1, 1) + d.Inc(1, "task", "error", "docker", "@hostname") } return err } diff --git a/glide.lock b/glide.lock index b215195..e34423a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: d7c3318fa4d64560e9149d2ebaf0e863232ab42ffbdf13473d4260a23d2ad2b8 -updated: 2017-04-05T13:35:00.148390772-07:00 +hash: 277e44e09c976b565ecfb0bd293ccff1c73f20947c4dc5602f3b004803d44dcd +updated: 2017-05-11T20:32:04.484469703+06:00 imports: - name: code.cloudfoundry.org/bytefmt version: a75017a21993c80187c7fa4f3c1ec22ddd6a8cd5 @@ -7,6 +7,8 @@ imports: version: c74861fe6a7bb8ede0a010ce4485bdbb4fc4c985 subpackages: - proto +- name: github.com/armon/go-metrics + version: 93f237eba9b0602f3e73710416558854a81d9337 - name: github.com/Azure/go-ansiterm version: fa152c58bc15761d0200cb75fe958b89a9d4888e subpackages: @@ -62,7 +64,7 @@ imports: - name: github.com/docker/libtrust version: fa567046d9b14f6aa788882a950d69651d230b21 - name: github.com/fsouza/go-dockerclient - version: e24e809e9db395f1e3c85af1b88f2002023610f5 + version: 30d142bbfdec74e09ecb4bdb89a44440ae4662ac - name: github.com/golang/protobuf version: 2402d76f3d41f928c7902a765dfc872356dd3aad subpackages: @@ -77,6 +79,10 @@ imports: version: 95467b6cacee2a06f112a3cf7e47a70fad6000cf subpackages: - registry +- name: github.com/mattn/go-colorable + version: ded68f7a9561c023e790de24279db7ebf473ea80 +- name: github.com/mattn/go-isatty + version: fc9e8d8ef48496124e79ae0df75490096eccf6fe - name: github.com/Microsoft/go-winio version: 24a3e3d3fc7451805e09d11e11e95d9a0a4f205e - name: github.com/opencontainers/runc @@ -99,6 +105,11 @@ imports: subpackages: - unix - windows +- name: gopkg.in/inconshreveable/log15.v2 + version: b105bd37f74e5d9dc7b6ad7806715c7a2b83fd3f + subpackages: + - stack + - term testImports: - name: github.com/vrischmann/envconfig version: 757beaaeac8d14bcc7ea3f71488d65cf45cf2eff diff --git a/glide.yaml b/glide.yaml index eb32597..ada4155 100644 --- a/glide.yaml +++ b/glide.yaml @@ -18,6 +18,7 @@ import: subpackages: - registry - package: code.cloudfoundry.org/bytefmt +- package: github.com/armon/go-metrics testImport: - package: github.com/vrischmann/envconfig version: 757beaaeac8d14bcc7ea3f71488d65cf45cf2eff