diff --git a/.gitignore b/.gitignore
index daf913b..0cdefe0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,4 @@ _testmain.go
*.exe
*.test
*.prof
+bin/
diff --git a/Makefile b/Makefile
index 70a5c47..cbe4830 100644
--- a/Makefile
+++ b/Makefile
@@ -1,15 +1,46 @@
-all: namenode_exporter resourcemanager_exporter
+# ###############
+# Go Project
+# ###############
+CPWD ?= $(PWD)
+BIN_PATH ?= ./cmd
+BIN_NAMES ?= namenode-exporter resourcemanager-exporter
+GO_FLAGS ?= GOOS=linux GOARCH=amd64
+export GO111MODULE=on
+
+# #######
+# General
+clean:
+ rm -rfv bin/*
+
+all: build
.PHONY: all
deps:
- go get github.com/prometheus/client_golang/prometheus
- go get github.com/prometheus/log
+ @mkdir ./bin || true
-namenode_exporter: deps namenode_exporter.go
- go build namenode_exporter.go
+# #######
+# Builder
-resourcemanager_exporter: deps resourcemanager_exporter.go
- go build resourcemanager_exporter.go
+build: deps
+ @for CMD in $(BIN_NAMES); do \
+ cd $(CPWD)/$(BIN_PATH)/$${CMD}; \
+ echo -e "** Building cmd: $${CMD} **"; \
+ GOOS=linux GOARCH=amd64 \
+ $(GO_FLAGS) go build -o $(CPWD)/bin/$${CMD}; \
+ done
+ @cd $(CPWD)
-clean:
- rm -rf namenode_exporter resourcemanager_exporter
+# #######
+# Release
+tag:
+ $(call deps_tag,$@)
+ git tag -a $(shell cat VERSION) -m "$(message)"
+ git push origin $(shell cat VERSION)
+
+tag-attach:
+ @for CMD in $(BIN_NAMES); do \
+ echo -e "** Publish bin to github: $${CMD} **"; \
+ strip $(CPWD)/bin/$${CMD}; \
+ ./hack/github-tag-attachment.sh $(shell cat VERSION) $(CPWD)/bin/$${CMD}; \
+ done
+ @cd $(CPWD)
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..1474d00
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+v0.2.0
diff --git a/namenode_exporter.go b/cmd/namenode-exporter/namenode_exporter.go
similarity index 60%
rename from namenode_exporter.go
rename to cmd/namenode-exporter/namenode_exporter.go
index 97114f2..cc5d6b6 100644
--- a/namenode_exporter.go
+++ b/cmd/namenode-exporter/namenode_exporter.go
@@ -11,7 +11,8 @@ import (
)
const (
- namespace = "namenode"
+ namespace = "namenode"
+ maxIdleConnections = 10
)
var (
@@ -21,26 +22,39 @@ var (
)
type Exporter struct {
- url string
- MissingBlocks prometheus.Gauge
- CapacityTotal prometheus.Gauge
- CapacityUsed prometheus.Gauge
- CapacityRemaining prometheus.Gauge
- CapacityUsedNonDFS prometheus.Gauge
- BlocksTotal prometheus.Gauge
- FilesTotal prometheus.Gauge
- CorruptBlocks prometheus.Gauge
- ExcessBlocks prometheus.Gauge
- StaleDataNodes prometheus.Gauge
- pnGcCount prometheus.Gauge
- pnGcTime prometheus.Gauge
- cmsGcCount prometheus.Gauge
- cmsGcTime prometheus.Gauge
- heapMemoryUsageCommitted prometheus.Gauge
- heapMemoryUsageInit prometheus.Gauge
- heapMemoryUsageMax prometheus.Gauge
- heapMemoryUsageUsed prometheus.Gauge
- isActive prometheus.Gauge
+ url string
+ MissingBlocks prometheus.Gauge
+ CapacityTotal prometheus.Gauge
+ CapacityUsed prometheus.Gauge
+ CapacityRemaining prometheus.Gauge
+ CapacityUsedNonDFS prometheus.Gauge
+ BlocksTotal prometheus.Gauge
+ FilesTotal prometheus.Gauge
+ CorruptBlocks prometheus.Gauge
+ ExcessBlocks prometheus.Gauge
+ StaleDataNodes prometheus.Gauge
+ pnGcCount prometheus.Gauge
+ pnGcTime prometheus.Gauge
+ cmsGcCount prometheus.Gauge
+ cmsGcTime prometheus.Gauge
+ heapMemoryUsageCommitted prometheus.Gauge
+ heapMemoryUsageInit prometheus.Gauge
+ heapMemoryUsageMax prometheus.Gauge
+ heapMemoryUsageUsed prometheus.Gauge
+ isActive prometheus.Gauge
+ BlockCapacity prometheus.Gauge
+ TotalLoad prometheus.Gauge
+ UnderReplicatedBlocks prometheus.Gauge
+ VolumeFailuresTotal prometheus.Gauge
+ NumLiveDataNodes prometheus.Gauge
+ NumDeadDataNodes prometheus.Gauge
+ GcCountConcurrentMarkSweep prometheus.Gauge
+ GcTimeMillisConcurrentMarkSweep prometheus.Gauge
+ MemNonHeapUsedM prometheus.Gauge
+ MemNonHeapCommittedM prometheus.Gauge
+ MemHeapUsedM prometheus.Gauge
+ MemHeapCommittedM prometheus.Gauge
+ MemHeapMaxM prometheus.Gauge
}
func NewExporter(url string) *Exporter {
@@ -141,6 +155,71 @@ func NewExporter(url string) *Exporter {
Name: "isActive",
Help: "isActive",
}),
+ BlockCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "BlockCapacity",
+ Help: "BlockCapacity",
+ }),
+ TotalLoad: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "TotalLoad",
+ Help: "TotalLoad",
+ }),
+ UnderReplicatedBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "UnderReplicatedBlocks",
+ Help: "UnderReplicatedBlocks",
+ }),
+ VolumeFailuresTotal: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "VolumeFailuresTotal",
+ Help: "VolumeFailuresTotal",
+ }),
+ NumLiveDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "NumLiveDataNodes",
+ Help: "NumLiveDataNodes",
+ }),
+ NumDeadDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "NumDeadDataNodes",
+ Help: "NumDeadDataNodes",
+ }),
+ GcCountConcurrentMarkSweep: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "GcCountConcurrentMarkSweep",
+ Help: "GcCountConcurrentMarkSweep",
+ }),
+ GcTimeMillisConcurrentMarkSweep: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "GcTimeMillisConcurrentMarkSweep",
+ Help: "GcTimeMillisConcurrentMarkSweep",
+ }),
+ MemNonHeapUsedM: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "MemNonHeapUsedM",
+ Help: "MemNonHeapUsedM",
+ }),
+ MemNonHeapCommittedM: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "MemNonHeapCommittedM",
+ Help: "MemNonHeapCommittedM",
+ }),
+ MemHeapUsedM: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "MemHeapUsedM",
+ Help: "MemHeapUsedM",
+ }),
+ MemHeapCommittedM: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "MemHeapCommittedM",
+ Help: "MemHeapCommittedM",
+ }),
+ MemHeapMaxM: prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: namespace,
+ Name: "MemHeapMaxM",
+ Help: "MemHeapMaxM",
+ }),
}
}
@@ -165,11 +244,26 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
e.heapMemoryUsageMax.Describe(ch)
e.heapMemoryUsageUsed.Describe(ch)
e.isActive.Describe(ch)
+ e.BlockCapacity.Describe(ch)
+ e.TotalLoad.Describe(ch)
+ e.UnderReplicatedBlocks.Describe(ch)
+ e.VolumeFailuresTotal.Describe(ch)
+ e.NumLiveDataNodes.Describe(ch)
+ e.NumDeadDataNodes.Describe(ch)
+ e.GcCountConcurrentMarkSweep.Describe(ch)
+ e.GcTimeMillisConcurrentMarkSweep.Describe(ch)
+ e.MemNonHeapUsedM.Describe(ch)
+ e.MemNonHeapCommittedM.Describe(ch)
+ e.MemHeapUsedM.Describe(ch)
+ e.MemHeapCommittedM.Describe(ch)
+ e.MemHeapMaxM.Describe(ch)
}
// Collect implements the prometheus.Collector interface.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
- resp, err := http.Get(e.url)
+ tr := &http.Transport{MaxIdleConns: maxIdleConnections}
+ client := &http.Client{Transport: tr}
+ resp, err := client.Get(e.url)
if err != nil {
log.Error(err)
}
@@ -245,6 +339,27 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.CorruptBlocks.Set(nameDataMap["CorruptBlocks"].(float64))
e.ExcessBlocks.Set(nameDataMap["ExcessBlocks"].(float64))
e.StaleDataNodes.Set(nameDataMap["StaleDataNodes"].(float64))
+ e.BlockCapacity.Set(nameDataMap["BlockCapacity"].(float64))
+ e.TotalLoad.Set(nameDataMap["TotalLoad"].(float64))
+ e.UnderReplicatedBlocks.Set(nameDataMap["UnderReplicatedBlocks"].(float64))
+ }
+ if nameDataMap["name"] == "Hadoop:service=NameNode,name=FSNamesystemState" {
+ e.VolumeFailuresTotal.Set(nameDataMap["VolumeFailuresTotal"].(float64))
+ e.NumLiveDataNodes.Set(nameDataMap["NumLiveDataNodes"].(float64))
+ e.NumDeadDataNodes.Set(nameDataMap["NumDeadDataNodes"].(float64))
+ }
+ if nameDataMap["name"] == "Hadoop:service=NameNode,name=JvmMetrics" {
+ if _, ok := nameDataMap["GcCountConcurrentMarkSweep"]; ok {
+ e.GcCountConcurrentMarkSweep.Set(nameDataMap["GcCountConcurrentMarkSweep"].(float64))
+ }
+ if _, ok := nameDataMap["GcTimeMillisConcurrentMarkSweep"]; ok {
+ e.GcTimeMillisConcurrentMarkSweep.Set(nameDataMap["GcTimeMillisConcurrentMarkSweep"].(float64))
+ }
+ e.MemNonHeapUsedM.Set(nameDataMap["MemNonHeapUsedM"].(float64))
+ e.MemNonHeapCommittedM.Set(nameDataMap["MemNonHeapCommittedM"].(float64))
+ e.MemHeapUsedM.Set(nameDataMap["MemHeapUsedM"].(float64))
+ e.MemHeapCommittedM.Set(nameDataMap["MemHeapCommittedM"].(float64))
+ e.MemHeapMaxM.Set(nameDataMap["MemHeapMaxM"].(float64))
}
if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ParNew" {
e.pnGcCount.Set(nameDataMap["CollectionCount"].(float64))
@@ -300,6 +415,19 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.heapMemoryUsageMax.Collect(ch)
e.heapMemoryUsageUsed.Collect(ch)
e.isActive.Collect(ch)
+ e.BlockCapacity.Collect(ch)
+ e.TotalLoad.Collect(ch)
+ e.UnderReplicatedBlocks.Collect(ch)
+ e.VolumeFailuresTotal.Collect(ch)
+ e.NumLiveDataNodes.Collect(ch)
+ e.NumDeadDataNodes.Collect(ch)
+ e.GcCountConcurrentMarkSweep.Collect(ch)
+ e.GcTimeMillisConcurrentMarkSweep.Collect(ch)
+ e.MemNonHeapUsedM.Collect(ch)
+ e.MemNonHeapCommittedM.Collect(ch)
+ e.MemHeapUsedM.Collect(ch)
+ e.MemHeapCommittedM.Collect(ch)
+ e.MemHeapMaxM.Collect(ch)
}
func main() {
@@ -315,6 +443,7 @@ func main() {
NameNode Exporter
NameNode Exporter
+ Parsing JMX counters over HTTP/HTTPS.
Metrics