make error_value global to metric and add error_expression

Readme.md

@@ -241,15 +241,15 @@ metrics:
   # A map of string to string for constant labels. This labels will be attached to every prometheus metric
    const_labels:
     sensor_type: ikea
+  # When specified, metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, expression fails, ...)
+  # If not specified, parsing error will occur.
+  error_value: 1
   # When specified, enables mapping between string values to metric values.
    string_value_mapping:
     # A map of string to metric value.
     map:
      off: 0
      low: 0
-    # Metric value to use if a match cannot be found in the map above.
-    # If not specified, parsing error will occur.
-    error_value: 1
   # The name of the metric in prometheus
  - prom_name: total_light_usage_seconds
   # The name of the metric in a MQTT JSON message
@@ -265,15 +265,15 @@ metrics:
   # A map of string to string for constant labels. This labels will be attached to every prometheus metric
    const_labels:
     sensor_type: ikea
+  # Metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, ...)
+  # If not specified, parsing error will occur.
+  error_value: 1
   # When specified, enables mapping between string values to metric values.
    string_value_mapping:
     # A map of string to metric value.
     map:
      off: 0
      low: 0
-    # Metric value to use if a match cannot be found in the map above.
-    # If not specified, parsing error will occur.
-    error_value: 1
   # Sum up the time the light is on, see the section "Expressions" below.
   expression: "value > 0 ? last_result + elapsed.Seconds() : last_result"
   # The name of the metric in prometheus

cmd/mqtt2prometheus.go

@@ -80,7 +80,7 @@ func main() {
 	logger := mustSetupLogger()
 	defer logger.Sync() //nolint:errcheck
 	c := make(chan os.Signal, 1)
-	cfg, err := config.LoadConfig(*configFlag)
+	cfg, err := config.LoadConfig(*configFlag, logger)
 	if err != nil {
 		logger.Fatal("Could not load config", zap.Error(err))
 	}

config.yaml.dist

@@ -81,12 +81,12 @@ metrics:
     # A map of string to string for constant labels. This labels will be attached to every prometheus metric
     const_labels:
       sensor_type: ikea
+    # Metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, ...)
+    # If not specified, parsing error will occur.
+    error_value: 1
     # When specified, enables mapping between string values to metric values.
     string_value_mapping:
       # A map of string to metric value.
       map:
         off: 0
         low: 0
-      # Metric value to use if a match cannot be found in the map above.
-      # If not specified, parsing error will occur.
-      error_value: 1

fuzzing/json_per_topic/fuzz.go

@@ -16,8 +16,8 @@ func Fuzz(data []byte) int {
 		{
 			PrometheusName: "enabled",
 			ValueType:      "gauge",
+			ErrorValue: floatP(12333),
 			StringValueMapping: &config.StringValueMappingConfig{
-				ErrorValue: floatP(12333),
 				Map: map[string]float64{
 					"foo": 112,
 					"bar": 2,

fuzzing/metric_per_topic/fuzz.go

@@ -17,8 +17,8 @@ func Fuzz(data []byte) int {
 		{
 			PrometheusName: "enabled",
 			ValueType:      "gauge",
+			ErrorValue: floatP(12333),
 			StringValueMapping: &config.StringValueMappingConfig{
-				ErrorValue: floatP(12333),
 				Map: map[string]float64{
 					"foo": 112,
 					"bar": 2,

hack/dht22.yaml

@@ -57,12 +57,12 @@ metrics:
     # A map of string to string for constant labels. This labels will be attached to every prometheus metric
     const_labels:
       sensor_type: ikea
+    # Metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, ...)
+    # If not specified, parsing error will occur.
+    error_value: 1
     # When specified, enables mapping between string values to metric values.
     string_value_mapping:
       # A map of string to metric value.
       map:
         off: 0
         low: 0
-      # Metric value to use if a match cannot be found in the map above.
-      # If not specified, parsing error will occur.
-      error_value: 1

pkg/config/config.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
@@ -144,11 +145,14 @@ type MetricConfig struct {
 	ConstantLabels     map[string]string         `yaml:"const_labels"`
 	StringValueMapping *StringValueMappingConfig `yaml:"string_value_mapping"`
 	MQTTValueScale     float64                   `yaml:"mqtt_value_scale"`
+	// ErrorValue is used while error during value parsing
+	ErrorValue         *float64                  `yaml:"error_value"`
 }
 
 // StringValueMappingConfig defines the mapping from string to float
 type StringValueMappingConfig struct {
-	// ErrorValue is used when no mapping is found in Map
+	// ErrorValue was used when no mapping is found in Map
+	// deprecated, a warning will be issued to migrate to metric level
 	ErrorValue *float64           `yaml:"error_value"`
 	Map        map[string]float64 `yaml:"map"`
 }
@@ -170,7 +174,7 @@ func (mc *MetricConfig) PrometheusValueType() prometheus.ValueType {
 	}
 }
 
-func LoadConfig(configFile string) (Config, error) {
+func LoadConfig(configFile string, logger *zap.Logger) (Config, error) {
 	configData, err := ioutil.ReadFile(configFile)
 	if err != nil {
 		return Config{}, err
@@ -232,6 +236,13 @@ func LoadConfig(configFile string) (Config, error) {
 		if m.ForceMonotonicy {
 			forcesMonotonicy = true
 		}
+
+		if m.StringValueMapping != nil && m.StringValueMapping.ErrorValue != nil {
+			if m.ErrorValue != nil {
+				return Config{}, fmt.Errorf("metric %s/%s: cannot set both string_value_mapping.error_value and error_value (string_value_mapping.error_value is deprecated).", m.MQTTName, m.PrometheusName)
+			}
+			logger.Warn("string_value_mapping.error_value is deprecated: please use error_value at the metric level.", zap.String("prometheusName", m.PrometheusName), zap.String("MQTTName", m.MQTTName))
+		}
 	}
 	if forcesMonotonicy {
 		if err := os.MkdirAll(cfg.Cache.StateDir, 0755); err != nil {

pkg/metrics/parser.go

@@ -179,8 +179,12 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in
 			floatValue, ok := cfg.StringValueMapping.Map[strValue]
 			if ok {
 				metricValue = floatValue
+
+			// deprecated, replaced by ErrorValue from the upper level
 			} else if cfg.StringValueMapping.ErrorValue != nil {
 				metricValue = *cfg.StringValueMapping.ErrorValue
+			} else if cfg.ErrorValue != nil {
+				metricValue = *cfg.ErrorValue
 			} else {
 				return Metric{}, fmt.Errorf("got unexpected string data '%s'", strValue)
 			}
@@ -190,27 +194,42 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in
 			// otherwise try to parse float
 			floatValue, err := strconv.ParseFloat(strValue, 64)
 			if err != nil {
-				return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value)
+				if cfg.ErrorValue != nil {
+					metricValue = *cfg.ErrorValue
+				} else {
+					return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value)
+				}
+			} else {
+				metricValue = floatValue
 			}
-			metricValue = floatValue
 
 		}
 
 	} else if floatValue, ok := value.(float64); ok {
 		metricValue = floatValue
+	} else if cfg.ErrorValue != nil {
+		metricValue = *cfg.ErrorValue
 	} else {
 		return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value)
 	}
 
 	if cfg.Expression != "" {
 		if metricValue, err = p.evalExpression(metricID, cfg.Expression, metricValue); err != nil {
-			return Metric{}, err
+			if cfg.ErrorValue != nil {
+				metricValue = *cfg.ErrorValue
+			} else {
+				return Metric{}, err
+			}
 		}
 	}
 
 	if cfg.ForceMonotonicy {
 		if metricValue, err = p.enforceMonotonicy(metricID, metricValue); err != nil {
-			return Metric{}, err
+			if cfg.ErrorValue != nil {
+				metricValue = *cfg.ErrorValue
+			} else {
+				return Metric{}, err
+			}
 		}
 	}
 

pkg/metrics/parser_test.go

@@ -39,6 +39,9 @@ func TestParser_parseMetric(t *testing.T) {
 		deviceID   string
 		value      interface{}
 	}
+
+	var errorValue float64 = 42.44
+
 	tests := []struct {
 		name      string
 		fields    fields
@@ -143,6 +146,32 @@ func TestParser_parseMetric(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "string value failure with errorValue",
+			fields: fields{
+				map[string][]*config.MetricConfig{
+					"temperature": {
+						{
+							PrometheusName: "temperature",
+							ValueType:      "gauge",
+							ErrorValue:     &errorValue,
+						},
+					},
+				},
+			},
+			args: args{
+				metricPath: "temperature",
+				deviceID:   "dht22",
+				value:      "12.6.5",
+			},
+			want: Metric{
+				Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil),
+				ValueType:   prometheus.GaugeValue,
+				Value:       errorValue,
+				IngestTime:  testNow(),
+				Topic:       "",
+			},
+		},
 		{
 			name: "float value",
 			fields: fields{
@@ -335,8 +364,8 @@ func TestParser_parseMetric(t *testing.T) {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
+							ErrorValue: floatP(12333),
 							StringValueMapping: &config.StringValueMappingConfig{
-								ErrorValue: floatP(12333),
 								Map: map[string]float64{
 									"foo": 112,
 									"bar": 2,
@@ -392,8 +421,8 @@ func TestParser_parseMetric(t *testing.T) {
 						{
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
+							ErrorValue: floatP(12333),
 							StringValueMapping: &config.StringValueMappingConfig{
-								ErrorValue: floatP(12333),
 								Map: map[string]float64{
 									"foo": 112,
 									"bar": 2,
@@ -419,7 +448,6 @@ func TestParser_parseMetric(t *testing.T) {
 							PrometheusName: "enabled",
 							ValueType:      "gauge",
 							StringValueMapping: &config.StringValueMappingConfig{
-								ErrorValue: floatP(12333),
 								Map: map[string]float64{
 									"foo": 112,
 									"bar": 2,