add raw_expression

Readme.md

@@ -288,6 +288,17 @@ metrics:
    type: counter
   # This setting requires an almost monotonic counter as the source. When monotonicy is enforced, the metric value is regularly written to disk. Thus, resets in the source counter can be detected and corrected by adding an offset as if the reset did not happen. The result is a true monotonic increasing time series, like an ever growing counter.
    force_monotonicy: true
+ - prom_name: linky_time
+  # The name of the metric in a MQTT JSON message
+   mqtt_name: linky_current_date
+  # Regular expression to only match sensors with the given name pattern
+   sensor_name_filter: "^linky.*$"
+  # The prometheus help text for this metric
+   help: current unix timestamp from linky
+  # The prometheus type for this metric. Valid values are: "gauge" and "counter"
+   type: gauge
+  # convert dynamic datetime string to unix timestamp
+   raw_expression: 'date(string(raw_value), "H060102150405", "Europe/Paris").Unix()'
 ```
 
 ### Environment Variables
@@ -335,13 +346,18 @@ Create a docker secret to store the password(`mqtt-credential` in the example be
 
 ### Expressions
 
-Metric values can be derived from sensor inputs using complex expressions. Set the metric config option `expression` to the desired formular to calculate the result from the input. Here's an example which integrates all positive values over time:
-
+Metric values can be derived from sensor inputs using complex expressions. Set the metric config option `raw_expression` or `expression` to the desired formular to calculate the result from the input. `raw_expression` and `expression` are mutually exclusives:
+* `raw_expression` is run without raw value conversion. It's `raw_expression` duty to handle the conversion. Only `raw_value` is set while `value` is always set to 0.0. Here is an example which convert datetime (format `HYYMMDDhhmmss`) to unix timestamp:
+```yaml
+raw_expression: 'date(string(raw_value), "H060102150405", "Europe/Paris").Unix()'
+```
+* `expression` is run after raw value conversion. If conversion fails, `expression` is not run. Here's an example which integrates all positive values over time:
 ```yaml
 expression: "value > 0 ? last_result + value * elapsed.Seconds() : last_result"
 ```
 
 During the evaluation, the following variables are available to the expression:
+* `raw_value` - the raw MQTT sensor value (without any conversion)
 * `value` - the current sensor value (after string-value mapping, if configured)
 * `last_value` - the `value` during the previous expression evaluation
 * `last_result` - the result from the previous expression evaluation
@@ -366,6 +382,7 @@ The `last_value`, `last_result`, and the timestamp of the last evaluation are re
 
 It is important to understand the sequence of transformations from a sensor input to the final output which is exported to Prometheus. The steps are as follows:
 
+If `raw_expression` is set, the generated value of the expression is exported to Prometheus. Otherwise:
 1. The sensor input is converted to a number. If a `string_value_mapping` is configured, it is consulted for the conversion.
 1. If an `expression` is configured, it is evaluated using the converted number. The result of the evaluation replaces the converted sensor value.
 1. If `force_monotonicy` is set to `true`, any new value that is smaller than the previous one is considered to be a counter reset. When a reset is detected, the previous value becomes the value offset which is automatically added to each consecutive value. The offset is persistet between restarts of mqtt2prometheus.

pkg/config/config.go

@@ -140,6 +140,7 @@ type MetricConfig struct {
 	Help               string                    `yaml:"help"`
 	ValueType          string                    `yaml:"type"`
 	OmitTimestamp      bool                      `yaml:"omit_timestamp"`
+	RawExpression      string                    `yaml:"raw_expression"`
 	Expression         string                    `yaml:"expression"`
 	ForceMonotonicy    bool                      `yaml:"force_monotonicy"`
 	ConstantLabels     map[string]string         `yaml:"const_labels"`
@@ -243,6 +244,10 @@ func LoadConfig(configFile string, logger *zap.Logger) (Config, error) {
 			}
 			logger.Warn("string_value_mapping.error_value is deprecated: please use error_value at the metric level.", zap.String("prometheusName", m.PrometheusName), zap.String("MQTTName", m.MQTTName))
 		}
+
+		if m.Expression != "" && m.RawExpression != ""  {
+			return Config{}, fmt.Errorf("metric %s/%s: expression and raw_expression are mutually exclusive.", m.MQTTName, m.PrometheusName)
+		}
 	}
 	if forcesMonotonicy {
 		if err := os.MkdirAll(cfg.Cache.StateDir, 0755); err != nil {

pkg/metrics/parser.go

@@ -24,6 +24,8 @@ type dynamicState struct {
 	// Last value that was used for evaluating the given expression
 	LastExprValue float64 `yaml:"last_expr_value"`
 	// Last result returned from evaluating the given expression
+	LastExprRawValue interface{} `yaml:"last_expr_raw_value"`
+	// Last result returned from evaluating the given expression
 	LastExprResult float64 `yaml:"last_expr_result"`
 	// Last result returned from evaluating the given expression
 	LastExprTimestamp time.Time `yaml:"last_expr_timestamp"`
@@ -53,19 +55,21 @@ type Parser struct {
 
 // Identifiers within the expression evaluation environment.
 const (
-	env_value       = "value"
-	env_last_value  = "last_value"
-	env_last_result = "last_result"
-	env_elapsed     = "elapsed"
-	env_now         = "now"
-	env_int         = "int"
-	env_float       = "float"
-	env_round       = "round"
-	env_ceil        = "ceil"
-	env_floor       = "floor"
-	env_abs         = "abs"
-	env_min         = "min"
-	env_max         = "max"
+	env_raw_value      = "raw_value"
+	env_value          = "value"
+	env_last_value     = "last_value"
+	env_last_raw_value = "last_raw_value"
+	env_last_result    = "last_result"
+	env_elapsed        = "elapsed"
+	env_now            = "now"
+	env_int            = "int"
+	env_float          = "float"
+	env_round          = "round"
+	env_ceil           = "ceil"
+	env_floor          = "floor"
+	env_abs            = "abs"
+	env_min            = "min"
+	env_max            = "max"
 )
 
 var now = time.Now
@@ -124,6 +128,7 @@ func toFloat64(i interface{}) float64 {
 func defaultExprEnv() map[string]interface{} {
 	return map[string]interface{}{
 		// Variables
+		env_raw_value:   nil,
 		env_value:       0.0,
 		env_last_value:  0.0,
 		env_last_result: 0.0,
@@ -177,60 +182,71 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in
 	var metricValue float64
 	var err error
 
-	if boolValue, ok := value.(bool); ok {
-		if boolValue {
-			metricValue = 1
-		} else {
-			metricValue = 0
+	if cfg.RawExpression != "" {
+		if metricValue, err = p.evalExpressionValue(metricID, cfg.RawExpression, value, metricValue); err != nil {
+			if cfg.ErrorValue != nil {
+				metricValue = *cfg.ErrorValue
+			} else {
+				return Metric{}, err
+			}
 		}
-	} else if strValue, ok := value.(string); ok {
-
-		// If string value mapping is defined, use that
-		if cfg.StringValueMapping != nil {
-
-			floatValue, ok := cfg.StringValueMapping.Map[strValue]
-			if ok {
-				metricValue = floatValue
+	} else {
 
-			// deprecated, replaced by ErrorValue from the upper level
-			} else if cfg.StringValueMapping.ErrorValue != nil {
-				metricValue = *cfg.StringValueMapping.ErrorValue
-			} else if cfg.ErrorValue != nil {
-				metricValue = *cfg.ErrorValue
+		if boolValue, ok := value.(bool); ok {
+			if boolValue {
+				metricValue = 1
 			} else {
-				return Metric{}, fmt.Errorf("got unexpected string data '%s'", strValue)
+				metricValue = 0
 			}
+		} else if strValue, ok := value.(string); ok {
 
-		} else {
+			// If string value mapping is defined, use that
+			if cfg.StringValueMapping != nil {
 
-			// otherwise try to parse float
-			floatValue, err := strconv.ParseFloat(strValue, 64)
-			if err != nil {
-				if cfg.ErrorValue != nil {
+				floatValue, ok := cfg.StringValueMapping.Map[strValue]
+				if ok {
+					metricValue = floatValue
+
+				// deprecated, replaced by ErrorValue from the upper level
+				} else if cfg.StringValueMapping.ErrorValue != nil {
+					metricValue = *cfg.StringValueMapping.ErrorValue
+				} else if cfg.ErrorValue != nil {
 					metricValue = *cfg.ErrorValue
 				} else {
-					return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value)
+					return Metric{}, fmt.Errorf("got unexpected string data '%s'", strValue)
 				}
+
 			} else {
-				metricValue = floatValue
+
+				// otherwise try to parse float
+				floatValue, err := strconv.ParseFloat(strValue, 64)
+				if err != nil {
+					if cfg.ErrorValue != nil {
+						metricValue = *cfg.ErrorValue
+					} else {
+						return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value)
+					}
+				} else {
+					metricValue = floatValue
+				}
+
 			}
 
+		} else if floatValue, ok := value.(float64); ok {
+			metricValue = floatValue
+		} else if cfg.ErrorValue != nil {
+			metricValue = *cfg.ErrorValue
+		} else {
+			return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value)
 		}
 
-	} else if floatValue, ok := value.(float64); ok {
-		metricValue = floatValue
-	} else if cfg.ErrorValue != nil {
-		metricValue = *cfg.ErrorValue
-	} else {
-		return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value)
-	}
-
-	if cfg.Expression != "" {
-		if metricValue, err = p.evalExpression(metricID, cfg.Expression, metricValue); err != nil {
-			if cfg.ErrorValue != nil {
-				metricValue = *cfg.ErrorValue
-			} else {
-				return Metric{}, err
+		if cfg.Expression != "" {
+			if metricValue, err = p.evalExpressionValue(metricID, cfg.Expression, value, metricValue); err != nil {
+				if cfg.ErrorValue != nil {
+					metricValue = *cfg.ErrorValue
+				} else {
+					return Metric{}, err
+				}
 			}
 		}
 	}
@@ -348,9 +364,9 @@ func (p *Parser) enforceMonotonicy(metricID string, value float64) (float64, err
 	return value + ms.dynamic.Offset, nil
 }
 
-// evalExpression runs the given code in the metric's environment and returns the result.
+// evalExpressionValue runs the given code in the metric's environment and returns the result.
 // In case of an error, the original value is returned.
-func (p *Parser) evalExpression(metricID, code string, value float64) (float64, error) {
+func (p *Parser) evalExpressionValue(metricID, code string, raw_value interface{}, value float64) (float64, error) {
 	ms, err := p.getMetricState(metricID)
 	if err != nil {
 		return value, err
@@ -366,8 +382,10 @@ func (p *Parser) evalExpression(metricID, code string, value float64) (float64,
 	}
 
 	// Update the environment
+	ms.env[env_raw_value] = raw_value
 	ms.env[env_value] = value
 	ms.env[env_last_value] = ms.dynamic.LastExprValue
+	ms.env[env_last_raw_value] = ms.dynamic.LastExprRawValue
 	ms.env[env_last_result] = ms.dynamic.LastExprResult
 	if ms.dynamic.LastExprTimestamp.IsZero() {
 		ms.env[env_elapsed] = time.Duration(0)
@@ -384,6 +402,7 @@ func (p *Parser) evalExpression(metricID, code string, value float64) (float64,
 
 	// Update the dynamic state
 	ms.dynamic.LastExprResult = ret
+	ms.dynamic.LastExprRawValue = raw_value
 	ms.dynamic.LastExprValue = value
 	ms.dynamic.LastExprTimestamp = now()
 

pkg/metrics/parser_test.go

@@ -667,6 +667,32 @@ func TestParser_parseMetric(t *testing.T) {
 				Value:       11.0, // 600 watts for 1 minute = 10 Wh
 			},
 		},
+		{
+			name: "raw expression, step 1",
+			fields: fields{
+				map[string][]*config.MetricConfig{
+					"apower": {
+						{
+							PrometheusName: "total_energy",
+							ValueType:      "gauge",
+							OmitTimestamp:  true,
+							RawExpression:  `float(join(filter(split(string(raw_value), ""), { # matches "^[0-9\\.]$" }), ""))`,
+					},
+					},
+				},
+			},
+			elapseNow: 3 * time.Minute,
+			args: args{
+				metricPath: "apower",
+				deviceID:   "shellyplus1pm-foo",
+				value:      "H42Jj.j44",
+			},
+			want: Metric{
+				Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil),
+				ValueType:   prometheus.GaugeValue,
+				Value:       42.44,
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -797,7 +823,7 @@ func TestParser_evalExpression(t *testing.T) {
 
 			p := NewParser(nil, ".", stateDir)
 			for i, value := range tt.values {
-				got, err := p.evalExpression(id, tt.expression, value)
+				got, err := p.evalExpressionValue(id, tt.expression, value, value)
 				want := tt.results[i]
 				if err != nil {
 					t.Errorf("evaluating the %dth value '%v' failed: %v", i, value, err)