From 5ede83bcb75ef196109c33cced3390f3018c79c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LOYET?= <822436+fatpat@users.noreply.github.com> Date: Mon, 30 Dec 2024 09:24:34 +0100 Subject: [PATCH] add raw_expression --- Readme.md | 21 +++++- pkg/config/config.go | 5 ++ pkg/metrics/parser.go | 129 +++++++++++++++++++++---------------- pkg/metrics/parser_test.go | 28 +++++++- 4 files changed, 125 insertions(+), 58 deletions(-) diff --git a/Readme.md b/Readme.md index 02c1017..fa4cc2d 100644 --- a/Readme.md +++ b/Readme.md @@ -288,6 +288,17 @@ metrics: type: counter # This setting requires an almost monotonic counter as the source. When monotonicy is enforced, the metric value is regularly written to disk. Thus, resets in the source counter can be detected and corrected by adding an offset as if the reset did not happen. The result is a true monotonic increasing time series, like an ever growing counter. force_monotonicy: true + - prom_name: linky_time + # The name of the metric in a MQTT JSON message + mqtt_name: linky_current_date + # Regular expression to only match sensors with the given name pattern + sensor_name_filter: "^linky.*$" + # The prometheus help text for this metric + help: current unix timestamp from linky + # The prometheus type for this metric. Valid values are: "gauge" and "counter" + type: gauge + # convert dynamic datetime string to unix timestamp + raw_expression: 'date(string(raw_value), "H060102150405", "Europe/Paris").Unix()' ``` ### Environment Variables @@ -335,13 +346,18 @@ Create a docker secret to store the password(`mqtt-credential` in the example be ### Expressions -Metric values can be derived from sensor inputs using complex expressions. Set the metric config option `expression` to the desired formular to calculate the result from the input. Here's an example which integrates all positive values over time: - +Metric values can be derived from sensor inputs using complex expressions. Set the metric config option `raw_expression` or `expression` to the desired formular to calculate the result from the input. `raw_expression` and `expression` are mutually exclusives: +* `raw_expression` is run without raw value conversion. It's `raw_expression` duty to handle the conversion. Only `raw_value` is set while `value` is always set to 0.0. Here is an example which convert datetime (format `HYYMMDDhhmmss`) to unix timestamp: +```yaml +raw_expression: 'date(string(raw_value), "H060102150405", "Europe/Paris").Unix()' +``` +* `expression` is run after raw value conversion. If conversion fails, `expression` is not run. Here's an example which integrates all positive values over time: ```yaml expression: "value > 0 ? last_result + value * elapsed.Seconds() : last_result" ``` During the evaluation, the following variables are available to the expression: +* `raw_value` - the raw MQTT sensor value (without any conversion) * `value` - the current sensor value (after string-value mapping, if configured) * `last_value` - the `value` during the previous expression evaluation * `last_result` - the result from the previous expression evaluation @@ -366,6 +382,7 @@ The `last_value`, `last_result`, and the timestamp of the last evaluation are re It is important to understand the sequence of transformations from a sensor input to the final output which is exported to Prometheus. The steps are as follows: +If `raw_expression` is set, the generated value of the expression is exported to Prometheus. Otherwise: 1. The sensor input is converted to a number. If a `string_value_mapping` is configured, it is consulted for the conversion. 1. If an `expression` is configured, it is evaluated using the converted number. The result of the evaluation replaces the converted sensor value. 1. If `force_monotonicy` is set to `true`, any new value that is smaller than the previous one is considered to be a counter reset. When a reset is detected, the previous value becomes the value offset which is automatically added to each consecutive value. The offset is persistet between restarts of mqtt2prometheus. diff --git a/pkg/config/config.go b/pkg/config/config.go index 763e881..abf9572 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -140,6 +140,7 @@ type MetricConfig struct { Help string `yaml:"help"` ValueType string `yaml:"type"` OmitTimestamp bool `yaml:"omit_timestamp"` + RawExpression string `yaml:"raw_expression"` Expression string `yaml:"expression"` ForceMonotonicy bool `yaml:"force_monotonicy"` ConstantLabels map[string]string `yaml:"const_labels"` @@ -243,6 +244,10 @@ func LoadConfig(configFile string, logger *zap.Logger) (Config, error) { } logger.Warn("string_value_mapping.error_value is deprecated: please use error_value at the metric level.", zap.String("prometheusName", m.PrometheusName), zap.String("MQTTName", m.MQTTName)) } + + if m.Expression != "" && m.RawExpression != "" { + return Config{}, fmt.Errorf("metric %s/%s: expression and raw_expression are mutually exclusive.", m.MQTTName, m.PrometheusName) + } } if forcesMonotonicy { if err := os.MkdirAll(cfg.Cache.StateDir, 0755); err != nil { diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index 416b5c4..4877384 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -24,6 +24,8 @@ type dynamicState struct { // Last value that was used for evaluating the given expression LastExprValue float64 `yaml:"last_expr_value"` // Last result returned from evaluating the given expression + LastExprRawValue interface{} `yaml:"last_expr_raw_value"` + // Last result returned from evaluating the given expression LastExprResult float64 `yaml:"last_expr_result"` // Last result returned from evaluating the given expression LastExprTimestamp time.Time `yaml:"last_expr_timestamp"` @@ -53,19 +55,21 @@ type Parser struct { // Identifiers within the expression evaluation environment. const ( - env_value = "value" - env_last_value = "last_value" - env_last_result = "last_result" - env_elapsed = "elapsed" - env_now = "now" - env_int = "int" - env_float = "float" - env_round = "round" - env_ceil = "ceil" - env_floor = "floor" - env_abs = "abs" - env_min = "min" - env_max = "max" + env_raw_value = "raw_value" + env_value = "value" + env_last_value = "last_value" + env_last_raw_value = "last_raw_value" + env_last_result = "last_result" + env_elapsed = "elapsed" + env_now = "now" + env_int = "int" + env_float = "float" + env_round = "round" + env_ceil = "ceil" + env_floor = "floor" + env_abs = "abs" + env_min = "min" + env_max = "max" ) var now = time.Now @@ -124,6 +128,7 @@ func toFloat64(i interface{}) float64 { func defaultExprEnv() map[string]interface{} { return map[string]interface{}{ // Variables + env_raw_value: nil, env_value: 0.0, env_last_value: 0.0, env_last_result: 0.0, @@ -177,60 +182,71 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in var metricValue float64 var err error - if boolValue, ok := value.(bool); ok { - if boolValue { - metricValue = 1 - } else { - metricValue = 0 + if cfg.RawExpression != "" { + if metricValue, err = p.evalExpressionValue(metricID, cfg.RawExpression, value, metricValue); err != nil { + if cfg.ErrorValue != nil { + metricValue = *cfg.ErrorValue + } else { + return Metric{}, err + } } - } else if strValue, ok := value.(string); ok { - - // If string value mapping is defined, use that - if cfg.StringValueMapping != nil { - - floatValue, ok := cfg.StringValueMapping.Map[strValue] - if ok { - metricValue = floatValue + } else { - // deprecated, replaced by ErrorValue from the upper level - } else if cfg.StringValueMapping.ErrorValue != nil { - metricValue = *cfg.StringValueMapping.ErrorValue - } else if cfg.ErrorValue != nil { - metricValue = *cfg.ErrorValue + if boolValue, ok := value.(bool); ok { + if boolValue { + metricValue = 1 } else { - return Metric{}, fmt.Errorf("got unexpected string data '%s'", strValue) + metricValue = 0 } + } else if strValue, ok := value.(string); ok { - } else { + // If string value mapping is defined, use that + if cfg.StringValueMapping != nil { - // otherwise try to parse float - floatValue, err := strconv.ParseFloat(strValue, 64) - if err != nil { - if cfg.ErrorValue != nil { + floatValue, ok := cfg.StringValueMapping.Map[strValue] + if ok { + metricValue = floatValue + + // deprecated, replaced by ErrorValue from the upper level + } else if cfg.StringValueMapping.ErrorValue != nil { + metricValue = *cfg.StringValueMapping.ErrorValue + } else if cfg.ErrorValue != nil { metricValue = *cfg.ErrorValue } else { - return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value) + return Metric{}, fmt.Errorf("got unexpected string data '%s'", strValue) } + } else { - metricValue = floatValue + + // otherwise try to parse float + floatValue, err := strconv.ParseFloat(strValue, 64) + if err != nil { + if cfg.ErrorValue != nil { + metricValue = *cfg.ErrorValue + } else { + return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value) + } + } else { + metricValue = floatValue + } + } + } else if floatValue, ok := value.(float64); ok { + metricValue = floatValue + } else if cfg.ErrorValue != nil { + metricValue = *cfg.ErrorValue + } else { + return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value) } - } else if floatValue, ok := value.(float64); ok { - metricValue = floatValue - } else if cfg.ErrorValue != nil { - metricValue = *cfg.ErrorValue - } else { - return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value) - } - - if cfg.Expression != "" { - if metricValue, err = p.evalExpression(metricID, cfg.Expression, metricValue); err != nil { - if cfg.ErrorValue != nil { - metricValue = *cfg.ErrorValue - } else { - return Metric{}, err + if cfg.Expression != "" { + if metricValue, err = p.evalExpressionValue(metricID, cfg.Expression, value, metricValue); err != nil { + if cfg.ErrorValue != nil { + metricValue = *cfg.ErrorValue + } else { + return Metric{}, err + } } } } @@ -348,9 +364,9 @@ func (p *Parser) enforceMonotonicy(metricID string, value float64) (float64, err return value + ms.dynamic.Offset, nil } -// evalExpression runs the given code in the metric's environment and returns the result. +// evalExpressionValue runs the given code in the metric's environment and returns the result. // In case of an error, the original value is returned. -func (p *Parser) evalExpression(metricID, code string, value float64) (float64, error) { +func (p *Parser) evalExpressionValue(metricID, code string, raw_value interface{}, value float64) (float64, error) { ms, err := p.getMetricState(metricID) if err != nil { return value, err @@ -366,8 +382,10 @@ func (p *Parser) evalExpression(metricID, code string, value float64) (float64, } // Update the environment + ms.env[env_raw_value] = raw_value ms.env[env_value] = value ms.env[env_last_value] = ms.dynamic.LastExprValue + ms.env[env_last_raw_value] = ms.dynamic.LastExprRawValue ms.env[env_last_result] = ms.dynamic.LastExprResult if ms.dynamic.LastExprTimestamp.IsZero() { ms.env[env_elapsed] = time.Duration(0) @@ -384,6 +402,7 @@ func (p *Parser) evalExpression(metricID, code string, value float64) (float64, // Update the dynamic state ms.dynamic.LastExprResult = ret + ms.dynamic.LastExprRawValue = raw_value ms.dynamic.LastExprValue = value ms.dynamic.LastExprTimestamp = now() diff --git a/pkg/metrics/parser_test.go b/pkg/metrics/parser_test.go index 32b8bf2..5a8036a 100644 --- a/pkg/metrics/parser_test.go +++ b/pkg/metrics/parser_test.go @@ -667,6 +667,32 @@ func TestParser_parseMetric(t *testing.T) { Value: 11.0, // 600 watts for 1 minute = 10 Wh }, }, + { + name: "raw expression, step 1", + fields: fields{ + map[string][]*config.MetricConfig{ + "apower": { + { + PrometheusName: "total_energy", + ValueType: "gauge", + OmitTimestamp: true, + RawExpression: `float(join(filter(split(string(raw_value), ""), { # matches "^[0-9\\.]$" }), ""))`, + }, + }, + }, + }, + elapseNow: 3 * time.Minute, + args: args{ + metricPath: "apower", + deviceID: "shellyplus1pm-foo", + value: "H42Jj.j44", + }, + want: Metric{ + Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), + ValueType: prometheus.GaugeValue, + Value: 42.44, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -797,7 +823,7 @@ func TestParser_evalExpression(t *testing.T) { p := NewParser(nil, ".", stateDir) for i, value := range tt.values { - got, err := p.evalExpression(id, tt.expression, value) + got, err := p.evalExpressionValue(id, tt.expression, value, value) want := tt.results[i] if err != nil { t.Errorf("evaluating the %dth value '%v' failed: %v", i, value, err)