diff --git a/varnish-mixin/alerts/alerts.libsonnet b/varnish-mixin/alerts.libsonnet similarity index 71% rename from varnish-mixin/alerts/alerts.libsonnet rename to varnish-mixin/alerts.libsonnet index 52c651ff9..8fd671154 100644 --- a/varnish-mixin/alerts/alerts.libsonnet +++ b/varnish-mixin/alerts.libsonnet @@ -1,14 +1,16 @@ { - prometheusAlerts+:: { - groups+: [ + new(this): { + local config = this.config, + + groups: [ { name: 'varnish-cache', rules: [ { alert: 'VarnishCacheLowCacheHitRate', expr: ||| - increase(varnish_main_cache_hit[10m]) / (clamp_min((increase(varnish_main_cache_hit[10m]) + increase(varnish_main_cache_miss[10m])), 1)) * 100 < %(alertsWarningCacheHitRate)s and (increase(varnish_main_cache_hit[10m]) + increase(varnish_main_cache_miss[10m]) > 0) - ||| % $._config, + increase(varnish_main_cache_hit{%(filteringSelector)s}[10m]) / (clamp_min((increase(varnish_main_cache_hit{%(filteringSelector)s}[10m]) + increase(varnish_main_cache_miss{%(filteringSelector)s}[10m])), 1)) * 100 < %(alertsWarningCacheHitRate)s and (increase(varnish_main_cache_hit{%(filteringSelector)s}[10m]) + increase(varnish_main_cache_miss{%(filteringSelector)s}[10m]) > 0) + ||| % config, 'for': '10m', labels: { severity: 'warning', @@ -19,14 +21,14 @@ ( 'The Cache hit rate is {{ printf "%%.0f" $value }} percent over the last 5 minutes on {{$labels.instance}}, ' + 'which is below the threshold of %(alertsWarningCacheHitRate)s percent.' - ) % $._config, + ) % config, }, }, { alert: 'VarnishCacheHighMemoryUsage', expr: ||| - (varnish_sma_g_bytes{type="s0"} / (varnish_sma_g_bytes{type="s0"} + varnish_sma_g_space{type="s0"})) * 100 > %(alertsWarningHighMemoryUsage)s - ||| % $._config, + (varnish_sma_g_bytes{%(filteringSelector)s,type="s0"} / (varnish_sma_g_bytes{%(filteringSelector)s,type="s0"} + varnish_sma_g_space{%(filteringSelector)s,type="s0"})) * 100 > %(alertsWarningHighMemoryUsage)s + ||| % config, 'for': '5m', labels: { severity: 'warning', @@ -37,14 +39,14 @@ ( 'Current Memory Usage is {{ printf "%%.0f" $value }} percent on {{$labels.instance}}, ' + 'which is above the threshold of %(alertsWarningHighMemoryUsage)s percent.' - ) % $._config, + ) % config, }, }, { alert: 'VarnishCacheHighCacheEvictionRate', expr: ||| - increase(varnish_main_n_lru_nuked[5m]) > %(alertsCriticalCacheEviction)s - ||| % $._config, + increase(varnish_main_n_lru_nuked{%(filteringSelector)s}[5m]) > %(alertsCriticalCacheEviction)s + ||| % config, 'for': '5m', labels: { severity: 'critical', @@ -55,14 +57,14 @@ ( 'The Cache has evicted {{ printf "%%.0f" $value }} objects over the last 5 minutes on {{$labels.instance}}, ' + 'which is above the threshold of %(alertsCriticalCacheEviction)s.' - ) % $._config, + ) % config, }, }, { alert: 'VarnishCacheHighSaturation', expr: ||| - varnish_main_thread_queue_len > %(alertsWarningHighSaturation)s - ||| % $._config, + varnish_main_thread_queue_len{%(filteringSelector)s} > %(alertsWarningHighSaturation)s + ||| % config, 'for': '5m', labels: { severity: 'warning', @@ -73,14 +75,14 @@ ( 'The thread queue length is {{ printf "%%.0f" $value }} over the last 5 minutes on {{$labels.instance}}, ' + 'which is above the threshold of %(alertsWarningHighSaturation)s.' - ) % $._config, + ) % config, }, }, { alert: 'VarnishCacheSessionsDropping', expr: ||| - increase(varnish_main_sessions{type="dropped"}[5m]) > %(alertsCriticalSessionsDropped)s - ||| % $._config, + increase(varnish_main_sessions{%(filteringSelector)s,type="dropped"}[5m]) > %(alertsCriticalSessionsDropped)s + ||| % config, 'for': '5m', labels: { severity: 'critical', @@ -91,14 +93,14 @@ ( 'The amount of sessions dropped is {{ printf "%%.0f" $value }} over the last 5 minutes on {{$labels.instance}}, ' + 'which is above the threshold of %(alertsCriticalSessionsDropped)s.' - ) % $._config, + ) % config, }, }, { alert: 'VarnishCacheBackendUnhealthy', expr: ||| - increase(varnish_main_backend_unhealthy[5m]) > %(alertsCriticalBackendUnhealthy)s - ||| % $._config, + increase(varnish_main_backend_unhealthy{%(filteringSelector)s}[5m]) > %(alertsCriticalBackendUnhealthy)s + ||| % config, 'for': '5m', labels: { severity: 'critical', @@ -109,7 +111,7 @@ ( 'The amount of unhealthy backend statuses detected is {{ printf "%%.0f" $value }} over the last 5 minutes on {{$labels.instance}}, ' + 'which is above the threshold of %(alertsCriticalBackendUnhealthy)s.' - ) % $._config, + ) % config, }, }, ], diff --git a/varnish-mixin/config.libsonnet b/varnish-mixin/config.libsonnet index 4db1b734e..11ef8b0b3 100644 --- a/varnish-mixin/config.libsonnet +++ b/varnish-mixin/config.libsonnet @@ -1,20 +1,43 @@ { - _config+:: { - dashboardTags: ['varnish-cache-mixin'], - dashboardPeriod: 'now-1h', - dashboardTimezone: 'default', - dashboardRefresh: '1m', + local this = self, + filteringSelector: 'job="integrations/varnish-cache"', + groupLabels: ['job', 'cluster'], + instanceLabels: ['instance'], + dashboardTags: ['varnish-mixin'], + uid: 'varnish', + dashboardNamePrefix: 'Varnish', - //alert thresholds - alertsWarningCacheHitRate: 80, //% - alertsWarningHighMemoryUsage: 90, //% - alertsCriticalCacheEviction: 0, - alertsWarningHighSaturation: 0, - alertsCriticalSessionsDropped: 0, - alertsCriticalBackendUnhealthy: 0, - enableLokiLogs: true, - enableMultiCluster: false, - multiclusterSelector: 'job=~"$job"', - varnishSelector: if self.enableMultiCluster then 'job=~"$job", cluster=~"$cluster"' else 'job=~"$job"', + // additional params + dashboardPeriod: 'now-1h', + dashboardTimezone: 'default', + dashboardRefresh: '1m', + + // logs lib related + enableLokiLogs: true, + logLabels: ['job', 'instance', 'cluster', 'level'], + extraLogLabels: [], // Required by logs-lib + logsVolumeGroupBy: 'level', + showLogsVolume: true, + + // alert thresholds + alertsWarningCacheHitRate: 80, //% + alertsWarningHighMemoryUsage: 90, //% + alertsCriticalCacheEviction: 0, + alertsWarningHighSaturation: 0, + alertsCriticalSessionsDropped: 0, + alertsCriticalBackendUnhealthy: 0, + + // metrics source for signals library + metricsSource: 'prometheus', + + legendCustomTemplate: std.join(' ', std.map(function(label) '{{' + label + '}}', this.instanceLabels)), + signals+: { + cache: (import './signals/cache.libsonnet')(this), + requests: (import './signals/requests.libsonnet')(this), + sessions: (import './signals/sessions.libsonnet')(this), + memory: (import './signals/memory.libsonnet')(this), + network: (import './signals/network.libsonnet')(this), + threads: (import './signals/threads.libsonnet')(this), + backend: (import './signals/backend.libsonnet')(this), }, } diff --git a/varnish-mixin/dashboards.libsonnet b/varnish-mixin/dashboards.libsonnet new file mode 100644 index 000000000..299dfd694 --- /dev/null +++ b/varnish-mixin/dashboards.libsonnet @@ -0,0 +1,88 @@ +local g = import './g.libsonnet'; +local logslib = import 'logs-lib/logs/main.libsonnet'; +{ + local root = self, + new(this):: + local prefix = this.config.dashboardNamePrefix; + local links = this.grafana.links; + local tags = this.config.dashboardTags; + local uid = g.util.string.slugify(this.config.uid); + local vars = this.grafana.variables; + local annotations = this.grafana.annotations; + local refresh = this.config.dashboardRefresh; + local period = this.config.dashboardPeriod; + local timezone = this.config.dashboardTimezone; + + { + 'varnish-overview.json': + g.dashboard.new(prefix + ' overview') + + g.dashboard.withPanels( + g.util.panel.resolveCollapsedFlagOnRows( + g.util.grid.wrapPanels( + [ + this.grafana.rows.varnishStats, + this.grafana.rows.varnishOverview, + ] + ) + ) + ) + + root.applyCommon( + vars.multiInstance, + uid + '_overview', + tags, + links { varnishOverview+:: {} }, + annotations, + timezone, + refresh, + period + ), + + } + + + if this.config.enableLokiLogs then + { + 'varnish-logs.json': + logslib.new( + prefix + ' logs', + datasourceName=this.grafana.variables.datasources.loki.name, + datasourceRegex=this.grafana.variables.datasources.loki.regex, + filterSelector=this.config.filteringSelector, + labels=this.config.groupLabels + this.config.extraLogLabels, + formatParser=null, + showLogsVolume=this.config.showLogsVolume, + ) + { + dashboards+: + { + logs+: + // reference to self, already generated variables, to keep them, but apply other common data in applyCommon + root.applyCommon(super.logs.templating.list, uid=uid + '-logs', tags=tags, links=links { logs+:: {} }, annotations=annotations, timezone=timezone, refresh=refresh, period=period), + }, + panels+: + { + // modify log panel + logs+: + g.panel.logs.options.withEnableLogDetails(true) + + g.panel.logs.options.withShowTime(false) + + g.panel.logs.options.withWrapLogMessage(false), + }, + variables+: { + // add prometheus datasource for annotations processing + toArray+: [ + this.grafana.variables.datasources.prometheus { hide: 2 }, + ], + }, + }.dashboards.logs, + } + else {}, + + applyCommon(vars, uid, tags, links, annotations, timezone, refresh, period): + g.dashboard.withTags(tags) + + g.dashboard.withUid(uid) + + g.dashboard.withLinks(std.objectValues(links)) + + g.dashboard.withTimezone(timezone) + + g.dashboard.withRefresh(refresh) + + g.dashboard.time.withFrom(period) + + g.dashboard.withVariables(vars) + + g.dashboard.withAnnotations(std.objectValues(annotations)), +} diff --git a/varnish-mixin/dashboards/varnish-overview.libsonnet b/varnish-mixin/dashboards/varnish-overview.libsonnet deleted file mode 100644 index 36124b10b..000000000 --- a/varnish-mixin/dashboards/varnish-overview.libsonnet +++ /dev/null @@ -1,1328 +0,0 @@ -local g = (import 'grafana-builder/grafana.libsonnet'); -local grafana = (import 'grafonnet/grafana.libsonnet'); -local dashboard = grafana.dashboard; -local template = grafana.template; -local prometheus = grafana.prometheus; - -local dashboardUid = 'varnish-overview'; - -local promDatasourceName = 'prometheus_datasource'; -local lokiDatasourceName = 'loki_datasource'; - -local getMatcher(cfg) = '%(varnishSelector)s, instance=~"$instance"' % cfg; -local frontendLogFilter = 'filename=~"/var/log/varnish/varnishncsa-frontend.*.log|/opt/varnish/log/varnishncsa-frontend.*.log"'; -local backendLogFilter = 'filename=~"/var/log/varnish/varnishncsa-backend.*.log|/opt/varnish/log/varnishncsa-backend.*.log"'; - -local promDatasource = { - uid: '${%s}' % promDatasourceName, -}; - -local lokiDatasource = { - uid: '${%s}' % lokiDatasourceName, -}; - -local cacheHitRatePanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'avg((rate(varnish_main_cache_hit{' + matcher + '}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{' + matcher + '}[$__rate_interval]) + rate(varnish_main_cache_miss{' + matcher + '}[$__rate_interval]), 1))) * 100\n', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'gauge', - title: 'Cache hit rate', - description: 'Rate of cache hits to misses.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - { - color: 'red', - value: 0, - }, - { - color: '#EAB839', - value: 50, - }, - { - color: 'green', - value: 80, - }, - ], - }, - unit: 'percent', - }, - overrides: [], - }, - options: { - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - showThresholdLabels: false, - showThresholdMarkers: true, - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local frontendRequestsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_client_req{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{nstance}}', - ), - ], - type: 'stat', - title: 'Frontend requests', - description: 'The rate of requests sent to the Varnish Cache frontend.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'reqps', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local backendRequestsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_backend_req{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'stat', - title: 'Backend requests', - description: 'The rate of requests sent to the Varnish Cache backends.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'reqps', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local sessionsRatePanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_sessions_total{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'stat', - title: 'Sessions rate', - description: 'The rate of total sessions created in the Varnish Cache instance.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: '/ sec', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local cacheHitsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_cache_hit{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'stat', - title: 'Cache hits', - description: 'The rate of cache hits.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: '/ sec', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local cacheHitPassPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_cache_hitpass{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'stat', - title: 'Cache hit pass', - description: 'Rate of cache hits for pass objects (fulfilled requests that are not cached).', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: '/ sec', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local sessionQueueLengthPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'varnish_main_thread_queue_len{' + matcher + '}', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'stat', - title: 'Session queue length', - description: 'Length of session queue waiting for threads.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'none', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local poolsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'varnish_main_pools{' + matcher + '}', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'stat', - title: 'Pools', - description: 'Number of thread pools.', - fieldConfig: { - defaults: { - color: { - mode: 'thresholds', - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'none', - }, - overrides: [], - }, - options: { - colorMode: 'value', - graphMode: 'none', - justifyMode: 'auto', - orientation: 'auto', - reduceOptions: { - calcs: [ - 'lastNotNull', - ], - fields: '', - values: false, - }, - textMode: 'auto', - }, - pluginVersion: '10.0.3-cloud.4.aed62623', - transparent: true, -}; - -local backendConnectionsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_backend_conn{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Accepted', - ), - prometheus.target( - 'irate(varnish_main_backend_recycle{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Recycled', - ), - prometheus.target( - 'irate(varnish_main_backend_reuse{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Reused', - ), - prometheus.target( - 'irate(varnish_main_backend_busy{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Busy', - ), - prometheus.target( - 'irate(varnish_main_backend_unhealthy{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Unhealthy', - ), - ], - type: 'timeseries', - title: 'Backend connections', - description: 'Rate of recycled, reused, busy, unhealthy, and accepted backend connections by Varnish Cache.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineStyle: { - fill: 'solid', - }, - lineWidth: 1, - pointSize: 1, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'normal', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'conn/s', - }, - overrides: [], - }, - options: { - legend: { - calcs: [], - displayMode: 'list', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'multi', - sort: 'none', - }, - }, -}; - -local sessionsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_sessions{' + matcher + ',type="conn"}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Connected', - ), - prometheus.target( - 'irate(varnish_main_sessions{' + matcher + ',type="queued"}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Queued', - ), - prometheus.target( - 'irate(varnish_main_sessions{' + matcher + ',type="dropped"}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Dropped', - ), - ], - type: 'timeseries', - title: 'Sessions', - description: 'Rate of new connected, queued, and dropped sessions.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'none', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'sess/s', - }, - overrides: [], - }, - options: { - legend: { - calcs: [], - displayMode: 'list', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'multi', - sort: 'none', - }, - }, -}; - -local requestsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_client_req{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Frontend', - ), - prometheus.target( - 'irate(varnish_main_backend_req{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Backend', - ), - ], - type: 'timeseries', - title: 'Requests', - description: 'Rate of frontend and backend requests received.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'none', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'reqps', - }, - overrides: [], - }, - options: { - legend: { - calcs: [], - displayMode: 'list', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'multi', - sort: 'none', - }, - }, -}; - -local cacheHitRatioPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'avg by (instance, job) ((rate(varnish_main_cache_hit{' + matcher + '}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{' + matcher + '}[$__rate_interval]) + rate(varnish_main_cache_miss{' + matcher + '}[$__rate_interval]), 1))) * 100', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'timeseries', - title: 'Cache hit ratio', - description: 'Ratio of cache hits to cache misses.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - axisSoftMax: 100, - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'normal', - }, - thresholdsStyle: { - mode: 'area', - }, - }, - mappings: [], - thresholds: { - mode: 'percentage', - steps: [ - { - color: 'transparent', - value: null, - }, - { - color: 'red', - value: 0, - }, - { - color: 'yellow', - value: 50, - }, - { - color: 'green', - value: 80, - }, - ], - }, - unit: 'percent', - }, - overrides: [], - }, - options: { - legend: { - calcs: [], - displayMode: 'list', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'single', - sort: 'none', - }, - }, -}; - -local memoryUsedPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'varnish_sma_g_bytes{' + matcher + ',type="s0"}', - datasource=promDatasource, - legendFormat='{{instance}}', - ), - ], - type: 'timeseries', - title: 'Memory used', - description: 'Bytes allocated from Varnish Cache storage.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'none', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'decbytes', - }, - overrides: [], - }, - options: { - legend: { - calcs: [ - 'min', - 'mean', - 'max', - ], - displayMode: 'table', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'single', - sort: 'none', - }, - }, -}; - -local cacheEventsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_n_expired{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Expired', - ), - prometheus.target( - 'irate(varnish_main_n_lru_nuked{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Nuked', - ), - ], - type: 'timeseries', - title: 'Cache events', - description: 'Rate of expired and LRU (least recently used) nuked objects.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'normal', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'ops', - }, - overrides: [], - }, - options: { - legend: { - calcs: [ - 'min', - 'mean', - 'max', - ], - displayMode: 'table', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'multi', - sort: 'none', - }, - }, -}; - -local networkPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'irate(varnish_main_s_resp_hdrbytes{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Frontend header', - ), - prometheus.target( - 'irate(varnish_main_s_resp_bodybytes{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - Frontend body', - ), - prometheus.target( - 'irate(varnish_backend_beresp_hdrbytes{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - {{backend}} - Backend header', - ), - prometheus.target( - 'irate(varnish_backend_beresp_bodybytes{' + matcher + '}[$__rate_interval])', - datasource=promDatasource, - legendFormat='{{instance}} - {{backend}} - Backend body', - ), - ], - type: 'timeseries', - title: 'Network', - description: 'Rate for the response bytes of header and body for frontend and backends.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 20, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'normal', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'decbytes', - }, - overrides: [], - }, - options: { - legend: { - calcs: [], - displayMode: 'list', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'multi', - sort: 'none', - }, - }, -}; - -local threadsPanel(matcher) = { - datasource: promDatasource, - targets: [ - prometheus.target( - 'increase(varnish_main_threads_failed{' + matcher + '}[$__interval:])', - datasource=promDatasource, - legendFormat='{{instance}} - Failed', - format='time_series', - interval='1m', - ), - prometheus.target( - 'increase(varnish_main_threads_created{' + matcher + '}[$__interval:])', - datasource=promDatasource, - legendFormat='{{instance}} - Created', - format='time_series', - interval='1m', - ), - prometheus.target( - 'increase(varnish_main_threads_limited{' + matcher + '}[$__interval:])', - datasource=promDatasource, - legendFormat='{{instance}} - Limited', - format='time_series', - interval='1m', - ), - prometheus.target( - 'varnish_main_threads{' + matcher + '}', - datasource=promDatasource, - legendFormat='{{instance}} - Total', - ), - ], - type: 'timeseries', - title: 'Threads', - description: 'Number of failed, created, limited, and current threads.', - fieldConfig: { - defaults: { - color: { - mode: 'palette-classic', - }, - custom: { - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - drawStyle: 'line', - fillOpacity: 0, - gradientMode: 'none', - hideFrom: { - legend: false, - tooltip: false, - viz: false, - }, - lineInterpolation: 'linear', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { - type: 'linear', - }, - showPoints: 'never', - spanNulls: false, - stacking: { - group: 'A', - mode: 'none', - }, - thresholdsStyle: { - mode: 'off', - }, - }, - mappings: [], - thresholds: { - mode: 'absolute', - steps: [ - { - color: 'green', - value: null, - }, - ], - }, - unit: 'none', - }, - overrides: [ - { - matcher: { - id: 'byFrameRefID', - options: 'A', - }, - properties: [ - { - id: 'custom.fillOpacity', - value: 20, - }, - ], - }, - { - matcher: { - id: 'byFrameRefID', - options: 'B', - }, - properties: [ - { - id: 'custom.fillOpacity', - value: 20, - }, - ], - }, - { - matcher: { - id: 'byFrameRefID', - options: 'C', - }, - properties: [ - { - id: 'custom.fillOpacity', - value: 20, - }, - ], - }, - ], - }, - options: { - legend: { - calcs: [], - displayMode: 'list', - placement: 'right', - showLegend: true, - }, - tooltip: { - mode: 'multi', - sort: 'none', - }, - }, -}; - -local logsRow = { - datasource: promDatasource, - targets: [], - type: 'row', - title: 'Logs', - collapsed: false, -}; - -local frontendLogsPanel(matcher) = { - datasource: lokiDatasource, - targets: [ - { - datasource: lokiDatasource, - editorMode: 'code', - expr: '{' + matcher + ', ' + frontendLogFilter + '}', - queryType: 'range', - refId: 'A', - }, - ], - type: 'logs', - title: 'Frontend logs', - description: 'Client logs for Varnish Cache.', - options: { - dedupStrategy: 'none', - enableLogDetails: true, - prettifyLogMessage: false, - showCommonLabels: false, - showLabels: false, - showTime: false, - sortOrder: 'Descending', - wrapLogMessage: false, - }, -}; - -local backendLogsPanel(matcher) = { - datasource: lokiDatasource, - targets: [ - { - datasource: lokiDatasource, - editorMode: 'code', - expr: '{' + matcher + ', ' + backendLogFilter + '}', - queryType: 'range', - refId: 'A', - }, - ], - type: 'logs', - title: 'Backend logs', - description: 'Backend logs for Varnish Cache.', - options: { - dedupStrategy: 'none', - enableLogDetails: true, - prettifyLogMessage: false, - showCommonLabels: false, - showLabels: false, - showTime: false, - sortOrder: 'Descending', - wrapLogMessage: false, - }, -}; - -{ - grafanaDashboards+:: { - 'varnish-overview.json': - dashboard.new( - 'Varnish overview', - time_from='%s' % $._config.dashboardPeriod, - tags=($._config.dashboardTags), - timezone='%s' % $._config.dashboardTimezone, - refresh='%s' % $._config.dashboardRefresh, - description='', - uid=dashboardUid, - ) - - .addTemplates( - std.flattenArrays([ - [ - template.datasource( - promDatasourceName, - 'prometheus', - null, - label='Data Source', - refresh='load' - ), - ], - if $._config.enableLokiLogs then [ - template.datasource( - lokiDatasourceName, - 'loki', - null, - label='Loki Datasource', - refresh='load' - ), - ] else [], - [ - template.new( - 'job', - promDatasource, - 'label_values(varnish_main_sessions,job)', - label='Job', - refresh=2, - includeAll=true, - multi=true, - allValues='.+', - sort=0 - ), - template.new( - 'cluster', - promDatasource, - 'label_values(varnish_main_sessions{%(multiclusterSelector)s}, cluster)' % $._config, - label='Cluster', - refresh=2, - includeAll=true, - multi=true, - allValues='.*', - hide=if $._config.enableMultiCluster then '' else 'variable', - sort=0 - ), - template.new( - 'instance', - promDatasource, - 'label_values(varnish_main_sessions{%(varnishSelector)s},instance)' % $._config, - label='Instance', - refresh=2, - includeAll=true, - multi=true, - allValues='.+', - sort=0 - ), - ], - ]) - ) - .addPanels( - std.flattenArrays([ - [ - cacheHitRatePanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 0, y: 0 } }, - frontendRequestsPanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 3, y: 0 } }, - backendRequestsPanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 6, y: 0 } }, - sessionsRatePanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 9, y: 0 } }, - cacheHitsPanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 12, y: 0 } }, - cacheHitPassPanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 15, y: 0 } }, - sessionQueueLengthPanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 18, y: 0 } }, - poolsPanel(getMatcher($._config)) { gridPos: { h: 4, w: 3, x: 21, y: 0 } }, - backendConnectionsPanel(getMatcher($._config)) { gridPos: { h: 8, w: 12, x: 0, y: 4 } }, - sessionsPanel(getMatcher($._config)) { gridPos: { h: 8, w: 12, x: 12, y: 4 } }, - requestsPanel(getMatcher($._config)) { gridPos: { h: 8, w: 12, x: 0, y: 12 } }, - cacheHitRatioPanel(getMatcher($._config)) { gridPos: { h: 8, w: 12, x: 12, y: 12 } }, - memoryUsedPanel(getMatcher($._config)) { gridPos: { h: 7, w: 12, x: 0, y: 20 } }, - cacheEventsPanel(getMatcher($._config)) { gridPos: { h: 7, w: 12, x: 12, y: 20 } }, - networkPanel(getMatcher($._config)) { gridPos: { h: 10, w: 12, x: 0, y: 27 } }, - threadsPanel(getMatcher($._config)) { gridPos: { h: 10, w: 12, x: 12, y: 27 } }, - logsRow { gridPos: { h: 1, w: 24, x: 0, y: 37 } }, - ], - if $._config.enableLokiLogs then [ - frontendLogsPanel(getMatcher($._config)) { gridPos: { h: 8, w: 24, x: 0, y: 38 } }, - ] else [], - [ - ], - if $._config.enableLokiLogs then [ - backendLogsPanel(getMatcher($._config)) { gridPos: { h: 7, w: 24, x: 0, y: 46 } }, - ] else [], - [ - ], - ]) - ), - }, -} diff --git a/varnish-mixin/dashboards_out/varnish-logs.json b/varnish-mixin/dashboards_out/varnish-logs.json new file mode 100644 index 000000000..4f6b30c34 --- /dev/null +++ b/varnish-mixin/dashboards_out/varnish-logs.json @@ -0,0 +1,288 @@ +{ + "annotations": { + "list": [ ] + }, + "links": [ + { + "targetBlank": true, + "title": "Varnish overview", + "type": "link", + "url": "/d/varnish_overview" + } + ], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "description": "Logs volume grouped by \"level\" label.", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "fillOpacity": 50, + "stacking": { + "mode": "normal" + } + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "(E|e)merg|(F|f)atal|(A|a)lert|(C|c)rit.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(E|e)(rr.*|RR.*)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(W|w)(arn.*|ARN.*|rn|RN)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(N|n)(otice|ote)|(I|i)(nf.*|NF.*)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "dbg.*|DBG.*|(D|d)(EBUG|ebug)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(T|t)(race|RACE)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "logs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "maxDataPoints": 100, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "sum by (level) (count_over_time({job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\"}\n|~ \"$regex_search\"\n\n[$__auto]))\n", + "legendFormat": "{{ level }}" + } + ], + "title": "Logs volume", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "logs" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 2, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": true, + "prettifyLogMessage": true, + "showTime": false, + "wrapLogMessage": false + }, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "expr": "{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\"} \n|~ \"$regex_search\"\n\n\n" + } + ], + "title": "Logs", + "type": "logs" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "varnish-mixin" + ], + "templating": { + "list": [ + { + "label": "Loki data source", + "name": "loki_datasource", + "query": "loki", + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "query": "label_values({job=\"integrations/varnish-cache\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "query": "label_values({job=\"integrations/varnish-cache\",job=~\"$job\"}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "label": "Regex search", + "name": "regex_search", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "hide": 2, + "label": "Prometheus data source", + "name": "prometheus_datasource", + "query": "prometheus", + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "default", + "title": "Varnish logs", + "uid": "varnish-logs" + } \ No newline at end of file diff --git a/varnish-mixin/dashboards_out/varnish-overview.json b/varnish-mixin/dashboards_out/varnish-overview.json index bc4113ca6..4662975f9 100644 --- a/varnish-mixin/dashboards_out/varnish-overview.json +++ b/varnish-mixin/dashboards_out/varnish-overview.json @@ -1,35 +1,42 @@ { - "__inputs": [ ], - "__requires": [ ], "annotations": { "list": [ ] }, - "description": "", - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ ], + "links": [ + { + "targetBlank": true, + "title": "Varnish logs", + "type": "link", + "url": "/d/varnish-logs" + } + ], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 0, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "Stats", + "type": "row" + }, { "datasource": { - "uid": "${prometheus_datasource}" + "type": "datasource", + "uid": "-- Mixed --" }, "description": "Rate of cache hits to misses.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ ], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, { "color": "red", "value": 0 @@ -45,38 +52,32 @@ ] }, "unit": "percent" - }, - "overrides": [ ] + } }, "gridPos": { "h": 4, "w": 3, "x": 0, - "y": 0 + "y": 1 }, "id": 2, "options": { "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, "showThresholdLabels": false, "showThresholdMarkers": true }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.4.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "avg((rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]), 1))) * 100\n", + "expr": "avg((rate(varnish_main_cache_hit{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]), 1))) by () * 100", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Cache hit rate" } ], "title": "Cache hit rate", @@ -85,59 +86,42 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "The rate of requests sent to the Varnish Cache frontend.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, - "unit": "reqps" - }, - "overrides": [ ] + "unit": "req" + } }, "gridPos": { "h": 4, "w": 3, "x": 3, - "y": 0 + "y": 1 }, "id": 3, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_client_req{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(varnish_main_client_req{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{nstance}}" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}}", + "refId": "Frontend requests" } ], "title": "Frontend requests", @@ -146,59 +130,41 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "description": "The rate of requests sent to the Varnish Cache backends.", + "description": "The requests sent to the Varnish Cache backends over the last $__interval.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, - "unit": "reqps" - }, - "overrides": [ ] + "unit": "req" + } }, "gridPos": { "h": 4, "w": 3, "x": 6, - "y": 0 + "y": 1 }, "id": 4, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_req{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "sum by () (rate(varnish_main_backend_req{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Backend requests" } ], "title": "Backend requests", @@ -207,59 +173,41 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "The rate of total sessions created in the Varnish Cache instance.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, "unit": "/ sec" - }, - "overrides": [ ] + } }, "gridPos": { "h": 4, "w": 3, "x": 9, - "y": 0 + "y": 1 }, "id": 5, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "sum by () (irate(varnish_main_sessions_total{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Sessions rate" } ], "title": "Sessions rate", @@ -268,59 +216,41 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "The rate of cache hits.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, - "unit": "/ sec" - }, - "overrides": [ ] + "unit": "none" + } }, "gridPos": { "h": 4, "w": 3, "x": 12, - "y": 0 + "y": 1 }, "id": 6, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "sum by () (rate(varnish_main_cache_hit{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Cache hits" } ], "title": "Cache hits", @@ -329,59 +259,41 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Rate of cache hits for pass objects (fulfilled requests that are not cached).", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, - "unit": "/ sec" - }, - "overrides": [ ] + "unit": "none" + } }, "gridPos": { "h": 4, "w": 3, "x": 15, - "y": 0 + "y": 1 }, "id": 7, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_cache_hitpass{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "sum by () (rate(varnish_main_cache_hitpass{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Cache hit pass" } ], "title": "Cache hit pass", @@ -390,59 +302,41 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Length of session queue waiting for threads.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, "unit": "none" - }, - "overrides": [ ] + } }, "gridPos": { "h": 4, "w": 3, "x": 18, - "y": 0 + "y": 1 }, "id": 8, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "varnish_main_thread_queue_len{job=~\"$job\", instance=~\"$instance\"}", + "expr": "sum by () (varnish_main_thread_queue_len{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Session queue length" } ], "title": "Session queue length", @@ -451,186 +345,161 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Number of thread pools.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "fixedColor": "text", + "mode": "fixed" }, "unit": "none" - }, - "overrides": [ ] + } }, "gridPos": { "h": 4, "w": 3, "x": 21, - "y": 0 + "y": 1 }, "id": 9, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "graphMode": "none" }, - "pluginVersion": "10.0.3-cloud.4.aed62623", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "varnish_main_pools{job=~\"$job\", instance=~\"$instance\"}", + "expr": "sum by () (varnish_main_pools{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Thread pools" } ], "title": "Pools", "transparent": true, "type": "stat" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 0, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [ ], + "title": "Overview", + "type": "row" + }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Rate of recycled, reused, busy, unhealthy, and accepted backend connections by Varnish Cache.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", "spanNulls": false, "stacking": { - "group": "A", "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" } }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "conn/s" - }, - "overrides": [ ] + "unit": "conn" + } }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 6 }, - "id": 10, + "id": 11, "options": { "legend": { "calcs": [ ], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_conn{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_backend_conn{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Accepted" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Accepted", + "refId": "Backend connections accepted" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_recycle{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_backend_recycle{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Recycled" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Recycled", + "refId": "Backend connections recycled" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_reuse{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_backend_reuse{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Reused" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Reused", + "refId": "Backend connections reused" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_busy{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_backend_busy{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Busy" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Busy", + "refId": "Backend connections busy" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_unhealthy{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_backend_unhealthy{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Unhealthy" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Unhealthy", + "refId": "Backend connections unhealthy" } ], "title": "Backend connections", @@ -638,104 +507,78 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Rate of new connected, queued, and dropped sessions.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "spanNulls": false }, - "unit": "sess/s" - }, - "overrides": [ ] + "unit": "sess" + } }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 6 }, - "id": 11, + "id": 12, "options": { "legend": { "calcs": [ ], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions{job=~\"$job\", instance=~\"$instance\",type=\"conn\"}[$__rate_interval])", + "expr": "increase(varnish_main_sessions{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\",type=\"conn\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Connected" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Connected", + "refId": "Sessions connected" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions{job=~\"$job\", instance=~\"$instance\",type=\"queued\"}[$__rate_interval])", + "expr": "increase(varnish_main_sessions{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\",type=\"queued\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Queued" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Queued", + "refId": "Sessions queued" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions{job=~\"$job\", instance=~\"$instance\",type=\"dropped\"}[$__rate_interval])", + "expr": "increase(varnish_main_sessions{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\",type=\"dropped\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Dropped" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Dropped", + "refId": "Sessions dropped" } ], "title": "Sessions", @@ -743,95 +586,66 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Rate of frontend and backend requests received.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "spanNulls": false }, - "unit": "reqps" - }, - "overrides": [ ] + "unit": "req" + } }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 14 }, - "id": 12, + "id": 13, "options": { "legend": { "calcs": [ ], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_client_req{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_client_req{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Frontend" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Frontend", + "refId": "Frontend requests" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_req{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_backend_req{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Backend" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Backend", + "refId": "Backend requests" } ], "title": "Requests", @@ -839,46 +653,27 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Ratio of cache hits to cache misses.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 100, - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", "spanNulls": false, "stacking": { - "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "area" } }, - "mappings": [ ], + "max": 100, "thresholds": { "mode": "percentage", "steps": [ @@ -901,37 +696,38 @@ ] }, "unit": "percent" - }, - "overrides": [ ] + } }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 14 }, - "id": 13, + "id": 14, "options": { "legend": { "calcs": [ ], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "avg by (instance, job) ((rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]), 1))) * 100", + "expr": "avg by (instance, job) ((rate(varnish_main_cache_hit{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval]), 1))) * 100", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Cache hit ratio" } ], "title": "Cache hit ratio", @@ -939,65 +735,30 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Bytes allocated from Varnish Cache storage.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "spanNulls": false }, "unit": "decbytes" - }, - "overrides": [ ] + } }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 22 }, - "id": 14, + "id": 15, "options": { "legend": { "calcs": [ @@ -1006,23 +767,25 @@ "max" ], "displayMode": "table", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "varnish_sma_g_bytes{job=~\"$job\", instance=~\"$instance\",type=\"s0\"}", + "expr": "varnish_sma_g_bytes{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\",type=\"s0\"}", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}" + "instant": false, + "legendFormat": "{{instance}}", + "refId": "Memory used" } ], "title": "Memory used", @@ -1030,65 +793,33 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Rate of expired and LRU (least recently used) nuked objects.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", "spanNulls": false, "stacking": { - "group": "A", "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" } }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "ops" - }, - "overrides": [ ] + } }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 22 }, - "id": 15, + "id": 16, "options": { "legend": { "calcs": [ @@ -1097,32 +828,38 @@ "max" ], "displayMode": "table", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_n_expired{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_n_expired{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Expired" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Expired", + "refId": "Cache expired" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_n_lru_nuked{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_n_lru_nuked{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Nuked" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Nuked", + "refId": "Cache LRU nuked" } ], "title": "Cache events", @@ -1130,113 +867,89 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Rate for the response bytes of header and body for frontend and backends.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", "spanNulls": false, "stacking": { - "group": "A", "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" } }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "decbytes" - }, - "overrides": [ ] + } }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 30 }, - "id": 16, + "id": 17, "options": { "legend": { "calcs": [ ], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_s_resp_hdrbytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_s_resp_hdrbytes{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Frontend header" + "instant": false, + "legendFormat": "{{instance}} - Frontend header", + "refId": "Frontend response header bytes" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_s_resp_bodybytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(varnish_main_s_resp_bodybytes{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Frontend body" + "instant": false, + "legendFormat": "{{instance}} - Frontend body", + "refId": "Frontend response body bytes" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_backend_beresp_hdrbytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(varnish_backend_beresp_hdrbytes{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{backend}} - Backend header" + "instant": false, + "legendFormat": "{{instance}} - {{backend}} - Backend header", + "refId": "Backend response header bytes" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_backend_beresp_bodybytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(varnish_backend_beresp_bodybytes{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{backend}} - Backend body" + "instant": false, + "legendFormat": "{{instance}} - {{backend}} - Backend body", + "refId": "Backend response body bytes" } ], "title": "Network", @@ -1244,343 +957,161 @@ }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Number of failed, created, limited, and current threads.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "spanNulls": false }, "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - } - ] + } }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 30 }, - "id": 17, + "id": 18, "options": { "legend": { "calcs": [ ], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "right" }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "increase(varnish_main_threads_failed{job=~\"$job\", instance=~\"$instance\"}[$__interval:])", + "expr": "increase(varnish_main_threads_failed{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Failed" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Failed", + "refId": "Threads failed" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "increase(varnish_main_threads_created{job=~\"$job\", instance=~\"$instance\"}[$__interval:])", + "expr": "increase(varnish_main_threads_created{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Created" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Created", + "refId": "Threads created" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "increase(varnish_main_threads_limited{job=~\"$job\", instance=~\"$instance\"}[$__interval:])", + "expr": "increase(varnish_main_threads_limited{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}[$__interval:] offset -$__interval)", "format": "time_series", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Limited" + "instant": false, + "interval": "2m", + "legendFormat": "{{instance}} - Limited", + "refId": "Threads limited" }, { "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "expr": "varnish_main_threads{job=~\"$job\", instance=~\"$instance\"}", + "expr": "varnish_main_threads{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\",instance=~\"$instance\"}", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - Total" + "instant": false, + "legendFormat": "{{instance}} - Total", + "refId": "Threads total" } ], "title": "Threads", "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "${prometheus_datasource}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 18, - "targets": [ ], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "uid": "${loki_datasource}" - }, - "description": "Client logs for Varnish Cache.", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 19, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "uid": "${loki_datasource}" - }, - "editorMode": "code", - "expr": "{job=~\"$job\", instance=~\"$instance\", filename=~\"/var/log/varnish/varnishncsa-frontend.*.log|/opt/varnish/log/varnishncsa-frontend.*.log\"}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Frontend logs", - "type": "logs" - }, - { - "datasource": { - "uid": "${loki_datasource}" - }, - "description": "Backend logs for Varnish Cache.", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 20, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "uid": "${loki_datasource}" - }, - "editorMode": "code", - "expr": "{job=~\"$job\", instance=~\"$instance\", filename=~\"/var/log/varnish/varnishncsa-backend.*.log|/opt/varnish/log/varnishncsa-backend.*.log\"}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Backend logs", - "type": "logs" } ], "refresh": "1m", - "rows": [ ], - "schemaVersion": 14, - "style": "dark", + "schemaVersion": 39, "tags": [ - "varnish-cache-mixin" + "varnish-mixin" ], "templating": { "list": [ { - "current": { }, - "hide": 0, - "label": "Data Source", + "label": "Prometheus data source", "name": "prometheus_datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { }, - "hide": 0, - "label": "Loki Datasource", - "name": "loki_datasource", - "options": [ ], - "query": "loki", - "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": ".+", - "current": { }, "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "hide": 0, "includeAll": true, "label": "Job", "multi": true, "name": "job", - "options": [ ], - "query": "label_values(varnish_main_sessions,job)", + "query": "label_values(varnish_main_sessions{job=\"integrations/varnish-cache\"}, job)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "sort": 1, + "type": "query" }, { "allValue": ".*", - "current": { }, "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "hide": 2, "includeAll": true, "label": "Cluster", "multi": true, "name": "cluster", - "options": [ ], - "query": "label_values(varnish_main_sessions{job=~\"$job\"}, cluster)", + "query": "label_values(varnish_main_sessions{job=\"integrations/varnish-cache\",job=~\"$job\"}, cluster)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "sort": 1, + "type": "query" }, { "allValue": ".+", - "current": { }, "datasource": { + "type": "prometheus", "uid": "${prometheus_datasource}" }, - "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", - "options": [ ], - "query": "label_values(varnish_main_sessions{job=~\"$job\"},instance)", + "query": "label_values(varnish_main_sessions{job=\"integrations/varnish-cache\",job=~\"$job\",cluster=~\"$cluster\"}, instance)", "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "query": "loki", "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "datasource" } ] }, @@ -1588,33 +1119,7 @@ "from": "now-1h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, "timezone": "default", "title": "Varnish overview", - "uid": "varnish-overview", - "version": 0 + "uid": "varnish_overview" } \ No newline at end of file diff --git a/varnish-mixin/g.libsonnet b/varnish-mixin/g.libsonnet new file mode 100644 index 000000000..e6a2060ee --- /dev/null +++ b/varnish-mixin/g.libsonnet @@ -0,0 +1 @@ +import 'github.com/grafana/grafonnet/gen/grafonnet-v11.4.0/main.libsonnet' diff --git a/varnish-mixin/jsonnetfile.json b/varnish-mixin/jsonnetfile.json index 65cebf84b..ed5b224a1 100644 --- a/varnish-mixin/jsonnetfile.json +++ b/varnish-mixin/jsonnetfile.json @@ -9,7 +9,35 @@ } }, "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "common-lib" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "grafana-cloud-integration-utils" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "logs-lib" + } + }, + "version": "master" } ], "legacyImports": true -} + } + \ No newline at end of file diff --git a/varnish-mixin/links.libsonnet b/varnish-mixin/links.libsonnet new file mode 100644 index 000000000..67c8eb8fc --- /dev/null +++ b/varnish-mixin/links.libsonnet @@ -0,0 +1,13 @@ +local g = import './g.libsonnet'; + +{ + new(this):: { + varnishOverview: + g.dashboard.link.link.new('Varnish overview', '/d/' + this.grafana.dashboards['varnish-overview.json'].uid) + + g.dashboard.link.link.options.withTargetBlank(true), + } + if this.config.enableLokiLogs then { + logs: + g.dashboard.link.link.new('Varnish logs', '/d/' + this.grafana.dashboards['varnish-logs.json'].uid) + + g.dashboard.link.link.options.withTargetBlank(true), + } else {}, +} diff --git a/varnish-mixin/main.libsonnet b/varnish-mixin/main.libsonnet new file mode 100644 index 000000000..3add74f2c --- /dev/null +++ b/varnish-mixin/main.libsonnet @@ -0,0 +1,49 @@ +local alerts = import './alerts.libsonnet'; +local config = import './config.libsonnet'; +local dashboards = import './dashboards.libsonnet'; +local links = import './links.libsonnet'; +local panels = import './panels.libsonnet'; +local rows = import './rows.libsonnet'; +local commonlib = import 'common-lib/common/main.libsonnet'; + +{ + withConfigMixin(config): { + config+: config, + }, + + new(): { + + local this = self, + config: config, + + signals: + { + [sig]: commonlib.signals.unmarshallJsonMulti( + this.config.signals[sig], + type=this.config.metricsSource + ) + for sig in std.objectFields(this.config.signals) + }, + + grafana: { + variables: commonlib.variables.new( + filteringSelector=this.config.filteringSelector, + groupLabels=this.config.groupLabels, + instanceLabels=this.config.instanceLabels, + varMetric='varnish_main_sessions', + customAllValue='.+', + enableLokiLogs=this.config.enableLokiLogs, + ), + annotations: {}, + links: links.new(this), + panels: panels.new(this), + dashboards: dashboards.new(this), + rows: rows.new(this), + }, + + prometheus: { + alerts: alerts.new(this), + recordingRules: {}, + }, + }, +} diff --git a/varnish-mixin/mixin.libsonnet b/varnish-mixin/mixin.libsonnet index 2f1d2c281..3f2772582 100644 --- a/varnish-mixin/mixin.libsonnet +++ b/varnish-mixin/mixin.libsonnet @@ -1,3 +1,33 @@ -(import 'dashboards/varnish-overview.libsonnet') + -(import 'alerts/alerts.libsonnet') + -(import 'config.libsonnet') +local varnishlib = import './main.libsonnet'; +local config = (import './config.libsonnet'); +local util = import 'grafana-cloud-integration-utils/util.libsonnet'; + + +local varnish = + varnishlib.new() + + varnishlib.withConfigMixin( + { + filteringSelector: config.filteringSelector, + uid: config.uid, + enableLokiLogs: config.enableLokiLogs, + } + ); + +local label_patch = { + cluster+: { + allValue: '.*', + }, +}; + +// populate monitoring-mixin: +{ + grafanaDashboards+:: { + [fname]: + local dashboard = varnish.grafana.dashboards[fname]; + dashboard + util.patch_variables(dashboard, label_patch) + + for fname in std.objectFields(varnish.grafana.dashboards) + }, + prometheusAlerts+:: varnish.prometheus.alerts, + prometheusRules+:: varnish.prometheus.recordingRules, +} diff --git a/varnish-mixin/panels.libsonnet b/varnish-mixin/panels.libsonnet new file mode 100644 index 000000000..45825b5d7 --- /dev/null +++ b/varnish-mixin/panels.libsonnet @@ -0,0 +1,276 @@ +local g = import './g.libsonnet'; +local commonlib = import 'common-lib/common/main.libsonnet'; + +{ + new(this):: + { + local signals = this.signals, + local barGauge = g.panel.barGauge, + + // + // Overview Dashboard Panels + // + + // Cache hit rate gauge + cacheHitRatePanel: + g.panel.gauge.new('Cache hit rate') + + g.panel.gauge.queryOptions.withTargets([ + signals.cache.cacheHitRate.asTarget(), + ]) + + g.panel.gauge.standardOptions.withUnit('percent') + + g.panel.gauge.standardOptions.withMin(0) + + g.panel.gauge.standardOptions.withMax(100) + + g.panel.gauge.standardOptions.thresholds.withMode('absolute') + + g.panel.gauge.standardOptions.thresholds.withSteps([ + g.panel.gauge.standardOptions.threshold.step.withColor('red') + + g.panel.gauge.standardOptions.threshold.step.withValue(0), + g.panel.gauge.standardOptions.threshold.step.withColor('#EAB839') + + g.panel.gauge.standardOptions.threshold.step.withValue(50), + g.panel.gauge.standardOptions.threshold.step.withColor('green') + + g.panel.gauge.standardOptions.threshold.step.withValue(80), + ]) + + g.panel.gauge.options.withOrientation('auto') + + g.panel.gauge.options.withShowThresholdLabels(false) + + g.panel.gauge.options.withShowThresholdMarkers(true) + + g.panel.gauge.panelOptions.withDescription('Rate of cache hits to misses.') + + g.panel.gauge.panelOptions.withTransparent(true), + + // Frontend requests stat + frontendRequestsPanel: + commonlib.panels.generic.stat.base.new( + 'Frontend requests', + targets=[ + signals.requests.frontendRequests.asTarget() { interval: '2m' }, + ], + description='The rate of requests sent to the Varnish Cache frontend.' + ) + + g.panel.stat.standardOptions.withUnit('req') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Backend requests stat + backendRequestsPanel: + commonlib.panels.generic.stat.base.new( + 'Backend requests', + targets=[ + signals.requests.backendRequests.asTarget(), + ], + description='The requests sent to the Varnish Cache backends over the last $__interval.' + ) + + g.panel.stat.standardOptions.withUnit('req') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Sessions rate stat + sessionsRatePanel: + commonlib.panels.generic.stat.base.new( + 'Sessions rate', + targets=[ + signals.sessions.sessionsRate.asTarget(), + ], + description='The rate of total sessions created in the Varnish Cache instance.' + ) + + g.panel.stat.standardOptions.withUnit('/ sec') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Cache hits stat + cacheHitsPanel: + commonlib.panels.generic.stat.base.new( + 'Cache hits', + targets=[ + signals.cache.cacheHits.asTarget(), + ], + description='The rate of cache hits.' + ) + + g.panel.stat.standardOptions.withUnit('none') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Cache hit pass stat + cacheHitPassPanel: + commonlib.panels.generic.stat.base.new( + 'Cache hit pass', + targets=[ + signals.cache.cacheHitPass.asTarget(), + ], + description='Rate of cache hits for pass objects (fulfilled requests that are not cached).' + ) + + g.panel.stat.standardOptions.withUnit('none') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Session queue length stat + sessionQueueLengthPanel: + commonlib.panels.generic.stat.base.new( + 'Session queue length', + targets=[ + signals.sessions.sessionQueueLength.asTarget(), + ], + description='Length of session queue waiting for threads.' + ) + + g.panel.stat.standardOptions.withUnit('none') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Thread pools stat + poolsPanel: + commonlib.panels.generic.stat.base.new( + 'Pools', + targets=[ + signals.threads.threadPools.asTarget(), + ], + description='Number of thread pools.' + ) + + g.panel.stat.standardOptions.withUnit('none') + + g.panel.stat.panelOptions.withTransparent(true) + + g.panel.stat.options.withGraphMode('none'), + + // Backend connections timeseries + backendConnectionsPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Backend connections', + targets=[ + signals.backend.backendConnectionsAccepted.asTarget() { interval: '2m' }, + signals.backend.backendConnectionsRecycled.asTarget() { interval: '2m' }, + signals.backend.backendConnectionsReused.asTarget() { interval: '2m' }, + signals.backend.backendConnectionsBusy.asTarget() { interval: '2m' }, + signals.backend.backendConnectionsUnhealthy.asTarget() { interval: '2m' }, + ], + description='Rate of recycled, reused, busy, unhealthy, and accepted backend connections by Varnish Cache.' + ) + + g.panel.timeSeries.standardOptions.withUnit('conn') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.stacking.withMode('normal'), + + // Sessions timeseries + sessionsPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Sessions', + targets=[ + signals.sessions.sessionsConnected.asTarget() { interval: '2m' }, + signals.sessions.sessionsQueued.asTarget() { interval: '2m' }, + signals.sessions.sessionsDropped.asTarget() { interval: '2m' }, + ], + description='Rate of new connected, queued, and dropped sessions.' + ) + + g.panel.timeSeries.standardOptions.withUnit('sess') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false), + + // Requests timeseries + requestsPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Requests', + targets=[ + signals.requests.frontendRequestsTimeseries.asTarget() { interval: '2m' }, + signals.requests.backendRequestsTimeseries.asTarget() { interval: '2m' }, + ], + description='Rate of frontend and backend requests received.' + ) + + g.panel.timeSeries.standardOptions.withUnit('req') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false), + + // Cache hit ratio timeseries + cacheHitRatioPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Cache hit ratio', + targets=[ + signals.cache.cacheHitRatio.asTarget(), + ], + description='Ratio of cache hits to cache misses.' + ) + + g.panel.timeSeries.standardOptions.withUnit('percent') + + g.panel.timeSeries.standardOptions.withMax(100) + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false) + + g.panel.timeSeries.fieldConfig.defaults.custom.stacking.withMode('normal') + + g.panel.timeSeries.fieldConfig.defaults.custom.thresholdsStyle.withMode('area') + + g.panel.timeSeries.standardOptions.thresholds.withMode('percentage') + + g.panel.timeSeries.standardOptions.thresholds.withSteps([ + g.panel.timeSeries.thresholdStep.withColor('transparent') + + g.panel.timeSeries.thresholdStep.withValue(null), + g.panel.timeSeries.thresholdStep.withColor('red') + + g.panel.timeSeries.thresholdStep.withValue(0), + g.panel.timeSeries.thresholdStep.withColor('yellow') + + g.panel.timeSeries.thresholdStep.withValue(50), + g.panel.timeSeries.thresholdStep.withColor('green') + + g.panel.timeSeries.thresholdStep.withValue(80), + ]), + + // Memory used timeseries + memoryUsedPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Memory used', + targets=[ + signals.memory.memoryUsed.asTarget(), + ], + description='Bytes allocated from Varnish Cache storage.' + ) + + g.panel.timeSeries.standardOptions.withUnit('decbytes') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false) + + g.panel.timeSeries.options.legend.withCalcs(['min', 'mean', 'max']) + + g.panel.timeSeries.options.legend.withDisplayMode('table'), + + // Cache events timeseries + cacheEventsPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Cache events', + targets=[ + signals.cache.cacheExpired.asTarget() { interval: '2m' }, + signals.cache.cacheLruNuked.asTarget() { interval: '2m' }, + ], + description='Rate of expired and LRU (least recently used) nuked objects.' + ) + + g.panel.timeSeries.standardOptions.withUnit('ops') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false) + + g.panel.timeSeries.fieldConfig.defaults.custom.stacking.withMode('normal') + + g.panel.timeSeries.options.legend.withCalcs(['min', 'mean', 'max']) + + g.panel.timeSeries.options.legend.withDisplayMode('table'), + + // Network timeseries + networkPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Network', + targets=[ + signals.network.frontendResponseHeaderBytes.asTarget(), + signals.network.frontendResponseBodyBytes.asTarget(), + signals.network.backendResponseHeaderBytes.asTarget(), + signals.network.backendResponseBodyBytes.asTarget(), + ], + description='Rate for the response bytes of header and body for frontend and backends.' + ) + + g.panel.timeSeries.standardOptions.withUnit('decbytes') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(20) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false) + + g.panel.timeSeries.fieldConfig.defaults.custom.stacking.withMode('normal'), + + // Threads timeseries + threadsPanel: + commonlib.panels.generic.timeSeries.base.new( + 'Threads', + targets=[ + signals.threads.threadsFailed.asTarget() { interval: '2m' }, + signals.threads.threadsCreated.asTarget() { interval: '2m' }, + signals.threads.threadsLimited.asTarget() { interval: '2m' }, + signals.threads.threadsTotal.asTarget(), + ], + description='Number of failed, created, limited, and current threads.' + ) + + g.panel.timeSeries.standardOptions.withUnit('none') + + g.panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(0) + + g.panel.timeSeries.options.legend.withPlacement('right') + + g.panel.timeSeries.fieldConfig.defaults.custom.withSpanNulls(false), + }, +} diff --git a/varnish-mixin/prometheus_rules_out/prometheus_alerts.yaml b/varnish-mixin/prometheus_rules_out/prometheus_alerts.yaml index 8d6f470fd..0680e8d73 100644 --- a/varnish-mixin/prometheus_rules_out/prometheus_alerts.yaml +++ b/varnish-mixin/prometheus_rules_out/prometheus_alerts.yaml @@ -6,7 +6,7 @@ groups: description: The Cache hit rate is {{ printf "%.0f" $value }} percent over the last 5 minutes on {{$labels.instance}}, which is below the threshold of 80 percent. summary: Cache is not answering a sufficient percentage of read requests. expr: | - increase(varnish_main_cache_hit[10m]) / (clamp_min((increase(varnish_main_cache_hit[10m]) + increase(varnish_main_cache_miss[10m])), 1)) * 100 < 80 and (increase(varnish_main_cache_hit[10m]) + increase(varnish_main_cache_miss[10m]) > 0) + increase(varnish_main_cache_hit{job="integrations/varnish-cache"}[10m]) / (clamp_min((increase(varnish_main_cache_hit{job="integrations/varnish-cache"}[10m]) + increase(varnish_main_cache_miss{job="integrations/varnish-cache"}[10m])), 1)) * 100 < 80 and (increase(varnish_main_cache_hit{job="integrations/varnish-cache"}[10m]) + increase(varnish_main_cache_miss{job="integrations/varnish-cache"}[10m]) > 0) for: 10m labels: severity: warning @@ -15,7 +15,7 @@ groups: description: Current Memory Usage is {{ printf "%.0f" $value }} percent on {{$labels.instance}}, which is above the threshold of 90 percent. summary: Varnish Cache is running low on available memory. expr: | - (varnish_sma_g_bytes{type="s0"} / (varnish_sma_g_bytes{type="s0"} + varnish_sma_g_space{type="s0"})) * 100 > 90 + (varnish_sma_g_bytes{job="integrations/varnish-cache",type="s0"} / (varnish_sma_g_bytes{job="integrations/varnish-cache",type="s0"} + varnish_sma_g_space{job="integrations/varnish-cache",type="s0"})) * 100 > 90 for: 5m labels: severity: warning @@ -24,7 +24,7 @@ groups: description: The Cache has evicted {{ printf "%.0f" $value }} objects over the last 5 minutes on {{$labels.instance}}, which is above the threshold of 0. summary: The cache is evicting too many objects. expr: | - increase(varnish_main_n_lru_nuked[5m]) > 0 + increase(varnish_main_n_lru_nuked{job="integrations/varnish-cache"}[5m]) > 0 for: 5m labels: severity: critical @@ -33,7 +33,7 @@ groups: description: The thread queue length is {{ printf "%.0f" $value }} over the last 5 minutes on {{$labels.instance}}, which is above the threshold of 0. summary: There are too many threads in queue, Varnish is saturated and responses are slowed. expr: | - varnish_main_thread_queue_len > 0 + varnish_main_thread_queue_len{job="integrations/varnish-cache"} > 0 for: 5m labels: severity: warning @@ -42,7 +42,7 @@ groups: description: The amount of sessions dropped is {{ printf "%.0f" $value }} over the last 5 minutes on {{$labels.instance}}, which is above the threshold of 0. summary: Incoming requests are being dropped due to a lack of free worker threads. expr: | - increase(varnish_main_sessions{type="dropped"}[5m]) > 0 + increase(varnish_main_sessions{job="integrations/varnish-cache",type="dropped"}[5m]) > 0 for: 5m labels: severity: critical @@ -51,7 +51,7 @@ groups: description: The amount of unhealthy backend statuses detected is {{ printf "%.0f" $value }} over the last 5 minutes on {{$labels.instance}}, which is above the threshold of 0. summary: Backend has been marked as unhealthy due to slow 200 responses. expr: | - increase(varnish_main_backend_unhealthy[5m]) > 0 + increase(varnish_main_backend_unhealthy{job="integrations/varnish-cache"}[5m]) > 0 for: 5m labels: severity: critical diff --git a/varnish-mixin/rows.libsonnet b/varnish-mixin/rows.libsonnet new file mode 100644 index 000000000..adee6be13 --- /dev/null +++ b/varnish-mixin/rows.libsonnet @@ -0,0 +1,34 @@ +local g = import './g.libsonnet'; + +{ + new(this): + { + varnishOverview: + g.panel.row.new('Overview') + + g.panel.row.withCollapsed(false) + + g.panel.row.withPanels([ + this.grafana.panels.backendConnectionsPanel { gridPos+: { w: 12 } }, + this.grafana.panels.sessionsPanel { gridPos+: { w: 12 } }, + this.grafana.panels.requestsPanel { gridPos+: { w: 12 } }, + this.grafana.panels.cacheHitRatioPanel { gridPos+: { w: 12 } }, + this.grafana.panels.memoryUsedPanel { gridPos+: { w: 12 } }, + this.grafana.panels.cacheEventsPanel { gridPos+: { w: 12 } }, + this.grafana.panels.networkPanel { gridPos+: { w: 12 } }, + this.grafana.panels.threadsPanel { gridPos+: { w: 12 } }, + ]), + + varnishStats: + g.panel.row.new('Stats') + + g.panel.row.withCollapsed(false) + + g.panel.row.withPanels([ + this.grafana.panels.cacheHitRatePanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.frontendRequestsPanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.backendRequestsPanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.sessionsRatePanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.cacheHitsPanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.cacheHitPassPanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.sessionQueueLengthPanel { gridPos+: { w: 3, h: 4 } }, + this.grafana.panels.poolsPanel { gridPos+: { w: 3, h: 4 } }, + ]), + }, +} diff --git a/varnish-mixin/signals/backend.libsonnet b/varnish-mixin/signals/backend.libsonnet new file mode 100644 index 000000000..d98d8f3a4 --- /dev/null +++ b/varnish-mixin/signals/backend.libsonnet @@ -0,0 +1,96 @@ +function(this) + local legendCustomTemplate = this.legendCustomTemplate; + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: legendCustomTemplate, + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Backend connections accepted + backendConnectionsAccepted: { + name: 'Backend connections accepted', + nameShort: 'Accepted', + type: 'counter', + description: 'Rate of accepted backend connections.', + unit: 'conn', + sources: { + prometheus: { + expr: 'varnish_main_backend_conn{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Accepted', + }, + }, + }, + + // Backend connections recycled + backendConnectionsRecycled: { + name: 'Backend connections recycled', + nameShort: 'Recycled', + type: 'counter', + description: 'Rate of recycled backend connections.', + unit: 'conn', + sources: { + prometheus: { + expr: 'varnish_main_backend_recycle{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Recycled', + }, + }, + }, + + // Backend connections reused + backendConnectionsReused: { + name: 'Backend connections reused', + nameShort: 'Reused', + type: 'counter', + description: 'Rate of reused backend connections.', + unit: 'conn', + sources: { + prometheus: { + expr: 'varnish_main_backend_reuse{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Reused', + }, + }, + }, + + // Backend connections busy + backendConnectionsBusy: { + name: 'Backend connections busy', + nameShort: 'Busy', + type: 'counter', + description: 'Rate of busy backend connections.', + unit: 'conn', + sources: { + prometheus: { + expr: 'varnish_main_backend_busy{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Busy', + }, + }, + }, + + // Backend connections unhealthy + backendConnectionsUnhealthy: { + name: 'Backend connections unhealthy', + nameShort: 'Unhealthy', + type: 'counter', + description: 'Rate of unhealthy backend connections.', + unit: 'conn', + sources: { + prometheus: { + expr: 'varnish_main_backend_unhealthy{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Unhealthy', + }, + }, + }, + }, + } diff --git a/varnish-mixin/signals/cache.libsonnet b/varnish-mixin/signals/cache.libsonnet new file mode 100644 index 000000000..1b1c7ac55 --- /dev/null +++ b/varnish-mixin/signals/cache.libsonnet @@ -0,0 +1,104 @@ +function(this) + local legendCustomTemplate = this.legendCustomTemplate; + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: legendCustomTemplate, + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Cache hit rate as percentage + cacheHitRate: { + name: 'Cache hit rate', + nameShort: 'Hit rate', + type: 'raw', + description: 'Rate of cache hits to misses.', + unit: 'percent', + sources: { + prometheus: { + expr: 'avg((rate(varnish_main_cache_hit{%(queriesSelector)s}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{%(queriesSelector)s}[$__rate_interval]) + rate(varnish_main_cache_miss{%(queriesSelector)s}[$__rate_interval]), 1))) by () * 100', + }, + }, + }, + + // Cache hit ratio for timeseries + cacheHitRatio: { + name: 'Cache hit ratio', + nameShort: 'Hit ratio', + type: 'raw', + description: 'Ratio of cache hits to cache misses.', + unit: 'percent', + sources: { + prometheus: { + expr: 'avg by (instance, job) ((rate(varnish_main_cache_hit{%(queriesSelector)s}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{%(queriesSelector)s}[$__rate_interval]) + rate(varnish_main_cache_miss{%(queriesSelector)s}[$__rate_interval]), 1))) * 100', + }, + }, + }, + + // Cache hits rate + cacheHits: { + name: 'Cache hits', + nameShort: 'Hits', + type: 'raw', + description: 'The rate of cache hits.', + unit: '/ sec', + sources: { + prometheus: { + expr: 'sum by () (rate(varnish_main_cache_hit{%(queriesSelector)s}[$__rate_interval]))', + }, + }, + }, + + // Cache hit pass rate + cacheHitPass: { + name: 'Cache hit pass', + nameShort: 'Hit pass', + type: 'raw', + description: 'Rate of cache hits for pass objects (fulfilled requests that are not cached).', + unit: '/ sec', + sources: { + prometheus: { + expr: 'sum by () (rate(varnish_main_cache_hitpass{%(queriesSelector)s}[$__rate_interval]))', + }, + }, + }, + + // Cache events - expired objects + cacheExpired: { + name: 'Cache expired', + nameShort: 'Expired', + type: 'counter', + description: 'Rate of expired objects.', + unit: 'ops', + sources: { + prometheus: { + expr: 'varnish_main_n_expired{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Expired', + }, + }, + }, + + // Cache events - LRU nuked objects + cacheLruNuked: { + name: 'Cache LRU nuked', + nameShort: 'LRU nuked', + type: 'counter', + description: 'Rate of LRU (least recently used) nuked objects.', + unit: 'ops', + sources: { + prometheus: { + expr: 'varnish_main_n_lru_nuked{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Nuked', + }, + }, + }, + }, + } diff --git a/varnish-mixin/signals/memory.libsonnet b/varnish-mixin/signals/memory.libsonnet new file mode 100644 index 000000000..b04678fdb --- /dev/null +++ b/varnish-mixin/signals/memory.libsonnet @@ -0,0 +1,44 @@ +function(this) + local legendCustomTemplate = this.legendCustomTemplate; + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: legendCustomTemplate, + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Memory used + memoryUsed: { + name: 'Memory used', + nameShort: 'Memory', + type: 'raw', + description: 'Bytes allocated from Varnish Cache storage.', + unit: 'decbytes', + sources: { + prometheus: { + expr: 'varnish_sma_g_bytes{%(queriesSelector)s,type="s0"}', + }, + }, + }, + + // Memory usage percentage + memoryUsagePercent: { + name: 'Memory usage percentage', + nameShort: 'Memory %', + type: 'raw', + description: 'Percentage of memory usage.', + unit: 'percent', + sources: { + prometheus: { + expr: '(varnish_sma_g_bytes{%(queriesSelector)s,type="s0"} / (varnish_sma_g_bytes{%(queriesSelector)s,type="s0"} + varnish_sma_g_space{%(queriesSelector)s,type="s0"})) * 100', + }, + }, + }, + }, + } diff --git a/varnish-mixin/signals/network.libsonnet b/varnish-mixin/signals/network.libsonnet new file mode 100644 index 000000000..447b6b581 --- /dev/null +++ b/varnish-mixin/signals/network.libsonnet @@ -0,0 +1,78 @@ +function(this) + local legendCustomTemplate = this.legendCustomTemplate; + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: legendCustomTemplate, + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Frontend response header bytes + frontendResponseHeaderBytes: { + name: 'Frontend response header bytes', + nameShort: 'Frontend header', + type: 'counter', + description: 'Rate of response header bytes for frontend.', + unit: 'decbytes', + sources: { + prometheus: { + expr: 'varnish_main_s_resp_hdrbytes{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Frontend header', + }, + }, + }, + + // Frontend response body bytes + frontendResponseBodyBytes: { + name: 'Frontend response body bytes', + nameShort: 'Frontend body', + type: 'counter', + description: 'Rate of response body bytes for frontend.', + unit: 'decbytes', + sources: { + prometheus: { + expr: 'varnish_main_s_resp_bodybytes{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Frontend body', + }, + }, + }, + + // Backend response header bytes + backendResponseHeaderBytes: { + name: 'Backend response header bytes', + nameShort: 'Backend header', + type: 'counter', + description: 'Rate of response header bytes for backend.', + unit: 'decbytes/s', + sources: { + prometheus: { + expr: 'varnish_backend_beresp_hdrbytes{%(queriesSelector)s}', + legendCustomTemplate: legendCustomTemplate + ' - {{backend}} - Backend header', + }, + }, + }, + + // Backend response body bytes + backendResponseBodyBytes: { + name: 'Backend response body bytes', + nameShort: 'Backend body', + type: 'counter', + description: 'Rate of response body bytes for backend.', + unit: 'decbytes/s', + sources: { + prometheus: { + expr: 'varnish_backend_beresp_bodybytes{%(queriesSelector)s}', + legendCustomTemplate: legendCustomTemplate + ' - {{backend}} - Backend body', + }, + }, + }, + }, + } diff --git a/varnish-mixin/signals/requests.libsonnet b/varnish-mixin/signals/requests.libsonnet new file mode 100644 index 000000000..494bef6e0 --- /dev/null +++ b/varnish-mixin/signals/requests.libsonnet @@ -0,0 +1,76 @@ +function(this) + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: std.join(' ', std.map(function(label) '{{' + label + '}}', this.instanceLabels)), + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Frontend requests rate + frontendRequests: { + name: 'Frontend requests', + nameShort: 'Frontend req', + type: 'counter', + description: 'The delta of requests sent to the Varnish Cache frontend.', + unit: 'reqps', + sources: { + prometheus: { + expr: 'varnish_main_client_req{%(queriesSelector)s}', + legendCustomTemplate: '{{instance}}', + }, + }, + }, + + // Backend requests rate + backendRequests: { + name: 'Backend requests', + nameShort: 'Backend req', + type: 'raw', + description: 'The rate of requests sent to the Varnish Cache backends.', + unit: 'reqps', + sources: { + prometheus: { + expr: 'sum by () (rate(varnish_main_backend_req{%(queriesSelector)s}[$__rate_interval]))', + }, + }, + }, + + // Frontend requests for timeseries + frontendRequestsTimeseries: { + name: 'Frontend requests', + nameShort: 'Frontend', + type: 'counter', + description: 'Rate of frontend requests received.', + unit: 'reqps', + sources: { + prometheus: { + expr: 'varnish_main_client_req{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: '{{instance}} - Frontend', + }, + }, + }, + + // Backend requests for timeseries + backendRequestsTimeseries: { + name: 'Backend requests', + nameShort: 'Backend', + type: 'counter', + description: 'Rate of backend requests received.', + unit: 'reqps', + sources: { + prometheus: { + expr: 'varnish_main_backend_req{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: '{{instance}} - Backend', + }, + }, + }, + }, + } diff --git a/varnish-mixin/signals/sessions.libsonnet b/varnish-mixin/signals/sessions.libsonnet new file mode 100644 index 000000000..7ca73bd4a --- /dev/null +++ b/varnish-mixin/signals/sessions.libsonnet @@ -0,0 +1,93 @@ +function(this) + local legendCustomTemplate = this.legendCustomTemplate; + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: legendCustomTemplate, + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Sessions rate + sessionsRate: { + name: 'Sessions rate', + nameShort: 'Sessions', + type: 'raw', + description: 'The rate of total sessions created in the Varnish Cache instance.', + unit: 'sess/s', + sources: { + prometheus: { + expr: 'sum by () (irate(varnish_main_sessions_total{%(queriesSelector)s}[$__rate_interval]))', + }, + }, + }, + + // Sessions connected + sessionsConnected: { + name: 'Sessions connected', + nameShort: 'Connected', + type: 'counter', + description: 'Rate of new connected sessions.', + unit: 'sess', + sources: { + prometheus: { + expr: 'varnish_main_sessions{%(queriesSelector)s,type="conn"}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Connected', + }, + }, + }, + + // Sessions queued + sessionsQueued: { + name: 'Sessions queued', + nameShort: 'Queued', + type: 'counter', + description: 'Rate of queued sessions.', + unit: 'sess', + sources: { + prometheus: { + expr: 'varnish_main_sessions{%(queriesSelector)s,type="queued"}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Queued', + }, + }, + }, + + // Sessions dropped + sessionsDropped: { + name: 'Sessions dropped', + nameShort: 'Dropped', + type: 'counter', + description: 'Rate of dropped sessions.', + unit: 'sess', + sources: { + prometheus: { + expr: 'varnish_main_sessions{%(queriesSelector)s,type="dropped"}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Dropped', + }, + }, + }, + + // Session queue length + sessionQueueLength: { + name: 'Session queue length', + nameShort: 'Queue length', + type: 'raw', + description: 'Length of session queue waiting for threads.', + unit: 'none', + sources: { + prometheus: { + expr: 'sum by () (varnish_main_thread_queue_len{%(queriesSelector)s})', + legendCustomTemplate: legendCustomTemplate, + }, + }, + }, + }, + } diff --git a/varnish-mixin/signals/threads.libsonnet b/varnish-mixin/signals/threads.libsonnet new file mode 100644 index 000000000..580b5cb3e --- /dev/null +++ b/varnish-mixin/signals/threads.libsonnet @@ -0,0 +1,93 @@ +function(this) + local legendCustomTemplate = this.legendCustomTemplate; + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + legendCustomTemplate: legendCustomTemplate, + aggLevel: 'none', + aggFunction: 'avg', + alertsInterval: '2m', + discoveryMetric: { + prometheus: 'varnish_main_sessions', + }, + signals: { + // Thread pools + threadPools: { + name: 'Thread pools', + nameShort: 'Pools', + type: 'raw', + description: 'Number of thread pools.', + unit: 'none', + sources: { + prometheus: { + expr: 'sum by () (varnish_main_pools{%(queriesSelector)s})', + }, + }, + }, + + // Threads failed + threadsFailed: { + name: 'Threads failed', + nameShort: 'Failed', + type: 'counter', + description: 'Number of failed threads.', + unit: 'none', + sources: { + prometheus: { + expr: 'varnish_main_threads_failed{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Failed', + }, + }, + }, + + // Threads created + threadsCreated: { + name: 'Threads created', + nameShort: 'Created', + type: 'counter', + description: 'Number of created threads.', + unit: 'none', + sources: { + prometheus: { + expr: 'varnish_main_threads_created{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Created', + }, + }, + }, + + // Threads limited + threadsLimited: { + name: 'Threads limited', + nameShort: 'Limited', + type: 'counter', + description: 'Number of limited threads.', + unit: 'none', + sources: { + prometheus: { + expr: 'varnish_main_threads_limited{%(queriesSelector)s}', + rangeFunction: 'increase', + legendCustomTemplate: legendCustomTemplate + ' - Limited', + }, + }, + }, + + // Threads total + threadsTotal: { + name: 'Threads total', + nameShort: 'Total', + type: 'gauge', + description: 'Total number of current threads.', + unit: 'none', + sources: { + prometheus: { + expr: 'varnish_main_threads{%(queriesSelector)s}', + legendCustomTemplate: legendCustomTemplate + ' - Total', + }, + }, + }, + }, + }