From 7d6c15c4e58e220a19ead1a0c7a5e7c40841ba1a Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 20 Sep 2025 21:48:55 +0200 Subject: [PATCH 1/3] StatsD: Index page and tutorial --- docs/ingest/telemetry/index.md | 6 ++ docs/integrate/statsd/index.md | 55 ++++++++++ docs/integrate/statsd/telegraf.conf | 30 ++++++ docs/integrate/statsd/tutorial.md | 154 ++++++++++++++++++++++++++++ 4 files changed, 245 insertions(+) create mode 100644 docs/integrate/statsd/index.md create mode 100644 docs/integrate/statsd/telegraf.conf create mode 100644 docs/integrate/statsd/tutorial.md diff --git a/docs/ingest/telemetry/index.md b/docs/ingest/telemetry/index.md index b9967b85..f39e0f3c 100644 --- a/docs/ingest/telemetry/index.md +++ b/docs/ingest/telemetry/index.md @@ -67,6 +67,12 @@ for collecting metrics data from applications and infrastructures. Send logs with rsyslog, a rocket‑fast system for log processing. :::: +::::{grid-item-card} StatsD +:link: statsd +:link-type: ref +Store metrics and statistics from StatsD, a daemon for stats aggregation. +:::: + ::::{grid-item-card} Telegraf :link: telegraf :link-type: ref diff --git a/docs/integrate/statsd/index.md b/docs/integrate/statsd/index.md new file mode 100644 index 00000000..f66924c6 --- /dev/null +++ b/docs/integrate/statsd/index.md @@ -0,0 +1,55 @@ +(statsd)= +# StatsD + +```{div} .float-right +[![StatsD logo](https://avatars.githubusercontent.com/u/8270030?s=200&v=4){height=100px loading=lazy}][StatsD] +``` +```{div} .clearfix +``` + +:::{rubric} About +::: + +[StatsD] is a daemon for easy but powerful system and application +metrics and stats aggregation. + +It is a network daemon that runs on the Node.js platform and listens for +statistics, like counters and timers, sent over UDP or TCP and sends +aggregates to one or more pluggable backend services. + +StatsD traditionally uses the Graphite backend and its successors, but it +can also use CrateDB as a backend, by relaying data through Telegraf and +using its built-in [CrateDB Output Plugin for Telegraf]. + +:::{rubric} Synopsis +::: + +Configure Telegraf using the StatsD input plugin and the CrateDB output plugin. + +:::{literalinclude} telegraf.conf +::: + + +:::{rubric} Learn +::: + +::::{grid} + +:::{grid-item-card} Tutorial: Use StatsD with CrateDB +:link: statsd-tutorial +:link-type: ref +How to configure StatsD and Telegraf to submit statistics to CrateDB. +::: + +:::: + + +:::{toctree} +:maxdepth: 1 +:hidden: +Tutorial +::: + + +[CrateDB Output Plugin for Telegraf]: https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cratedb +[StatsD]: https://github.com/statsd/statsd diff --git a/docs/integrate/statsd/telegraf.conf b/docs/integrate/statsd/telegraf.conf new file mode 100644 index 00000000..0b87171c --- /dev/null +++ b/docs/integrate/statsd/telegraf.conf @@ -0,0 +1,30 @@ +# StatsD Input Plugin +# https://github.com/influxdata/telegraf/tree/master/plugins/inputs/statsd +[[inputs.statsd]] + ## Protocol, must be "tcp", "udp4", "udp6" or "udp" (default=udp) + protocol = "udp" + + ## Address and port to host UDP listener on + service_address = ":8125" + + interval = "5s" + +# CrateDB Output Plugin +# https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cratedb +[[outputs.cratedb]] + ## Connection parameters for accessing the database see + ## https://pkg.go.dev/github.com/jackc/pgx/v4#ParseConfig + ## for available options + url = "postgres://crate:crate@cratedb/doc?sslmode=disable" + + ## Timeout for all CrateDB queries. + # timeout = "5s" + + ## Name of the table to store metrics in. + # table = "metrics" + + ## If true, and the metrics table does not exist, create it automatically. + table_create = true + + ## The character(s) to replace any '.' in an object key with + # key_separator = "_" diff --git a/docs/integrate/statsd/tutorial.md b/docs/integrate/statsd/tutorial.md new file mode 100644 index 00000000..155c8b02 --- /dev/null +++ b/docs/integrate/statsd/tutorial.md @@ -0,0 +1,154 @@ +(statsd-tutorial)= +# Load data into CrateDB using StatsD and Telegraf + +This tutorial walks you through configuring [Telegraf] to receive [StatsD] +metrics and store them into CrateDB. + +## Prerequisites + +Docker is used for running all components. This approach works consistently +across Linux, macOS, and Windows. Alternatively, you can use Podman. + +### CLI + +Prepare shortcut for the psql command. + +::::{tab-set} +:sync-group: os + +:::{tab-item} Linux and macOS +:sync: unix +To make the settings persistent, add them to your shell profile (`~/.profile`). +```shell +alias nc="docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23" +alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql" +``` +::: +:::{tab-item} Windows PowerShell +:sync: powershell +To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). +```powershell +function nc { docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 @args } +function psql { docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql @args } +``` +::: +:::{tab-item} Windows Command +:sync: dos +```shell +doskey nc=docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 $* +doskey psql=docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql $* +``` +::: + +:::: + +### CrateDB + +Create a shared network. +```shell +docker network create cratedb-demo +``` + +Start CrateDB. +```shell +docker run --name=cratedb --rm -it --network=cratedb-demo \ + --publish=4200:4200 --publish=5432:5432 \ + --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node +``` + + +## Configure Telegraf + +Configure Telegraf to receive StatsD and to store them +in CrateDB, by using the configuration blueprint outlined below, possibly +adjusting it to match your environment. Store this file under the name +`telegraf.conf`. + +:::{literalinclude} telegraf.conf +::: + + +## Start Telegraf + +::::{tab-set} +:sync-group: os + +:::{tab-item} Linux and macOS +:sync: unix +```shell +docker run --name=telegraf --rm -it --network=cratedb-demo \ + --volume "$(pwd)"/telegraf.conf:/etc/telegraf/telegraf.conf \ + --publish 8125:8125/udp \ + docker.io/telegraf +``` +::: +:::{tab-item} Windows PowerShell +:sync: powershell +```powershell +docker run --name=telegraf --rm -it --network=cratedb-demo ` + --volume "${PWD}\telegraf.conf:/etc/telegraf/telegraf.conf" ` + --publish 8125:8125/udp ` + docker.io/telegraf +``` +::: +:::{tab-item} Windows Command +:sync: dos +```shell +docker run --name=telegraf --rm -it --network=cratedb-demo ^ + --volume "%cd%\telegraf.conf:/etc/telegraf/telegraf.conf" ^ + --publish 8125:8125/udp ^ + docker.io/telegraf +``` +::: +:::: + +## Submit data + +### netcat +Use [netcat] for submitting data. +```shell +echo "temperature:42|g\nhumidity:84|g" | nc -C -w 1 -u telegraf 8125 +``` + +### Python +Use the [statsd package] to submit data from your Python application. +```shell +uv pip install statsd +``` +```python +from statsd import StatsClient + +statsd = StatsClient("localhost", 8125) +statsd.gauge("temperature", 42) +statsd.gauge("humidity", 84) +statsd.close() +``` + +### Any + +Use any of the available [StatsD client libraries] for Node.js, Java, Python, +Ruby, Perl, PHP, Clojure, Io, C, C++, .NET, Go, Apache, Varnish, PowerShell, +Browser, Objective-C, ActionScript, WordPress, Drupal, Haskell, R, Lua, or +Nim. + +## Explore data + +After receiving data, the first metrics will appear in the designated table in +CrateDB, ready to be inspected. +```shell +psql "postgresql://crate:crate@cratedb:5432/" -c "SELECT * FROM doc.metrics ORDER BY timestamp LIMIT 5;" +``` +```psql + hash_id | timestamp | name | tags | fields | day +----------------------+----------------------------+-------------+-----------------------------------------------+--------------+---------------------------- + -8005856065082590291 | 2025-09-20 19:30:45.000+00 | temperature | {"host":"2748411a9651","metric_type":"gauge"} | {"value":42} | 2025-09-20 00:00:00.000+00 + 7068016256787696496 | 2025-09-20 19:30:45.000+00 | humidity | {"host":"2748411a9651","metric_type":"gauge"} | {"value":84} | 2025-09-20 00:00:00.000+00 +(2 rows) +``` + + +[netcat]: https://en.wikipedia.org/wiki/Netcat +[StatsD]: https://github.com/statsd/statsd +[statsd package]: https://pypi.org/project/statsd/ +[StatsD client libraries]: https://github.com/statsd/statsd/wiki#client-implementations +[Telegraf]: https://www.influxdata.com/time-series-platform/telegraf/ From a56806a653d43d886b3e18ec74cedaa64a5eb6ed Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 20 Sep 2025 22:07:11 +0200 Subject: [PATCH 2/3] StatsD: Implement suggestions by CodeRabbit --- docs/integrate/statsd/index.md | 12 ++++++------ docs/integrate/statsd/tutorial.md | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/integrate/statsd/index.md b/docs/integrate/statsd/index.md index f66924c6..cf8aa2fb 100644 --- a/docs/integrate/statsd/index.md +++ b/docs/integrate/statsd/index.md @@ -10,16 +10,16 @@ :::{rubric} About ::: -[StatsD] is a daemon for easy but powerful system and application +[StatsD] provides easy but powerful system and application metrics and stats aggregation. -It is a network daemon that runs on the Node.js platform and listens for -statistics, like counters and timers, sent over UDP or TCP and sends +This network daemon runs on the Node.js platform and listens for +statistics, like counters and timers, sent over UDP or TCP. It then sends aggregates to one or more pluggable backend services. -StatsD traditionally uses the Graphite backend and its successors, but it -can also use CrateDB as a backend, by relaying data through Telegraf and -using its built-in [CrateDB Output Plugin for Telegraf]. +StatsD traditionally uses the Graphite backend and its successors, but you +can also configure it to use CrateDB as a backend by relaying data through Telegraf +with its built-in [CrateDB Output Plugin for Telegraf]. :::{rubric} Synopsis ::: diff --git a/docs/integrate/statsd/tutorial.md b/docs/integrate/statsd/tutorial.md index 155c8b02..510f54bd 100644 --- a/docs/integrate/statsd/tutorial.md +++ b/docs/integrate/statsd/tutorial.md @@ -18,7 +18,7 @@ Prepare shortcut for the psql command. :::{tab-item} Linux and macOS :sync: unix -To make the settings persistent, add them to your shell profile (`~/.profile`). +Add these settings to your shell profile (`~/.profile`) to make them persistent. ```shell alias nc="docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23" alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql" @@ -26,7 +26,7 @@ alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql ::: :::{tab-item} Windows PowerShell :sync: powershell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). +Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent. ```powershell function nc { docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 @args } function psql { docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql @args } @@ -59,10 +59,9 @@ docker run --name=cratedb --rm -it --network=cratedb-demo \ ## Configure Telegraf -Configure Telegraf to receive StatsD and to store them -in CrateDB, by using the configuration blueprint outlined below, possibly -adjusting it to match your environment. Store this file under the name -`telegraf.conf`. +Use the configuration blueprint below to configure Telegraf to receive StatsD +metrics and store them in CrateDB. Adjust the configuration to match your +environment and save this file as `telegraf.conf`. :::{literalinclude} telegraf.conf ::: @@ -133,8 +132,9 @@ Nim. ## Explore data -After receiving data, the first metrics will appear in the designated table in -CrateDB, ready to be inspected. +After Telegraf receives data, CrateDB stores the metrics in the designated table, +ready for inspection. + ```shell psql "postgresql://crate:crate@cratedb:5432/" -c "SELECT * FROM doc.metrics ORDER BY timestamp LIMIT 5;" ``` From b503656b010bb3c5b35c2b031a16f29f0e23a854 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 23 Sep 2025 20:50:04 +0200 Subject: [PATCH 3/3] StatsD: s/tutorial/usage guide/ --- docs/integrate/statsd/index.md | 6 +++--- docs/integrate/statsd/{tutorial.md => usage.md} | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) rename docs/integrate/statsd/{tutorial.md => usage.md} (97%) diff --git a/docs/integrate/statsd/index.md b/docs/integrate/statsd/index.md index cf8aa2fb..5ddee73c 100644 --- a/docs/integrate/statsd/index.md +++ b/docs/integrate/statsd/index.md @@ -35,8 +35,8 @@ Configure Telegraf using the StatsD input plugin and the CrateDB output plugin. ::::{grid} -:::{grid-item-card} Tutorial: Use StatsD with CrateDB -:link: statsd-tutorial +:::{grid-item-card} Use StatsD with CrateDB +:link: statsd-usage :link-type: ref How to configure StatsD and Telegraf to submit statistics to CrateDB. ::: @@ -47,7 +47,7 @@ How to configure StatsD and Telegraf to submit statistics to CrateDB. :::{toctree} :maxdepth: 1 :hidden: -Tutorial +Usage ::: diff --git a/docs/integrate/statsd/tutorial.md b/docs/integrate/statsd/usage.md similarity index 97% rename from docs/integrate/statsd/tutorial.md rename to docs/integrate/statsd/usage.md index 510f54bd..5c5c604f 100644 --- a/docs/integrate/statsd/tutorial.md +++ b/docs/integrate/statsd/usage.md @@ -1,7 +1,7 @@ -(statsd-tutorial)= +(statsd-usage)= # Load data into CrateDB using StatsD and Telegraf -This tutorial walks you through configuring [Telegraf] to receive [StatsD] +This usage guide walks you through configuring [Telegraf] to receive [StatsD] metrics and store them into CrateDB. ## Prerequisites