feat: upgrade the elastic stack (#130)

Author: FantasticFiasco

.github/workflows/ci-cd.yml

@@ -15,11 +15,3 @@ jobs:
       - uses: docker/setup-buildx-action@v3
       - run: docker-compose -f elastic-stack/docker-compose.yml build
       - run: docker-compose -f serilog/docker-compose.yml build
-
-  update_readme:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - run: ./scripts/update-readme.sh

README.md

@@ -1,12 +1,16 @@
+<!-- omit from toc -->
 # Serilog and the Elastic Stack
 
+<!-- omit from toc -->
 ## Table of contents
 
 - [Introduction](#introduction)
 - [What you will end up with](#what-you-will-end-up-with)
 - [Requirements](#requirements)
-- [Usage on Windows](#usage-on-windows)
-- [Usage on Linux and macOS](#usage-on-linux-and-macos)
+- [Usage](#usage)
+  - [Bringing up Elastic Stack](#bringing-up-elastic-stack)
+  - [Publishing log events using Serilog](#publishing-log-events-using-serilog)
+  - [Using Kibana to render the log events](#using-kibana-to-render-the-log-events)
 - [Credit](#credit)
 
 ---
@@ -25,38 +29,28 @@ With a running Elastic Stack and Serilog producing log events you are now ready
 
 - [Docker](https://www.docker.com/community-edition#/download)
 
-## Usage on Windows
+## Usage
 
 ### Bringing up Elastic Stack
 
 Start the stack using `docker`:
 
 ```posh
-PS> cd .\elastic-stack\
-PS> docker compose up
-```
+cd .\elastic-stack\
 
-If this is the first time the stack is started, you'll have to create a Logstash index pattern. Give the stack some time to initialize and then run the following commands in PowerShell:
+# This command is only necessary the first time the stack is started
+docker compose up setup
 
-```posh
-PS> $Headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
-PS> $Headers.Add("Content-Type", "application/json")
-PS> $Headers.Add("kbn-version", "7.17.0")
-PS> $Auth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("elastic:changeme"))
-PS> $Headers.Add("Authorization", "Basic {0}" -f $Auth)
-PS> Invoke-RestMethod "http://localhost:5601/api/saved_objects/index-pattern" `
-      -Method Post `
-      -Headers $Headers `
-      -Body '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
+docker compose up
 ```
 
 ### Publishing log events using Serilog
 
 Run the following commands to publish log events to Logstash using Serilog:
 
 ```posh
-PS> cd .\serilog\
-PS> docker compose up
+cd .\serilog\
+docker compose up
 ```
 
 If you decide to run the application outside of Docker in your terminal, don't forget to change the request URI to `http://localhost:31311`. More information can be found in `.\serilog\Program.cs`.
@@ -65,42 +59,6 @@ If you decide to run the application outside of Docker in your terminal, don't f
 
 Access the Kibana web UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser, and when prompted enter username `elastic` and password `changeme`.
 
-## Usage on Linux and macOS
-
-### Bringing up Elastic Stack
-
-Start the stack using `docker`:
-
-```bash
-$ cd elastic-stack/
-$ docker compose up
-```
-
-If this is the first time the stack is started, you'll have to create a Logstash index pattern. Give the stack some time to initialize and then run the following commands:
-
-```bash
-$ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \
-    -H 'Content-Type: application/json' \
-    -H 'kbn-version: 7.17.0' \
-    -u elastic:changeme \
-    -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
-```
-
-### Publishing log events using Serilog
-
-Run the following commands to publish log events to Logstash using Serilog:
-
-```bash
-$ cd serilog/
-$ docker compose up
-```
-
-If you decide to run the application outside of Docker in your terminal, don't forget to change the request URI to `http://localhost:31311`. More information can be found in `./serilog/Program.cs`.
-
-### Using Kibana to render the log events
-
-Access the Kibana web UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser, and when prompted enter username `elastic` and password `changeme`.
-
 ## Credit
 
 The `elastic-stack` directory is a clone of [docker-elk](https://github.com/deviantony/docker-elk) with minor modifications. Credit to [deviantony](https://github.com/deviantony) for publishing the Elastic Stack boilerplate.

elastic-stack/.env

@@ -1 +1,22 @@
-ELK_VERSION=7.17.0
+ELASTIC_VERSION=8.13.2
+
+## Passwords for stack users
+#
+
+# User 'elastic' (built-in)
+#
+# Superuser role, full access to cluster management and data indices.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+ELASTIC_PASSWORD='changeme'
+
+# User 'logstash_internal' (custom)
+#
+# The user Logstash uses to connect and send data to Elasticsearch.
+# https://www.elastic.co/guide/en/logstash/current/ls-security.html
+LOGSTASH_INTERNAL_PASSWORD='changeme'
+
+# User 'kibana_system' (built-in)
+#
+# The user Kibana uses to connect and communicate with Elasticsearch.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+KIBANA_SYSTEM_PASSWORD='changeme'

elastic-stack/docker-compose.yml

@@ -1,57 +1,100 @@
-version: '3.2'
-
 services:
+  # The 'setup' service runs a one-off script which initializes users inside
+  # Elasticsearch — such as 'logstash_internal' and 'kibana_system' — with the
+  # values of the passwords defined in the '.env' file. It also creates the
+  # roles required by some of these users.
+  #
+  # This task only needs to be performed once, during the *initial* startup of
+  # the stack. Any subsequent run will reset the passwords of existing users to
+  # the values defined inside the '.env' file, and the built-in roles to their
+  # default permissions.
+  #
+  # By default, it is excluded from the services started by 'docker compose up'
+  # due to the non-default profile it belongs to. To run it, either provide the
+  # '--profile=setup' CLI flag to Compose commands, or "up" the service by name
+  # such as 'docker compose up setup'.
+  setup:
+    profiles:
+      - setup
+    build:
+      context: setup/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    init: true
+    volumes:
+      - ./setup/entrypoint.sh:/entrypoint.sh:ro,Z
+      - ./setup/lib.sh:/lib.sh:ro,Z
+      - ./setup/roles:/roles:ro,Z
+    environment:
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
   elasticsearch:
     build:
       context: elasticsearch/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
     volumes:
-      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z
-      - elasticsearch:/usr/share/elasticsearch/data:z
+      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,Z
+      - elasticsearch:/usr/share/elasticsearch/data:Z
     ports:
-      - "9200:9200"
-      - "9300:9300"
+      - 9200:9200
+      - 9300:9300
     environment:
-      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
-      ELASTIC_PASSWORD: changeme
+      node.name: elasticsearch
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+      # Bootstrap password.
+      # Used to initialize the keystore during the initial startup of
+      # Elasticsearch. Ignored on subsequent runs.
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
       # Use single node discovery in order to disable production mode and avoid bootstrap checks.
       # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
       discovery.type: single-node
     networks:
       - elk
+    restart: unless-stopped
 
   logstash:
     build:
       context: logstash/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
     volumes:
-      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,z
-      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,z
+      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z
+      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
     ports:
-      - "9600:9600"
-      - "31311:31311"
+      - 9600:9600
+      - 31311:31311
     environment:
-      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+      LS_JAVA_OPTS: -Xms256m -Xmx256m
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
     networks:
       - elk
     depends_on:
       - elasticsearch
+    restart: unless-stopped
 
   kibana:
     build:
       context: kibana/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
     volumes:
-      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,z
+      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
     ports:
-      - "5601:5601"
+      - 5601:5601
+    environment:
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
     networks:
       - elk
     depends_on:
       - elasticsearch
+    restart: unless-stopped
 
 networks:
   elk:

elastic-stack/elasticsearch/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

elastic-stack/elasticsearch/Dockerfile

@@ -1,7 +1,7 @@
-ARG ELK_VERSION
+ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
 
 # Add your elasticsearch plugins setup here
 # Example: RUN elasticsearch-plugin install analysis-icu

elastic-stack/elasticsearch/config/elasticsearch.yml

@@ -1,13 +1,12 @@
 ---
 ## Default Elasticsearch configuration from Elasticsearch base image.
-## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
+## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml
 #
-cluster.name: "docker-cluster"
+cluster.name: docker-cluster
 network.host: 0.0.0.0
 
 ## X-Pack settings
-## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html
 #
 xpack.license.self_generated.type: trial
 xpack.security.enabled: true
-xpack.monitoring.collection.enabled: true

elastic-stack/kibana/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

elastic-stack/kibana/Dockerfile

@@ -1,7 +1,7 @@
-ARG ELK_VERSION
+ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
+FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION}
 
 # Add your kibana plugins setup here
 # Example: RUN kibana-plugin install <name|url>