New config for latest elasticsearch and kibana

README.md

@@ -3,18 +3,41 @@ Docker compose file for setting up a EFK service
 
 A basic docker compose file that will set up Elasticsearch, Fluentd, and Kibana.
 
+Increase virtual memory
+----------------------
+
+Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions.
+
+On Linux, you can increase the limits by running the following command as root:
+
+    sysctl -w vm.max_map_count=262144
+
+To set this value permanently, update the vm.max_map_count setting in /etc/sysctl.conf. To verify after rebooting, run sysctl vm.max_map_count.
+
 Example
 -------
 
 The file `example/httpd.yml` shows how to configure a service to use EFK as its logging facility. To test using this file, just run:
 
     docker-compose -f docker-compose.yml -f example/httpd.yml up
 
+with latest elasticsearch 7.0.1 and kibana 7.0.1 run:
+
+    docker-compose -f efk7.yml -f example/httpd.yml up
+
+
 Then, go to your browser and access `http://localhost:80` (httpd) and `http://localhost:5601` (kibana). You should be able to see the httpd's logs in kibana's discovery tab. By the way, if you are wondering what is this index kibana asks the fist time you access it, it is `fluentd-*`.
 
 After you are done, just run:
 
     docker-compose -f docker-compose.yml -f example/httpd.yml rm -f
 
+with latest elasticsearch 7.0.1 and kibana 7.0.1 run:
+
+    docker-compose -f efk7.yml.yml -f example/httpd.yml rm -f
+
 And all services will be reclaimed.
 
+
+
+

efk7.yml

@@ -0,0 +1,74 @@
+# EFK docker stack using latest elasticsearch and kibana
+# Author: buldozer911@github
+
+version: '3.6'
+
+volumes:
+  esdata01:
+  esdata02:
+
+services:
+
+  fluentd:
+    build: ./fluentd
+    volumes:
+      - ./fluentd/latest_conf:/fluentd/etc
+    depends_on:
+      - elastic01
+      - elastic02
+      - kibana01
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    logging:
+      driver: "json-file"
+      options:
+          max-size: 100m
+          max-file: "5"
+
+  elastic01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.0.1
+    container_name: elastic01
+    environment:
+      - node.name=elastic01
+      - discovery.seed_hosts=elastic02
+      - cluster.initial_master_nodes=elastic01,elastic02
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+
+  elastic02:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.0.1
+    container_name: elastic02
+    environment:
+      - node.name=elastic02
+      - discovery.seed_hosts=elastic01
+      - cluster.initial_master_nodes=elastic01,elastic02
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata02:/usr/share/elasticsearch/data
+
+  kibana01:
+    image: docker.elastic.co/kibana/kibana:7.0.1
+    container_name: kibana01
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elastic01:9200
+    ports:
+      - 5601:5601
+    depends_on: 
+      - elastic01
+      - elastic02

fluentd/latest_conf/fluent.conf

@@ -0,0 +1,23 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<match *.**>
+  @type copy
+  <store>
+    @type elasticsearch
+    hosts elastic01:9200,elastic02:9200
+    logstash_format true
+    logstash_prefix fluentd
+    logstash_dateformat %Y%m%d
+    include_tag_key true
+    type_name access_log
+    tag_key @log_name
+    flush_interval 1s
+  </store>
+  <store>
+    @type stdout
+  </store>
+</match>