diff --git a/README.md b/README.md index 1816fc9..232f09d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,17 @@ Docker compose file for setting up a EFK service A basic docker compose file that will set up Elasticsearch, Fluentd, and Kibana. +Increase virtual memory +---------------------- + +Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions. + +On Linux, you can increase the limits by running the following command as root: + + sysctl -w vm.max_map_count=262144 + +To set this value permanently, update the vm.max_map_count setting in /etc/sysctl.conf. To verify after rebooting, run sysctl vm.max_map_count. + Example ------- @@ -10,11 +21,23 @@ The file `example/httpd.yml` shows how to configure a service to use EFK as its docker-compose -f docker-compose.yml -f example/httpd.yml up +with latest elasticsearch 7.0.1 and kibana 7.0.1 run: + + docker-compose -f efk7.yml -f example/httpd.yml up + + Then, go to your browser and access `http://localhost:80` (httpd) and `http://localhost:5601` (kibana). You should be able to see the httpd's logs in kibana's discovery tab. By the way, if you are wondering what is this index kibana asks the fist time you access it, it is `fluentd-*`. After you are done, just run: docker-compose -f docker-compose.yml -f example/httpd.yml rm -f +with latest elasticsearch 7.0.1 and kibana 7.0.1 run: + + docker-compose -f efk7.yml.yml -f example/httpd.yml rm -f + And all services will be reclaimed. + + + diff --git a/efk7.yml b/efk7.yml new file mode 100644 index 0000000..2970444 --- /dev/null +++ b/efk7.yml @@ -0,0 +1,74 @@ +# EFK docker stack using latest elasticsearch and kibana +# Author: buldozer911@github + +version: '3.6' + +volumes: + esdata01: + esdata02: + +services: + + fluentd: + build: ./fluentd + volumes: + - ./fluentd/latest_conf:/fluentd/etc + depends_on: + - elastic01 + - elastic02 + - kibana01 + ports: + - "24224:24224" + - "24224:24224/udp" + logging: + driver: "json-file" + options: + max-size: 100m + max-file: "5" + + elastic01: + image: docker.elastic.co/elasticsearch/elasticsearch:7.0.1 + container_name: elastic01 + environment: + - node.name=elastic01 + - discovery.seed_hosts=elastic02 + - cluster.initial_master_nodes=elastic01,elastic02 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + + elastic02: + image: docker.elastic.co/elasticsearch/elasticsearch:7.0.1 + container_name: elastic02 + environment: + - node.name=elastic02 + - discovery.seed_hosts=elastic01 + - cluster.initial_master_nodes=elastic01,elastic02 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + + kibana01: + image: docker.elastic.co/kibana/kibana:7.0.1 + container_name: kibana01 + environment: + - ELASTICSEARCH_HOSTS=http://elastic01:9200 + ports: + - 5601:5601 + depends_on: + - elastic01 + - elastic02 \ No newline at end of file diff --git a/fluentd/latest_conf/fluent.conf b/fluentd/latest_conf/fluent.conf new file mode 100644 index 0000000..4488caa --- /dev/null +++ b/fluentd/latest_conf/fluent.conf @@ -0,0 +1,23 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type copy + + @type elasticsearch + hosts elastic01:9200,elastic02:9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + type_name access_log + tag_key @log_name + flush_interval 1s + + + @type stdout + +