From b1e068f0f0c07bc925bf423ac1dfdd343dcd3239 Mon Sep 17 00:00:00 2001 From: sushil Date: Mon, 20 May 2019 13:45:18 +0530 Subject: [PATCH 1/7] Added new config with lastest elasticsearch and kibana --- docker-compose-latest-elastic.yml | 86 +++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 docker-compose-latest-elastic.yml diff --git a/docker-compose-latest-elastic.yml b/docker-compose-latest-elastic.yml new file mode 100644 index 0000000..f1eee43 --- /dev/null +++ b/docker-compose-latest-elastic.yml @@ -0,0 +1,86 @@ +# EFK docker stack using latest elasticsearch and kibana +# Author: buldozer911@github + +version: '3.6' + +volumes: + esdata01: + esdata02: + +services: + + web: + image: httpd:2.2.32 + ports: + - "80:80" + depends_on: + - fluentd + logging: + driver: "fluentd" + options: + fluentd-address: localhost:24224 + tag: httpd.access + + fluentd: + build: ./fluentd + volumes: + - ./fluentd/conf:/fluentd/etc + depends_on: + - elastic01 + - elastic02 + - kibana01 + ports: + - "24224:24224" + - "24224:24224/udp" + logging: + driver: "json-file" + options: + max-size: 100m + max-file: "5" + + elastic01: + image: docker.elastic.co/elasticsearch/elasticsearch:7.0.1 + container_name: elastic01 + environment: + - node.name=elastic01 + - discovery.seed_hosts=elastic02 + - cluster.initial_master_nodes=elastic01,elastic02 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + + elastic02: + image: docker.elastic.co/elasticsearch/elasticsearch:7.0.1 + container_name: elastic02 + environment: + - node.name=elastic02 + - discovery.seed_hosts=elastic01 + - cluster.initial_master_nodes=elastic01,elastic02 + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata02:/usr/share/elasticsearch/data + + kibana01: + image: docker.elastic.co/kibana/kibana:7.0.1 + container_name: kibana01 + environment: + - ELASTICSEARCH_HOSTS=http://elastic01:9200 + ports: + - 5601:5601 + depends_on: + - elastic01 + - elastic02 \ No newline at end of file From 2a8e6dc543ab6cbdd0429732f8623a9bb0a57693 Mon Sep 17 00:00:00 2001 From: sushil Date: Mon, 20 May 2019 14:04:19 +0530 Subject: [PATCH 2/7] cp --- docker-compose-latest-elastic.yml => efk7.yml | 12 ------------ 1 file changed, 12 deletions(-) rename docker-compose-latest-elastic.yml => efk7.yml (88%) diff --git a/docker-compose-latest-elastic.yml b/efk7.yml similarity index 88% rename from docker-compose-latest-elastic.yml rename to efk7.yml index f1eee43..8d55328 100644 --- a/docker-compose-latest-elastic.yml +++ b/efk7.yml @@ -9,18 +9,6 @@ volumes: services: - web: - image: httpd:2.2.32 - ports: - - "80:80" - depends_on: - - fluentd - logging: - driver: "fluentd" - options: - fluentd-address: localhost:24224 - tag: httpd.access - fluentd: build: ./fluentd volumes: From a3e37cf19174031c6bc364819cc53893423703be Mon Sep 17 00:00:00 2001 From: buldozer911 Date: Mon, 20 May 2019 14:08:33 +0530 Subject: [PATCH 3/7] Update README.md --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 1816fc9..28a99d1 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,17 @@ Docker compose file for setting up a EFK service A basic docker compose file that will set up Elasticsearch, Fluentd, and Kibana. +Increase virtal memory +---------------------- + +Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions. + +On Linux, you can increase the limits by running the following command as root: + + sysctl -w vm.max_map_count=262144 + +To set this value permanently, update the vm.max_map_count setting in /etc/sysctl.conf. To verify after rebooting, run sysctl vm.max_map_count. + Example ------- @@ -10,11 +21,22 @@ The file `example/httpd.yml` shows how to configure a service to use EFK as its docker-compose -f docker-compose.yml -f example/httpd.yml up +with latest elasticsearch 7.0.1 and kibana 7.0.1 run: + + docker-compose -f efk7.yml -f example/httpd.yml + Then, go to your browser and access `http://localhost:80` (httpd) and `http://localhost:5601` (kibana). You should be able to see the httpd's logs in kibana's discovery tab. By the way, if you are wondering what is this index kibana asks the fist time you access it, it is `fluentd-*`. After you are done, just run: docker-compose -f docker-compose.yml -f example/httpd.yml rm -f +with latest elasticsearch 7.0.1 and kibana 7.0.1 run: + + docker-compose -f efk7.yml.yml -f example/httpd.yml up + And all services will be reclaimed. + + + From 516cc8598241e652f802bd28de2c3feaf0554f92 Mon Sep 17 00:00:00 2001 From: buldozer911 Date: Mon, 20 May 2019 14:16:38 +0530 Subject: [PATCH 4/7] Create fluentd.latest.conf fluentd conf for elasticsearch 7.0.1 --- fluentd/conf/fluentd.latest.conf | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 fluentd/conf/fluentd.latest.conf diff --git a/fluentd/conf/fluentd.latest.conf b/fluentd/conf/fluentd.latest.conf new file mode 100644 index 0000000..4488caa --- /dev/null +++ b/fluentd/conf/fluentd.latest.conf @@ -0,0 +1,23 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type copy + + @type elasticsearch + hosts elastic01:9200,elastic02:9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + type_name access_log + tag_key @log_name + flush_interval 1s + + + @type stdout + + From 5e8931b424cc12ae8efe9c524eb5a451de492ce2 Mon Sep 17 00:00:00 2001 From: sushil Date: Mon, 20 May 2019 14:19:45 +0530 Subject: [PATCH 5/7] fixed fluentd conf error --- efk7.yml | 2 +- fluentd/{conf/fluentd.latest.conf => latest_conf/fluent.conf} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename fluentd/{conf/fluentd.latest.conf => latest_conf/fluent.conf} (100%) diff --git a/efk7.yml b/efk7.yml index 8d55328..2970444 100644 --- a/efk7.yml +++ b/efk7.yml @@ -12,7 +12,7 @@ services: fluentd: build: ./fluentd volumes: - - ./fluentd/conf:/fluentd/etc + - ./fluentd/latest_conf:/fluentd/etc depends_on: - elastic01 - elastic02 diff --git a/fluentd/conf/fluentd.latest.conf b/fluentd/latest_conf/fluent.conf similarity index 100% rename from fluentd/conf/fluentd.latest.conf rename to fluentd/latest_conf/fluent.conf From 5b70d1ed5fd4a6eb6825ed41b49d2e98393814cf Mon Sep 17 00:00:00 2001 From: buldozer911 Date: Mon, 20 May 2019 14:20:21 +0530 Subject: [PATCH 6/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 28a99d1..60e42ab 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Docker compose file for setting up a EFK service A basic docker compose file that will set up Elasticsearch, Fluentd, and Kibana. -Increase virtal memory +Increase virtual memory ---------------------- Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions. From 977c7e8c7d1c2a6b9e076f14bb95c18fec45f16d Mon Sep 17 00:00:00 2001 From: buldozer911 Date: Wed, 12 Jun 2019 22:23:00 +0530 Subject: [PATCH 7/7] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60e42ab..232f09d 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ The file `example/httpd.yml` shows how to configure a service to use EFK as its with latest elasticsearch 7.0.1 and kibana 7.0.1 run: - docker-compose -f efk7.yml -f example/httpd.yml + docker-compose -f efk7.yml -f example/httpd.yml up + Then, go to your browser and access `http://localhost:80` (httpd) and `http://localhost:5601` (kibana). You should be able to see the httpd's logs in kibana's discovery tab. By the way, if you are wondering what is this index kibana asks the fist time you access it, it is `fluentd-*`. @@ -33,7 +34,7 @@ After you are done, just run: with latest elasticsearch 7.0.1 and kibana 7.0.1 run: - docker-compose -f efk7.yml.yml -f example/httpd.yml up + docker-compose -f efk7.yml.yml -f example/httpd.yml rm -f And all services will be reclaimed.