From 191aa4859e8d245be7979b76c057baad7cfa95b3 Mon Sep 17 00:00:00 2001 From: Dorab Patel Date: Mon, 19 Aug 2019 18:15:31 -0700 Subject: [PATCH 01/26] Use ACME staging servers for vagrant hosts. --- clojurians_log/Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clojurians_log/Vagrantfile b/clojurians_log/Vagrantfile index edd0cdc..f128fa0 100644 --- a/clojurians_log/Vagrantfile +++ b/clojurians_log/Vagrantfile @@ -76,7 +76,8 @@ Vagrant.configure("2") do |config| ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS'] ansible.extra_vars = { ansible_python_interpreter: "/usr/bin/python3", clojurians_app_fqdn: "clojurians-log-staging.clojureverse.org", - use_demo_logs: true, # for only loading the demo logs on a vagrant host + use_demo_logs: true, # Load the demo logs on a vagrant host + acme_sh_default_staging: true # Use ACME default servers on a vagrant host acme_sh_default_force_issue: true, # Force ACME cert issue on a vagrant host } end From 65e396d4521a65e06e22158d6c8084b469953e62 Mon Sep 17 00:00:00 2001 From: Dorab Patel Date: Mon, 19 Aug 2019 18:16:30 -0700 Subject: [PATCH 02/26] Set up default terraform for deployment. Use the ACME staging servers in development. Always use python3 for ansible. --- clojurians_log/main.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index c4ca5b8..f5224ba 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -7,13 +7,12 @@ provider "exoscale" { data "template_file" "userdata" { template = "${file("userdata.sh.tmpl")}" vars = { - # For development / testing, use (uncomment) the line below - ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_force_issue=true\"" - git_clone_params = "--single-branch --branch exoscale-staging" - # For normal production use, the values set in ansible are appropriate. - # So, can leave the following empty. - # ansible_playbook_params = "" - # git_clone_params = "" + # For development / testing, use (uncomment) the lines below + # ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_staging=true acme_sh_default_force_issue=true\"" + # git_clone_params = "--single-branch --branch exoscale-deploy" + # The following can be used for production + ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\"" + git_clone_params = "" } } From f78e5221ee22dfe370320918d02ce58844aa7e22 Mon Sep 17 00:00:00 2001 From: Dorab Patel Date: Mon, 19 Aug 2019 20:58:56 -0700 Subject: [PATCH 03/26] Deploying to Exoscale. --- clojurians_log/main.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index f5224ba..5dfddb2 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -12,10 +12,13 @@ data "template_file" "userdata" { # git_clone_params = "--single-branch --branch exoscale-deploy" # The following can be used for production ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\"" - git_clone_params = "" + git_clone_params = "--single-branch --branch exoscale-deploy" } } +# TODO: It would be useful to have the instance name and the state to be +# parameterized so that you could use this same terraform file for the +# staging server and the production server. resource "exoscale_compute" "clojurians_log" { display_name = "clojurians-log" template = "Linux Ubuntu 18.04 LTS 64-bit" From 363a6482362a01a178ba65104dca01815d00e130 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Thu, 22 Aug 2019 14:03:57 +0200 Subject: [PATCH 04/26] Add terraform_apply script --- clojurians_log/README.md | 22 ++++++++++---------- clojurians_log/bin/terraform_apply | 32 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) create mode 100755 clojurians_log/bin/terraform_apply diff --git a/clojurians_log/README.md b/clojurians_log/README.md index b51da50..4faaecb 100644 --- a/clojurians_log/README.md +++ b/clojurians_log/README.md @@ -14,7 +14,7 @@ This is a work in progress and is subject to significant changes over time. Contact either of the [maintainers](https://github.com/clojureverse/nebula#maintainers) for this. - Create your own SSH keypair on the Console: [guide](https://community.exoscale.com/documentation/compute/ssh-keypairs/) -- Install Terraform v0.11+. +- Install Terraform v0.12.6+. - Download the latest Exoscale terraform provider for your OS from [here](https://github.com/exoscale/terraform-provider-exoscale/releases). - Decompress the archive and follow [the plugin installation](https://www.terraform.io/docs/configuration/providers.html#third-party-plugins). - Install [GPG](https://gnupg.org/download/) @@ -24,16 +24,24 @@ This is a work in progress and is subject to significant changes over time. - Run `export TF_VAR_exoscale_api_key="The Exoscale API key here"` - Run `export TF_VAR_exoscale_secret_key="The Exoscale secret key here"` - Run `export TF_VAR_exoscale_ssh_keypair_name="The Exoscale key pair name you created"` + - Run `bin/terraform_apply` + - To destroy: `terraform destroy` + - To ssh into the instance + - `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $(terraform output username)@$(terraform output ip_address)` + - You wil want to reboot the instance after provisioning the first time. (why?) + - You will need to update the IP address on the host name in the DNS as well. + +The `terraform_apply script does the following` - In the `playbooks/vars` dir: - Run `gpg --decrypt clojurians_log_secrets.yml.gpg > clojurians_log_secrets.yml` - - (Optional, for maintainers) Run: + - If you need to update the secret file later on you'll have to do this: ```bash gpg --encrypt --recipient your_email \ --recipient others_emails \ ... all the other maintainers \ clojurians_log_secrets.yml ``` - to produce a newly encrypted file and check it in. + to produce a newly encrypted file and check it in. - Run: ```bash terraform init \ @@ -41,13 +49,7 @@ This is a work in progress and is subject to significant changes over time. -backend-config="secret_key=${TF_VAR_exoscale_secret_key}" ``` - Run `terraform plan -out plan` - - Make sure all looks good. - Run `terraform apply plan` - - To destroy: `terraform destroy` - - To ssh into the instance - - `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $(terraform output username)@$(terraform output ip_address)` - - You wil want to reboot the instance after provisioning the first time. - - You will need to update the IP address on the host name in the DNS as well. ## How to use Vagrant @@ -87,7 +89,7 @@ This is a work in progress and is subject to significant changes over time. You can log into it using `vagrant ssh`. - If you need to re-provision (for example, if you edit the Ansible scripts) you can use `vagrant provision`. -- To shut down the VM, use `vagrant halt`. +- To shut down the VM, use `vagrant halt`. ## How to deploy the Clojurians-log app (with demo data) diff --git a/clojurians_log/bin/terraform_apply b/clojurians_log/bin/terraform_apply new file mode 100755 index 0000000..4f03dd7 --- /dev/null +++ b/clojurians_log/bin/terraform_apply @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +if [[ -z "${TF_VAR_exoscale_ssh_keypair_name}" ]]; then + echo "TF_VAR_exoscale_ssh_keypair_name not set, aborting." + exit 1 +fi + +if [[ -z "${TF_VAR_exoscale_api_key}" ]]; then + echo "TF_VAR_exoscale_api_key not set, aborting." + exit 1 +fi + +if [[ -z "${TF_VAR_exoscale_secret_key}" ]]; then + echo "TF_VAR_exoscale_secret_key not set, aborting." + exit 1 +fi + +# File does not exist, or encrypted version is newer: decrypt +if [[ "playbooks/vars/clojurians_log_secrets.yml.gpg" -nt "playbooks/vars/clojurians_log_secrets.yml" ]]; then + gpg --decrypt playbooks/vars/clojurians_log_secrets.yml.gpg > playbooks/vars/clojurians_log_secrets.yml +fi + +if [[ ! -d ".terraform" ]]; then + terraform init \ + -backend-config="access_key=${TF_VAR_exoscale_api_key}" \ + -backend-config="secret_key=${TF_VAR_exoscale_secret_key}" +fi + +terraform plan -out plan +terraform apply plan From 64474aa6c541e4070f95d439a48a886bf03c362a Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Thu, 22 Aug 2019 14:04:19 +0200 Subject: [PATCH 05/26] Use a large instance, we need the memory --- clojurians_log/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index 5dfddb2..8177f2c 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -23,7 +23,7 @@ resource "exoscale_compute" "clojurians_log" { display_name = "clojurians-log" template = "Linux Ubuntu 18.04 LTS 64-bit" zone = "de-fra-1" - size = "Medium" + size = "Large" disk_size = 10 key_pair = "${var.exoscale_ssh_keypair_name}" user_data = "${data.template_file.userdata.rendered}" From a1a76af14045afcb0a36d603d6da8c01729fd6a0 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Thu, 22 Aug 2019 14:04:38 +0200 Subject: [PATCH 06/26] Use rtmbot from its own repo --- .../roles/plexus.rtmbot/tasks/main.yml | 35 +++---------------- .../plexus.rtmbot/templates/rtmbot.conf.j2 | 1 + .../plexus.rtmbot/templates/rtmbot.service.j2 | 4 +-- 3 files changed, 8 insertions(+), 32 deletions(-) diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml index 8b169de..89ac163 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml @@ -1,38 +1,13 @@ --- -- name: Install rtmbot dependency - pip: - name: 'slackclient' - tags: rtmbot - -- name: Create rtmbot directory - file: - path: "{{ clojure_app_home_dir }}/rtmbot" - state: directory - owner: "{{ clojure_app_user }}" - group: "{{ clojure_app_user }}" - tags: rtmbot - -- name: Create rtmbot logs directory - file: - path: "{{ clojure_app_home_dir }}/rtmbot/logs" - state: directory - owner: "{{ clojure_app_user }}" - group: "{{ clojure_app_user }}" - tags: rtmbot - -- name: Install run-rtmbot.sh - template: - src: templates/run-rtmbot.sh.j2 - dest: "{{ clojure_app_home_dir }}/rtmbot/run-rtmbot.sh" - owner: "{{ clojure_app_user }}" - group: "{{ clojure_app_user }}" - mode: 0755 - tags: rtmbot +- name: Install rtmbot + git: + repo: "https://github.com/clojureverse/rtmbot.git" + dest: "/var/rtmbot" - name: Install rtmbot.conf template: src: templates/rtmbot.conf.j2 - dest: "{{ clojure_app_home_dir }}/rtmbot/rtmbot.conf" + dest: "/var/rtmbot/rtmbot.conf" owner: "{{ clojure_app_user }}" group: "{{ clojure_app_user }}" mode: 0644 diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 index 977fda7..7aefc94 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 @@ -1,2 +1,3 @@ SLACK_TOKEN: "{{ slack_api_token }}" LOGFILE: "{{ clojure_app_home_dir }}/rtmbot/rtmbot.log" +DESTINATION: "{{ clojure_app_home_dir }}/logs" \ No newline at end of file diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 index f474847..1358fe9 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 @@ -3,8 +3,8 @@ Description=Slack RTM logging bot After=network.target [Service] -WorkingDirectory={{ clojure_app_home_dir }}/rtmbot -ExecStart={{ clojure_app_home_dir}}/rtmbot/run-rtmbot.sh +WorkingDirectory=/var/rtmbot +ExecStart=/var/rtmbot/rtmbot.sh User={{ clojure_app_user }} Restart=always RestartSec=5 From 380f91158b3916d74a354e934e1f93ddd77ba62d Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Thu, 22 Aug 2019 14:32:52 +0200 Subject: [PATCH 07/26] Let rtmbot log to directory that exists --- .../playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 index 7aefc94..15d33e6 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 @@ -1,3 +1,3 @@ SLACK_TOKEN: "{{ slack_api_token }}" -LOGFILE: "{{ clojure_app_home_dir }}/rtmbot/rtmbot.log" -DESTINATION: "{{ clojure_app_home_dir }}/logs" \ No newline at end of file +LOGFILE: "{{ clojure_app_home_dir }}/rtmbot.log" +DESTINATION: "{{ clojure_app_home_dir }}/logs/" \ No newline at end of file From 1e0b822843f29326db0eedce810a61dc4769785b Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Thu, 22 Aug 2019 16:17:41 +0200 Subject: [PATCH 08/26] Increase disk size --- clojurians_log/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index 8177f2c..be9b5e8 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -24,7 +24,7 @@ resource "exoscale_compute" "clojurians_log" { template = "Linux Ubuntu 18.04 LTS 64-bit" zone = "de-fra-1" size = "Large" - disk_size = 10 + disk_size = 50 key_pair = "${var.exoscale_ssh_keypair_name}" user_data = "${data.template_file.userdata.rendered}" From 99f2ff0ba1f60b7905345750c1780483d7dc5679 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Thu, 22 Aug 2019 18:52:42 +0200 Subject: [PATCH 09/26] Tweak datomic and app memory settings --- clojurians_log/playbooks/clojurians-log.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 065bbf9..964879a 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -22,8 +22,9 @@ clojurians_app_http_port: 4242 datomic_pro_version: 0.9.5561.56 datomic_object_cache_max: 2g + datomic_transactor_max_heap: 3g clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx8g -Xms2g -Ddatomic.ObjectCacheMax=7g -Ddatomic.memcachedServers=127.0.0.1:11211" + JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx4g -Xms1g -Ddatomic.ObjectCacheMax=4g -Ddatomic.memcachedServers=127.0.0.1:11211" tasks: From 52b1b9af42d74b7a2753fdd7474a2b74d85d268c Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 09:21:05 +0200 Subject: [PATCH 10/26] Immediately clone current master for clojurians-log-app --- clojurians_log/playbooks/clojurians-log.yml | 17 ++++++++++++++++- .../plexus.clojurians-log/templates/config.edn | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 964879a..74fa73d 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -178,11 +178,26 @@ clojure_app_health_check_url: "http://localhost:{{clojurians_app_http_port}}/healthcheck" clojure_app_service_start_after: datomic.service - - name: Install the Clojurians Log app + - name: Create the repo for clojurians-log-app with push-to-deploy tags: clojurians-log import_role: name: plexus.clojurians-log + - name: Clone the Clojurians log app + tags: clojurians-log + git: + repo: "https://github.com/clojureverse/clojurians-log-app.git" + dest: "/root/clojurians-log-tmp" + + - name: Push master to the bare repo + tags: clojurians-log + command: "git push {{ clojure_app_repo_dir }} master" + become: yes + become_user: "{{ clojure_app_user }}" + args: + chdir: "/root/clojurians-log-tmp" + creates: "{{ clojure_app_repo_dir }}/HEAD" + - name: Prepare for ACME installs tags: setup-acme import_role: diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn b/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn index b4d6664..9836b6c 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn @@ -1,6 +1,6 @@ {:datomic {:uri "datomic:sql://{{ clojure_app_name }}?jdbc:postgresql://localhost:5432/{{ database_name }}?user={{ database_user }}&password={{ database_password }}"} :http {:port {{ clojurians_app_http_port }} - :origin "https://clojurians-log.clojureverse.org"} + :origin "https://{{ clojurians_app_fqdn }}"} :slack {:api-token "{{ slack_api_token }}" :log-dir "{{ clojure_app_home_dir }}/logs"} :cache-time 2678400 ;; ~one month From f8b7b40f1c73eb8a9236d194c39cb552a6731b91 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 09:22:46 +0200 Subject: [PATCH 11/26] Set up DNS records through cloudflare, support multi-workspace setup This allows easily creating staging environments --- clojurians_log/README.md | 2 ++ clojurians_log/main.tf | 30 +++++++++++++++++++++++++----- clojurians_log/security_groups.tf | 2 +- clojurians_log/userdata.sh.tmpl | 1 + clojurians_log/vars.tf | 12 ++++++++++++ 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/clojurians_log/README.md b/clojurians_log/README.md index 4faaecb..86d1801 100644 --- a/clojurians_log/README.md +++ b/clojurians_log/README.md @@ -24,6 +24,8 @@ This is a work in progress and is subject to significant changes over time. - Run `export TF_VAR_exoscale_api_key="The Exoscale API key here"` - Run `export TF_VAR_exoscale_secret_key="The Exoscale secret key here"` - Run `export TF_VAR_exoscale_ssh_keypair_name="The Exoscale key pair name you created"` + - Run `export TF_VAR_cloudflare_email="Your cloudflare account"` + - Run `export TF_VAR_cloudlfare_api_key="Cloudflare 'global API key' (not to be confused with cloudflare tokens)"` - Run `bin/terraform_apply` - To destroy: `terraform destroy` - To ssh into the instance diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index be9b5e8..f7de8b7 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -1,9 +1,24 @@ +locals { + instance_name = "${terraform.workspace == "default" ? "clojurians-log" : terraform.workspace}" +} + provider "exoscale" { version = "~> 0.10" key = "${var.exoscale_api_key}" secret = "${var.exoscale_secret_key}" } +provider "cloudflare" { + version = "~> 1.17" + + email = "${var.cloudflare_email}" + token = "${var.cloudflare_api_key}" +} + +provider "template" { + version = "~> 2.1" +} + data "template_file" "userdata" { template = "${file("userdata.sh.tmpl")}" vars = { @@ -11,16 +26,13 @@ data "template_file" "userdata" { # ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_staging=true acme_sh_default_force_issue=true\"" # git_clone_params = "--single-branch --branch exoscale-deploy" # The following can be used for production - ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\"" + ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\" --extra-vars \"clojurians_app_fqdn=${local.instance_name}.clojureverse.org\"" git_clone_params = "--single-branch --branch exoscale-deploy" } } -# TODO: It would be useful to have the instance name and the state to be -# parameterized so that you could use this same terraform file for the -# staging server and the production server. resource "exoscale_compute" "clojurians_log" { - display_name = "clojurians-log" + display_name = "${local.instance_name}" template = "Linux Ubuntu 18.04 LTS 64-bit" zone = "de-fra-1" size = "Large" @@ -36,6 +48,14 @@ resource "exoscale_compute" "clojurians_log" { output "ip_address" { value = exoscale_compute.clojurians_log.ip_address } + output "username" { value = exoscale_compute.clojurians_log.username } + +resource "cloudflare_record" "clojurians_log" { + domain = "clojureverse.org" + name = "${local.instance_name}" + value = "${exoscale_compute.clojurians_log.ip_address}" + type = "A" +} diff --git a/clojurians_log/security_groups.tf b/clojurians_log/security_groups.tf index 2fbf114..9762ce1 100644 --- a/clojurians_log/security_groups.tf +++ b/clojurians_log/security_groups.tf @@ -1,5 +1,5 @@ resource "exoscale_security_group" "clojurians_log" { - name = "clojurians_log" + name = "${local.instance_name}" description = "Security Group for Clojurians Log" } diff --git a/clojurians_log/userdata.sh.tmpl b/clojurians_log/userdata.sh.tmpl index ae74f9e..35a3cd1 100755 --- a/clojurians_log/userdata.sh.tmpl +++ b/clojurians_log/userdata.sh.tmpl @@ -14,6 +14,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt install -y ansible cd ~ +# TODO: copy from the local filesystem, instead of going over github # Clone repo and switch to it git clone ${git_clone_params} https://github.com/clojureverse/nebula cd nebula/clojurians_log diff --git a/clojurians_log/vars.tf b/clojurians_log/vars.tf index 1e33cfd..d0215e0 100644 --- a/clojurians_log/vars.tf +++ b/clojurians_log/vars.tf @@ -15,3 +15,15 @@ variable "exoscale_ssh_keypair_name" { type = string description = "The SSH keypair to be allowed on the instance" } + +variable "cloudflare_email" { + default = "" + type = string + description = "The email to authenticate with Cloudflare" +} + +variable "cloudflare_api_key" { + default = "" + type = string + description = "The email to authenticate with Cloudflare" +} From ed761417412e3fe6bc7c64a9d5973f321e6409be Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 09:27:33 +0200 Subject: [PATCH 12/26] Be more conservative with memory --- clojurians_log/playbooks/clojurians-log.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 74fa73d..029fae2 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -22,9 +22,9 @@ clojurians_app_http_port: 4242 datomic_pro_version: 0.9.5561.56 datomic_object_cache_max: 2g - datomic_transactor_max_heap: 3g + datomic_transactor_max_heap: 2g clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx4g -Xms1g -Ddatomic.ObjectCacheMax=4g -Ddatomic.memcachedServers=127.0.0.1:11211" + JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3g -Xms1g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" tasks: From d7ced0e4174f2d9458a095dc5c44ff83fadd1527 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 09:36:29 +0200 Subject: [PATCH 13/26] Move cloning of clojurians-log-app into the right role --- clojurians_log/playbooks/clojurians-log.yml | 17 +---------------- .../roles/plexus.clojurians-log/tasks/main.yml | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 029fae2..3c13e4c 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -178,26 +178,11 @@ clojure_app_health_check_url: "http://localhost:{{clojurians_app_http_port}}/healthcheck" clojure_app_service_start_after: datomic.service - - name: Create the repo for clojurians-log-app with push-to-deploy + - name: Install and configure the clojurians-log-app tags: clojurians-log import_role: name: plexus.clojurians-log - - name: Clone the Clojurians log app - tags: clojurians-log - git: - repo: "https://github.com/clojureverse/clojurians-log-app.git" - dest: "/root/clojurians-log-tmp" - - - name: Push master to the bare repo - tags: clojurians-log - command: "git push {{ clojure_app_repo_dir }} master" - become: yes - become_user: "{{ clojure_app_user }}" - args: - chdir: "/root/clojurians-log-tmp" - creates: "{{ clojure_app_repo_dir }}/HEAD" - - name: Prepare for ACME installs tags: setup-acme import_role: diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml index af97f5c..37ea9f7 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml @@ -37,6 +37,21 @@ group: "{{ clojure_app_user }}" mode: 0600 +- name: Clone the Clojurians log app + become: yes + become_user: "{{ clojure_app_user }}" + git: + repo: "https://github.com/clojureverse/clojurians-log-app.git" + dest: "/tmp/clojurians-log-tmp" + +- name: Push master to the bare repo + command: "git push {{ clojure_app_repo_dir }} master:master" + become: yes + become_user: "{{ clojure_app_user }}" + args: + chdir: "/tmp/clojurians-log-tmp" + creates: "{{ clojure_app_repo_dir }}/refs/heads/master" + - name: Check out existing logs become: yes become_user: "{{ clojure_app_user }}" From 8d446a6945db344608fe24ba8d7cded7047587f9 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 10:15:40 +0200 Subject: [PATCH 14/26] Give datomic and app each 3.5g of memory (leaving 1g for the rest) --- clojurians_log/playbooks/clojurians-log.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 3c13e4c..cc15425 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -22,9 +22,9 @@ clojurians_app_http_port: 4242 datomic_pro_version: 0.9.5561.56 datomic_object_cache_max: 2g - datomic_transactor_max_heap: 2g + datomic_transactor_max_heap: 3584m clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3g -Xms1g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" + JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" tasks: From 27c6d26f42d5f3a1997a8fdaa7bee585613651e6 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 11:04:16 +0200 Subject: [PATCH 15/26] Add librato agent for collecting metrics --- clojurians_log/playbooks/clojurians-log.yml | 9 +- .../roles/librato.librato/.gitignore | 5 + .../roles/librato.librato/.kitchen.yml | 102 ++++ .../roles/librato.librato/Ansiblefile | 4 + .../playbooks/roles/librato.librato/Gemfile | 5 + .../playbooks/roles/librato.librato/README.md | 559 ++++++++++++++++++ .../roles/librato.librato/defaults/main.yml | 73 +++ .../librato.librato/files/collectd-core.pref | 3 + .../roles/librato.librato/files/collectd.pref | 3 + .../roles/librato.librato/files/cpu.conf | 4 + .../roles/librato.librato/files/df.conf | 15 + .../roles/librato.librato/files/disk.conf | 1 + .../roles/librato.librato/files/load.conf | 1 + .../roles/librato.librato/files/memory.conf | 5 + .../roles/librato.librato/files/swap.conf | 1 + .../roles/librato.librato/handlers/main.yml | 5 + .../librato.librato/meta/.galaxy_install_info | 1 + .../roles/librato.librato/meta/main.yml | 30 + .../roles/librato.librato/requirements.yml | 3 + .../roles/librato.librato/tasks/agent.yml | 83 +++ .../roles/librato.librato/tasks/apache.yml | 9 + .../roles/librato.librato/tasks/docker.yml | 9 + .../librato.librato/tasks/elasticsearch.yml | 9 + .../roles/librato.librato/tasks/haproxy.yml | 9 + .../roles/librato.librato/tasks/jvm.yml | 9 + .../roles/librato.librato/tasks/main.yml | 3 + .../roles/librato.librato/tasks/memcached.yml | 9 + .../roles/librato.librato/tasks/mongodb.yml | 9 + .../roles/librato.librato/tasks/mysql.yml | 9 + .../roles/librato.librato/tasks/nginx.yml | 9 + .../librato.librato/tasks/nginx_plus.yml | 9 + .../librato.librato/tasks/postgresql.yml | 9 + .../roles/librato.librato/tasks/redis.yml | 9 + .../roles/librato.librato/tasks/repo.yml | 76 +++ .../roles/librato.librato/tasks/varnish.yml | 9 + .../roles/librato.librato/tasks/zookeeper.yml | 9 + .../templates/apache.conf.jinja | 12 + .../templates/collectd.conf.jinja | 23 + .../templates/docker.conf.jinja | 4 + .../templates/elasticsearch.conf.jinja | 17 + .../templates/haproxy.conf.jinja | 12 + .../librato.librato/templates/jvm.conf.jinja | 51 ++ .../templates/librato.conf.jinja | 11 + .../templates/logging.conf.jinja | 37 ++ .../templates/memcached.conf.jinja | 7 + .../templates/mongodb.conf.jinja | 23 + .../templates/mysql.conf.jinja | 23 + .../templates/nginx.conf.jinja | 4 + .../templates/nginx_plus.conf.jinja | 11 + .../templates/postgresql.conf.jinja | 34 ++ .../templates/redis.conf.jinja | 8 + .../templates/varnish.conf.jinja | 14 + .../templates/zookeeper.conf.jinja | 5 + .../test/integration/default/default.yml | 55 ++ .../default/serverspec/agent_spec.rb | 28 + .../default/serverspec/apache_spec.rb | 8 + .../default/serverspec/docker_spec.rb | 8 + .../default/serverspec/elasticsearch_spec.rb | 8 + .../default/serverspec/haproxy_spec.rb | 8 + .../default/serverspec/jvm_spec.rb | 10 + .../default/serverspec/memcached_spec.rb | 9 + .../default/serverspec/mongodb_spec.rb | 10 + .../default/serverspec/mysql_spec.rb | 10 + .../default/serverspec/nginx_plus_spec.rb | 8 + .../default/serverspec/nginx_spec.rb | 8 + .../default/serverspec/postgresql_spec.rb | 11 + .../default/serverspec/redis_spec.rb | 9 + .../default/serverspec/repo_spec.rb | 1 + .../default/serverspec/varnish_spec.rb | 7 + .../default/serverspec/zookeeper_spec.rb | 9 + .../vars/clojurians_log_secrets.yml.gpg | Bin 5480 -> 5558 bytes 71 files changed, 1619 insertions(+), 1 deletion(-) create mode 100644 clojurians_log/playbooks/roles/librato.librato/.gitignore create mode 100644 clojurians_log/playbooks/roles/librato.librato/.kitchen.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/Ansiblefile create mode 100644 clojurians_log/playbooks/roles/librato.librato/Gemfile create mode 100644 clojurians_log/playbooks/roles/librato.librato/README.md create mode 100644 clojurians_log/playbooks/roles/librato.librato/defaults/main.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/collectd.pref create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/cpu.conf create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/df.conf create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/disk.conf create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/load.conf create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/memory.conf create mode 100644 clojurians_log/playbooks/roles/librato.librato/files/swap.conf create mode 100644 clojurians_log/playbooks/roles/librato.librato/handlers/main.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info create mode 100644 clojurians_log/playbooks/roles/librato.librato/meta/main.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/requirements.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/main.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb create mode 100644 clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index cc15425..3d760c1 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -24,9 +24,13 @@ datomic_object_cache_max: 2g datomic_transactor_max_heap: 3584m clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" + JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211 -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264 " tasks: + - name: Set up librato first so we have eyes + import_role: librato.librato + vars: + librato_enabled_plugins: ['nginx', 'jvm', 'memcached'] # Keys first, then repos, then the dist-upgrade. - name: Add repository for ansible @@ -238,6 +242,9 @@ state: "present" nginx_access_log: off extra_parameters: | + location = /basic_status { + stub_status; + } location / { root {{ clojure_app_app_dir }}/resources/public; try_files $uri @proxy_app; diff --git a/clojurians_log/playbooks/roles/librato.librato/.gitignore b/clojurians_log/playbooks/roles/librato.librato/.gitignore new file mode 100644 index 0000000..58d5086 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/.gitignore @@ -0,0 +1,5 @@ +tmp/ +.kitchen/ +.librarian/ +Ansiblefile.lock +hosts diff --git a/clojurians_log/playbooks/roles/librato.librato/.kitchen.yml b/clojurians_log/playbooks/roles/librato.librato/.kitchen.yml new file mode 100644 index 0000000..e96da29 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/.kitchen.yml @@ -0,0 +1,102 @@ +driver: + name: vagrant + +provisioner: + name: ansible_playbook + require_ansible_repo: true + ansible_verbose: true + require_chef_for_busser: false + require_ruby_for_busser: true + hosts: hosts + roles_path: ../ + +verifier: + ruby_bindir: '/usr/bin' + +platforms: + - name: centos-6 + driver: + box: centos/6 + - name: centos-7 + driver: + box: centos/7 + - name: fedora-23 + driver: + box: fedora/23-cloud-base +# Due to a bug in kitchen-ansible, statically set the ansible_platform to Amazon +# See: https://github.com/neillturner/kitchen-ansible/issues/216 + - name: amazonlinux-2016.03 + driver: + name: ec2 + image_id: ami-7172b611 + aws_ssh_key_id: test-kitchen + region: us-west-2 + availability_zone: a + instance_type: t2.large + associate_public_ip: true + interface: dns + transport: + username: ec2-user + ssh_key: ~/.ssh/id_rsa + provisioner: + ansible_platform: 'amazon' + require_chef_for_busser: true + require_ruby_for_busser: false + verifier: + ruby_bindir: '/opt/chef/embedded/bin' + - name: amazonlinux-2016.09 + driver: + name: ec2 + image_id: ami-5ec1673e + aws_ssh_key_id: test-kitchen + region: us-west-2 + availability_zone: a + instance_type: t2.large + associate_public_ip: true + interface: dns + transport: + username: ec2-user + ssh_key: ~/.ssh/id_rsa + provisioner: + ansible_platform: 'amazon' + require_chef_for_busser: true + require_ruby_for_busser: false + verifier: + ruby_bindir: '/opt/chef/embedded/bin' +# Ansible doesn't provide Debian repos so we use the omnibus installer to provide Ansible + - name: debian-7 + driver: + box: debian/wheezy64 + provisioner: + require_ansible_omnibus: true + require_ansible_repo: false + - name: debian-8 + driver: + box: debian/jessie64 + provisioner: + require_ansible_omnibus: true + require_ansible_repo: false +# Ubuntu 12.04 uses Ruby 1.8.7 while busser requires Ruby 1.9+, so we need to use Chef to run busser + - name: ubuntu-12.04 + driver: + box: ubuntu/precise64 + provisioner: + require_chef_for_busser: true + require_ruby_for_busser: false + verifier: + ruby_bindir: '/opt/chef/embedded/bin' + - name: ubuntu-14.04 + driver: + box: ubuntu/trusty64 + - name: ubuntu-15.04 + driver: + box: larryli/vivid64 + - name: ubuntu-15.10 + driver: + box: ubuntu/wily64 + - name: ubuntu-16.04 + driver: + box: ubuntu/xenial64 + +suites: + - name: default diff --git a/clojurians_log/playbooks/roles/librato.librato/Ansiblefile b/clojurians_log/playbooks/roles/librato.librato/Ansiblefile new file mode 100644 index 0000000..59c70e2 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/Ansiblefile @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +#^syntax detection + +site "https://galaxy.ansible.com/api/v1"; diff --git a/clojurians_log/playbooks/roles/librato.librato/Gemfile b/clojurians_log/playbooks/roles/librato.librato/Gemfile new file mode 100644 index 0000000..3b9bd23 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +gem 'kitchen-ansible' +gem 'kitchen-vagrant' +gem 'kitchen-verifier-serverspec' diff --git a/clojurians_log/playbooks/roles/librato.librato/README.md b/clojurians_log/playbooks/roles/librato.librato/README.md new file mode 100644 index 0000000..1b47390 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/README.md @@ -0,0 +1,559 @@ +# Ansible Role: Librato + +An Ansible role that installs and configures the Librato Agent and plugins + +## Requirements + +None + +## Dependencies + +None + +## Quickstart + +The bare minimum required to get started is to set the `email` and `token` variables and include the role: + +```yaml +- hosts: all + roles: + - { role: librato.librato, librato_email: 'you@domain.com', librato_token: 'your-token-here' } +``` + +By default, the following plugins are enabled: `cpu`, `df`, `disk`, `swap`, `memory`, `load`. + +To include a plugin, set any variables you need for it, then add it to the `librato_enabled_plugins` variable: + +```yaml +- hosts: all + vars: + librato_apache_path: '/my-status-endpoint' + librato_enabled_plugins: ['apache'] +``` + +### Supported Plugins + +* `cpu` +* `df` +* `memory` +* `load` +* `disk` +* `swap` +* `apache` +* `nginx` +* `nginx_plus` +* `jvm` +* `memcached` +* `varnish` +* `zookeeper` +* `docker` +* `elasticsearch` +* `mongodb` +* `postgresql` +* `mysql` +* `redis` +* `haproxy` + +## Usage + +### Using custom or upstream collectd plugins + +To use your own custom or upstream collectd plugin, simply have another module drop the config into `/opt/collectd/etc/collectd.conf.d/` and any custom plugins into `/opt/collectd/share/collectd/`. + +## Reference + +### Plugin: `apache` + - `librato_apache_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_apache_host` + + **Type**: string + + The hostname to use. Defaults to `localhost`. + - `librato.apache.path` + + **Type**: string + + The path to the status page. Defaults to `/server-status`. `?auto` is automatically appended, so no need to include it. + + - `librato_apache_user` + + **Type**: string + + The username to use for password-protected status pages. Defaults to empty. + - `librato_apache_password` + + **Type**: string + + The password to use for password-protected status pages. Defaults to empty. + +### Plugin: `docker` + - `librato_docker_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_docker_host` + + **Type**: string + + The Docker hostname. Defaults to `localhost`. + + - `librato_docker_port` + + **Type**: string + + The Docker port. Defaults to `2735` + +### Plugin: `elasticsearch` + + - `librato_elasticsearch_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_elasticsearch_host` + + **Type**: string + + The ElasticSearch hostname. Defaults to `localhost`. + + - `librato_elasticsearch_port_` + + **Type**: string + + The ElasticSearch port. Defaults to `9200`. + + - `librato_elasticsearch_cluster_name` + + **Type**: string + + The ElasticSearch cluster name, if set. Defaults to `nil`. + + - `librato_elasticsearch_verbose` + + **Type**: true/false + + Verbosity trigger of the plugin. Defaults to `true`. + +### Plugin: `haproxy` + - `librato_haproxy_socket_file` + + **Type**: string + + The HAProxy socket file. Defaults to `/run/haproxy/admin.sock`. + + - `librato_haproxy_proxies` + + **Type**: array + + The default proxies to collect. Defaults to `server`, `frontend`, `backend`. + +### Plugin: `jvm` + - `librato_jvm_host` + + **Type**: string + + The JVM host. Defaults to `localhost`. + + - `librato_jvm_service_url` + + **Type**: string + + The JVM service URL to query. Defaults to `service:jmx:rmi:///jndi/rmi://localhost:17264/jmxrmi`. + + - `librato_jvm_mbeans` + + **Type**: mapping + + Additional mbeans to collect. Defaults to empty. + + Format of the mapping is: + ```yaml + librato_jvm_mbeans: [ + { + name: 'mbean name' + object_name: 'object name' + instance_prefix': 'instance prefix (optional)', + instance_from': 'instance from (optional)', + values: [ + { + type: 'value type', + table: true|false, + attribute: 'attribute' + instance_prefix: 'instance prefix (optional)', + instance_from: 'instance from (optional)' + } + ] + } + ] + ``` + +### Plugin: `memcached` + - `librato_memcached_host` + + **Type**: string + + The memcached hostname. Defaults to `localhost`. + + - `librato_memcached_port` + + **Type**: string + + The memcached port. Defaults to `11211`. + +### Plugin: `mongodb` + - `librato_mongodb_host` + + **Type**: string + + The MongoDB hostname. Defaults to `localhost`. + + - `librato_mongodb_port` + + **Type**: string + + The MongoDB port. Defaults to `27017`. + + - `librato_mongodb_user` + + **Type**: string + + The MongoDB username to connect with. Defaults to `nil`. + + - `librato_mongodb_password` + + **Type**: string + + The MongoDB password to connect with. Defaults to `nil`. + + - `librato_mongodb_databases` + + **Type**: array + + Databases to collect metrics for. Defaults to empty. `admin` database is automatically included in the array. + + - `librato_mongodb_name` + + **Type**: string + + Set the name of the plugin instance. Defaults to `mongodb`. + +### Plugin: `mysql` + - `librato_mysql_databases` + + **Type**: mapping + + Databases to collect metrics for. Defaults to empty. + + Format of the hash is: + ```yaml + databases: [ + { + name: 'mydb' + host: 'localhost' + port: 3306 + user: '' + password: '' + innodb_stats: true + slave_stats: false + } + ] + ``` + +### Plugin: `nginx` + - `librato_nginx_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_nginx_host` + + **Type**: string + + The hostname to use. Defaults to `localhost`. + + - `librato_nginx_path` + + **Type**: string + + The path to the status page. Defaults to `/basic_status`. + +### Plugin: `nginx_plus` + - `librato_nginx_plus_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_nginx_plus_host` + + **Type**: string + + The hostname to use. Defaults to `localhost`. + + - `librato_nginx_plus_path` + + **Type**: string + + The path to the status page. Defaults to `/status`. + + - `librato_nginx_plus_verbose` + + **Type**: string + + Verbosity on/off for the plugin. Defaults to `false`. + +### Plugin: `postgresql` + - `librato_postgresql_socket_file` + + **Type**: string + + The PostgreSQL socket file. Defaults to `/var/run/postgresql`. + + - `librato_postgresql_user` + + **Type**: string + + The PostgreSQL user to connect as. Defaults to `postgresql`. + + - `librato_postgresql_databases` + + **Type**: mapping + + The databases to collect metrics for. Defaults to empty. + + The format of the hash is: + ```yaml + databases = [ + { + name: 'mydb' + instance: 'baz' + host: 'localhost' + port: 5432 + user: '' + password: '' + ssl_mode: 'prefer' + } + ] + ``` + +### Plugin: `redis` + - `librato_redis_host` + + **Type**: string + + The Redis hostname. Defaults to `localhost`. + + - `librato_redis_port` + + **Type**: string + + The Redis port. Defaults to `6379`. + + - `librato_redis_timeout` + + **Type**: string + + The timeout for connecting to Redis in milliseconds. Defaults to `2000`. + +### Plugin: `varnish` + + Varnish has no configurable attributes. + +### Plugin: `zookeeper` + - `librato_zookeeper_host` + + **Type**: string + + The ZooKeeper hostname. Defaults to `localhost`. + + - `librato_zookeeper_port` + + **Type**: string + + The ZooKeeper port. Defaults to `2181`. + +### Global Variables + +- `librato_email` + + **Type**: string + + The email to use for sending metrics. Use in conjunction with `token`. This attribute is required and defaults to empty. + +- `librato_token` + + **Type**: string + + The API token to use for sending metrics. Use in conjunction with `email`. This attribute is required and defaults to empty. + +- `librato_version` + + **Type**: string + + The version of the Librato Agent to install. + +- `librato_repo_url` + + **Type**: string + + The base URL for the packages. Defaults to Librato's repo URL `https://packagecloud.io/librato/`. + +- `librato_repo_base` + + **Type**: string + + The repo base to use. Defaults to Librato's repo collection `librato-collectd`. + +- `librato_config_base` + + **Type**: string + + The base path for collectd's config files. Defaults to `/opt/collectd/etc`. + +- `librato_plugin_config_path` + + **Type**: string + + The path for collectd's plugin configs. Defaults to `/opt/collectd/etc/collectd.conf.d`. + +- `librato_hostname` + + **Type**: string + + The hostname to use for the node. Defaults to `nil` in favor of using `FQDNLookup`. + +- `librato_fqdn_lookup` + + **Type**: true/false + + Perform an FQDN lookup for the hostname. Defaults to `true`. + +- `librato_interval` + + **Type**: integer + + The global plugin polling interval in seconds. Defaults to `60`. + +- `librato_use_log_file` + + **Type**: true/false + + Write collectd logs to a file. Defaults to `true`. + +- `librato_use_syslog` + + **Type**: true/false + + Write collectd logs to syslog. Defaults to `false`. + +- `librato_use_logstash` + + **Type**: true/false + + Write collectd logs to a logstash-formatted file. Defaults to `false`. + +- `librato_log_file_log_level` + + **Type**: string + + The log level to use for `log_file`. Defaults to `info`. + +- `librato_log_file_filename` + + **Type**: string + + The filename to use for `log_file`. Defaults to `/opt/collectd/var/log/collectd.log`. + +- `librato_log_file_timestamp` + + **Type**: true/false + + Use timestamps in the log file or not. Defaults to `true`. + +- `librato_log_file_print_severity` + + **Type**: true/false + + Include severity levels in the log file or not. Defaults to `true`. + +- `librato_syslog_log_level` + + **Type**: string + + The log level to use for `syslog`. Defaults to `info`. + +- `librato_logstash_log_level` + + **Type**: string + + The log level to use for `logstash`. Defaults to `info`. + +- `librato_logstash_filename` + + **Type**: string + + The file name to use for `logstash`. Defaults to `/opt/collectd/var/log/collectd.json.log` + +- `librato_default_plugins` + + **Type**: array + + A list of default plugins to include. Defaults to: `cpu`, `df`, `disk`, `swap`, `memory`, `load`. + +- `librato_enabled_plugins` + + **Type**: array + + A list of plugins to enable. This is separate from the default plugins and is used to include plugins aside from those. Defaults to empty. + +### Supported Platforms + +* RHEL 6 / CentOS 6 +* RHEL 7 / CentOS 7 +* Fedora 23 +* Amazon Linux 2016.03 +* Ubuntu 12.04 +* Ubuntu 14.04 +* Ubuntu 15.04 +* Ubuntu 15.10 +* Ubuntu 16.04 +* Debian 7 +* Debian 8 + +### Supported Ansible Versions + +This role requires Ansible 2.2 or later. + +## Development + +### Testing + +Integration tests utilize `kitchen-ansible` and `serverspec`. To run the test suite: + +1. Run `bundle install` +2. Run `kitchen test` + +#### Testing Amazon Linux + +Testing Amazon Linux through `test-kitchen` requires a bit more setup: + +1. Ensure `kitchen-ec2` is installed: `gem install kitchen-ec2` +2. Update `.kitchen.yml` to have the correct AWS key ID you're going to use +3. Set `security_group_ids` in the driver section to include a security group accessible from your laptop. Not setting this will use the `default` security group. +4. Set `transport.ssh_key` to the path of your SSH key. It looks for `id_rsa` by default. + +## License + +MIT + +## Authors & Contributors +Mike Julian (@mjulian) diff --git a/clojurians_log/playbooks/roles/librato.librato/defaults/main.yml b/clojurians_log/playbooks/roles/librato.librato/defaults/main.yml new file mode 100644 index 0000000..07b2496 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/defaults/main.yml @@ -0,0 +1,73 @@ +librato_email: '' +librato_token: '' +librato_repo_url: 'https://packagecloud.io/librato/' +librato_repo_base: 'librato-collectd' +librato_deb_version: '5.5.0-librato51.251' +librato_rh_version: '5.5.0_librato51.251' +librato_config_base: '/opt/collectd/etc' +librato_plugin_config_path: '/opt/collectd/etc/collectd.conf.d' +librato_fqdn_lookup: true +librato_interval: 60 +librato_default_plugins: ['cpu', 'df', 'disk', 'swap', 'memory', 'load'] +librato_enabled_plugins: [] + +librato_logging_use_log_file: true +librato_logging_use_syslog: false +librato_logging_use_logstash: false +librato_logging_log_file_log_level: 'info' +librato_logging_log_file_filename: '/opt/collectd/var/log/collectd.log' +librato_logging_log_file_timestamp: true +librato_logging_log_file_print_severity: false +librato_logging_syslog_log_level: 'info' +librato_logging_logstash_log_level: 'info' +librato_logging_logstash_filename: '/opt/collectd/var/log/collectd.json.log' + +librato_apache_protocol: 'http' +librato_apache_host: 'localhost' +librato_apache_path: '/server-status' + +librato_nginx_protocol: 'http' +librato_nginx_host: 'localhost' +librato_nginx_path: '/basic_status' + +librato_nginx_plus_protocol: 'http' +librato_nginx_plus_host: 'localhost' +librato_nginx_plus_path: '/status' +librato_nginx_plus_verbose: false + +librato_memcached_host: 'localhost' +librato_memcached_port: '11211' + +librato_elasticsearch_protocol: 'http' +librato_elasticsearch_host: 'localhost' +librato_elasticsearch_port: '9200' +librato_elasticsearch_verbose: true + +librato_mongodb_host: 'localhost' +librato_mongodb_port: '27017' +librato_mongodb_databases: [] +librato_mongodb_name: 'mongodb' + +librato_zookeeper_host: 'localhost' +librato_zookeeper_port: '2181' + +librato_haproxy_socket_file: '/run/haproxy/admin.sock' +librato_haproxy_proxies: ['server', 'frontend', 'backend'] + +librato_redis_host: 'localhost' +librato_redis_port: '6379' +librato_redis_timeout: '2000' + +librato_mysql_databases: [] + +librato_postgresql_databases: [] +librato_postgresql_socket_file: '/var/run/postgresql' +librato_postgresql_user: 'postgresql' + +librato_docker_protocol: 'http' +librato_docker_host: 'localhost' +librato_docker_port: '2735' + +librato_jvm_host: 'localhost' +librato_jvm_service_url: 'service:jmx:rmi:///jndi/rmi://localhost:17264/jmxrmi' +librato_jvm_mbeans: [] diff --git a/clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref b/clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref new file mode 100644 index 0000000..17d7d98 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref @@ -0,0 +1,3 @@ +Package: collectd-core +Pin: release l=librato-collectd +Pin-Priority: 1001 diff --git a/clojurians_log/playbooks/roles/librato.librato/files/collectd.pref b/clojurians_log/playbooks/roles/librato.librato/files/collectd.pref new file mode 100644 index 0000000..c34a0cb --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/collectd.pref @@ -0,0 +1,3 @@ +Package: collectd +Pin: release l=librato-collectd +Pin-Priority: 1001 diff --git a/clojurians_log/playbooks/roles/librato.librato/files/cpu.conf b/clojurians_log/playbooks/roles/librato.librato/files/cpu.conf new file mode 100644 index 0000000..6a403d0 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/cpu.conf @@ -0,0 +1,4 @@ +LoadPlugin cpu + + ReportByCpu false + diff --git a/clojurians_log/playbooks/roles/librato.librato/files/df.conf b/clojurians_log/playbooks/roles/librato.librato/files/df.conf new file mode 100644 index 0000000..cd0f609 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/df.conf @@ -0,0 +1,15 @@ +LoadPlugin df + + FSType "ext2" + FSType "ext3" + FSType "ext4" + FSType "xfs" + FSType "jfs" + FSType "btrfs" + FSType "reiserfs" + FSType "vboxsf" + FSType "tmpfs" + ReportInodes false + ValuesAbsolute false + ValuesPercentage true + diff --git a/clojurians_log/playbooks/roles/librato.librato/files/disk.conf b/clojurians_log/playbooks/roles/librato.librato/files/disk.conf new file mode 100644 index 0000000..db9bddb --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/disk.conf @@ -0,0 +1 @@ +LoadPlugin disk diff --git a/clojurians_log/playbooks/roles/librato.librato/files/load.conf b/clojurians_log/playbooks/roles/librato.librato/files/load.conf new file mode 100644 index 0000000..5a65eda --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/load.conf @@ -0,0 +1 @@ +LoadPlugin load diff --git a/clojurians_log/playbooks/roles/librato.librato/files/memory.conf b/clojurians_log/playbooks/roles/librato.librato/files/memory.conf new file mode 100644 index 0000000..2a1db34 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/memory.conf @@ -0,0 +1,5 @@ +LoadPlugin memory + + ValuesAbsolute true + ValuesPercentage false + diff --git a/clojurians_log/playbooks/roles/librato.librato/files/swap.conf b/clojurians_log/playbooks/roles/librato.librato/files/swap.conf new file mode 100644 index 0000000..9ac8d80 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/swap.conf @@ -0,0 +1 @@ +LoadPlugin swap diff --git a/clojurians_log/playbooks/roles/librato.librato/handlers/main.yml b/clojurians_log/playbooks/roles/librato.librato/handlers/main.yml new file mode 100644 index 0000000..c055939 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: collectd + service: + name: collectd + state: restarted diff --git a/clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info b/clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info new file mode 100644 index 0000000..b2a7238 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info @@ -0,0 +1 @@ +{install_date: 'Fri Aug 23 08:21:22 2019', version: 1.0.3} diff --git a/clojurians_log/playbooks/roles/librato.librato/meta/main.yml b/clojurians_log/playbooks/roles/librato.librato/meta/main.yml new file mode 100644 index 0000000..7e99b35 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/meta/main.yml @@ -0,0 +1,30 @@ +--- +galaxy_info: + author: librato + description: Install and configure the Librato Agent + company: Librato + license: MIT + min_ansible_version: 2.2 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Fedora + versions: + - 23 + - name: Ubuntu + versions: + - trusty + - precise + - xenial + - wily + - vivid + - name: Debian + versions: + - jessie + - wheezy + galaxy_tags: + - librato + - collectd + - monitoring diff --git a/clojurians_log/playbooks/roles/librato.librato/requirements.yml b/clojurians_log/playbooks/roles/librato.librato/requirements.yml new file mode 100644 index 0000000..41a6d0d --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/requirements.yml @@ -0,0 +1,3 @@ +- librato.librato +- name: role-public + src: https://github.com/librato/ansible-librato.git diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml new file mode 100644 index 0000000..261f566 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml @@ -0,0 +1,83 @@ +- name: Install package (Debian/Ubuntu) + apt: + name: collectd-core={{ librato_deb_version }} + state: present + notify: + - collectd + when: ansible_os_family == 'Debian' + +- name: Install package (RedHat/CentOS/Amazon) + yum: + name: collectd-core-{{ librato_rh_version }} + state: present + notify: + - collectd + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + +- name: Install package (Fedora) + dnf: + name: collectd-core-{{ librato_rh_version }} + state: present + notify: + - collectd + when: ansible_distribution == 'Fedora' + +- name: setup_collectd_config + template: + src: collectd.conf.jinja + dest: "{{ librato_config_base }}/collectd.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + +- name: setup_logging_plugin + template: + src: logging.conf.jinja + dest: "{{ librato_plugin_config_path }}/logging.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + +- name: setup_librato_plugin + template: + src: librato.conf.jinja + dest: "{{ librato_plugin_config_path }}/librato.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + +- name: Default plugins + copy: + src: "{{ item }}.conf" + dest: "{{ librato_plugin_config_path }}/{{ item }}.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + with_items: "{{ librato_default_plugins }}" + +- name: Additional enabled plugins + template: + src: "{{ item }}.conf.jinja" + dest: "{{ librato_plugin_config_path }}/{{ item }}.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + with_items: "{{ librato_enabled_plugins }}" + +- name: collectd_service + service: + name: collectd + state: started + enabled: yes diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml new file mode 100644 index 0000000..2a12b93 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml @@ -0,0 +1,9 @@ +- name: Apache plugin + copy: + src: "apache.conf.jinja" + dest: "{{ plugin_config_path }}/apache.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml new file mode 100644 index 0000000..b58b91b --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml @@ -0,0 +1,9 @@ +- name: Docker plugin + copy: + src: "docker.conf.jinja" + dest: "{{ plugin_config_path }}/docker.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml new file mode 100644 index 0000000..34d2643 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml @@ -0,0 +1,9 @@ +- name: Elasticsearch plugin + copy: + src: "elasticsearch.conf.jinja" + dest: "{{ plugin_config_path }}/elasticsearch.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml new file mode 100644 index 0000000..601ffbb --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml @@ -0,0 +1,9 @@ +- name: haproxy plugin + copy: + src: "haproxy.conf.jinja" + dest: "{{ plugin_config_path }}/haproxy.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml new file mode 100644 index 0000000..0d34b23 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml @@ -0,0 +1,9 @@ +- name: JVM plugin + copy: + src: "jvm.conf.jinja" + dest: "{{ plugin_config_path }}/jvm.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/main.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/main.yml new file mode 100644 index 0000000..9912c8b --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- include: repo.yml tags=librato +- include: agent.yml tags=librato diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml new file mode 100644 index 0000000..1ea14f5 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml @@ -0,0 +1,9 @@ +- name: memcached plugin + copy: + src: "memcached.conf.jinja" + dest: "{{ plugin_config_path }}/memcached.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml new file mode 100644 index 0000000..c21ae90 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml @@ -0,0 +1,9 @@ +- name: MongoDB plugin + copy: + src: "mongodb.conf.jinja" + dest: "{{ plugin_config_path }}/mongodb.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml new file mode 100644 index 0000000..7d07537 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml @@ -0,0 +1,9 @@ +- name: MySQL plugin + copy: + src: "mysql.conf.jinja" + dest: "{{ plugin_config_path }}/mysql.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml new file mode 100644 index 0000000..477b883 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml @@ -0,0 +1,9 @@ +- name: nginx plugin + copy: + src: "nginx.conf.jinja" + dest: "{{ plugin_config_path }}/nginx.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml new file mode 100644 index 0000000..9be1966 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml @@ -0,0 +1,9 @@ +- name: nginx_plus plugin + copy: + src: "nginx_plus.conf.jinja" + dest: "{{ plugin_config_path }}/nginx_plus.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml new file mode 100644 index 0000000..fd73e42 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml @@ -0,0 +1,9 @@ +- name: PostgreSQL plugin + copy: + src: "postgresql.conf.jinja" + dest: "{{ plugin_config_path }}/postgresql.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml new file mode 100644 index 0000000..399aad4 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml @@ -0,0 +1,9 @@ +- name: Redis plugin + copy: + src: "redis.conf.jinja" + dest: "{{ plugin_config_path }}/redis.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml new file mode 100644 index 0000000..96260a6 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml @@ -0,0 +1,76 @@ +- name: Install EPEL repo (RedHat/CentOS/Amazon) + yum: + name: epel-release + state: present + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + +- name: Install gpg key + apt_key: + url: http://packagecloud.io/gpg.key + id: 418A7F2FB0E1E6E7EABF6FE8C2E73424D59097AB + state: present + when: ansible_os_family == 'Debian' + +- name: Install apt-transport-https + apt: + name: apt-transport-https + state: present + when: ansible_os_family == 'Debian' + +- name: Create apt repo + apt_repository: + repo: 'deb {{ librato_repo_url }}{{ librato_repo_base }}/{{ ansible_distribution|lower }}/ {{ ansible_distribution_release }} main' + state: present + when: ansible_os_family == 'Debian' + +- name: collectd_apt_pin + copy: + src: collectd.pref + dest: /etc/apt/preferences.d/collectd.pref + owner: root + group: root + mode: 0644 + when: ansible_os_family == 'Debian' + +- name: collectd-core_apt_pin + copy: + src: collectd-core.pref + dest: /etc/apt/preferences.d/collectd-core.pref + owner: root + group: root + mode: 0644 + when: ansible_os_family == 'Debian' + +- name: Run apt-get update + apt: + update_cache: yes + when: ansible_os_family == 'Debian' + +- name: Create yum repo + yum_repository: + name: 'librato_{{ librato_repo_base }}' + description: 'librato_{{ librato_repo_base }}' + baseurl: '{{ librato_repo_url }}{{ librato_repo_base }}/el/$releasever/$basearch' + gpgcheck: no + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + - ansible_distribution != 'Amazon' + +- name: Create yum repo (Fedora) + yum_repository: + name: 'librato_{{ librato_repo_base }}' + description: 'librato_{{ librato_repo_base }}' + baseurl: '{{ librato_repo_url }}{{ librato_repo_base }}/fedora/$releasever/$basearch' + gpgcheck: no + when: ansible_distribution == 'Fedora' + +- name: Create yum repo (Amazon Linux) + yum_repository: + name: 'librato_{{ librato_repo_base }}' + description: 'librato_{{ librato_repo_base }}' + baseurl: '{{ librato_repo_url }}librato-amazonlinux-collectd/el/6/$basearch' + gpgcheck: no + when: ansible_distribution == 'Amazon' diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml new file mode 100644 index 0000000..d6883df --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml @@ -0,0 +1,9 @@ +- name: Varnish plugin + copy: + src: "varnish.conf.jinja" + dest: "{{ plugin_config_path }}/varnish.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml new file mode 100644 index 0000000..d7d0db1 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml @@ -0,0 +1,9 @@ +- name: ZooKeeper plugin + copy: + src: "zookeeper.conf.jinja" + dest: "{{ plugin_config_path }}/zookeeper.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja new file mode 100644 index 0000000..4ea621e --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja @@ -0,0 +1,12 @@ +LoadPlugin "apache" + + + URL "{{ librato_apache_protocol }}://{{ librato_apache_host }}{{ librato_apache_path }}?auto" +{% if librato_apache_user is defined %} + User "{{ librato_apache_user }}" +{% endif %} +{% if librato_apache_password is defined %} + Password "{{ librato_apache_password }}" +{% endif %} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja new file mode 100644 index 0000000..64b76d3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja @@ -0,0 +1,23 @@ +# This file is managed by Ansible. Local changes will be overwritten. +# +# +# Config file for collectd(1). +# Please read collectd.conf(5) for a list of options. +# http://collectd.org/ +# + +############################################################################## +# Global # +#----------------------------------------------------------------------------# +# Global settings for the daemon. # +############################################################################## + +{% if librato_hostname is defined %} +Hostname "{{ librato_hostname }}" +{% endif %} +FQDNLookup {{ librato_fqdn_lookup }} +Interval {{ librato_interval }} + +# Include logging config before plugins collectd logs plugin log entries +Include "{{ librato_plugin_config_path }}/logging.conf" +Include "{{ librato_plugin_config_path }}/*.conf" diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja new file mode 100644 index 0000000..8692167 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja @@ -0,0 +1,4 @@ +LoadPlugin exec + + Exec "nobody" "/opt/collectd/share/collectd/collectd-docker.py" "{{ librato_docker_protocol }}://{{ librato_docker_host }}:{{ librato_docker_port }}" + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja new file mode 100644 index 0000000..4daf697 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja @@ -0,0 +1,17 @@ + + Globals true + + + + ModulePath "/opt/collectd/share/collectd/" + + Import "collectd-elasticsearch" + + + Url "{{ librato_elasticsearch_protocol }}://{{ librato_elasticsearch_host }}:{{ librato_elasticsearch_port }}" + {% if librato_elasticsearch_cluster_name is defined %} + Name "{{ librato_elasticsearch_cluster_name }}" + {% endif %} + Verbose {{ librato_elasticsearch_verbose }} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja new file mode 100644 index 0000000..e907445 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja @@ -0,0 +1,12 @@ +LoadPlugin python + + ModulePath "/opt/collectd/share/collectd" + Import "collectd-haproxy" + + + Socket "{{ librato_haproxy_socket_file }}" + {% for proxy in librato_haproxy_proxies %} + ProxyMonitor "{{ proxy }}}" + {% endfor %} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja new file mode 100644 index 0000000..17784a3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja @@ -0,0 +1,51 @@ +LoadPlugin java + +Include "/opt/collectd/share/collectd/jvm_default.conf" + + + + + Host "{{ librato_jvm_host }}" + ServiceURL "{{ librato_jvm_service_url }}" + Collect "memory_heap" + Collect "memory_nonheap" + Collect "classes" + Collect "threads" + Collect "compilation" + Collect "garbage_collector" + Collect "memory_pool" + {% if librato_jvm_mbeans is iterable %} + {% for mbean in librato_jvm_mbeans %} + Collect "{{ mbean.name }}" + {% endfor %} + {% endif %} + +{% if librato_jvm_mbeans is iterable %} + {% for mbean in librato_jvm_mbeans %} + + ObjectName "{{ mbean.object_name }}" + {% if 'instance_prefix' in mbean %} + InstancePrefix "{{ mbean.instance_prefix }}" + {% endif %} + {% if 'instance_from' in mbean %} + InstanceFrom "{{ mbean.instance_from }}" + {% endif %} + + {% for value in mbean['values'] %} + + Type "{{ value.type }}" + Table {{ value.table }} + Attribute "{{ value.attribute }}" + {% if 'instance_prefix' in value %} + InstancePrefix "{{ value.instance_prefix }}" + {% endif %} + {% if 'instance_from' in value %} + InstanceFrom "{{ value.instance_from }}" + {% endif %} + + {% endfor %} + + {% endfor %} +{% endif %} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja new file mode 100644 index 0000000..039eba3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja @@ -0,0 +1,11 @@ +LoadPlugin write_http + + + URL "https://collectd.librato.com/v1/measurements" + Format "JSON" + BufferSize 8192 + + User "{{ librato_email }}" + Password "{{ librato_token }}" + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja new file mode 100644 index 0000000..e07178c --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja @@ -0,0 +1,37 @@ +############################################################################## +# Logging # +#----------------------------------------------------------------------------# +# Plugins which provide logging functions should be loaded first, so log # +# messages generated when loading or configuring other plugins can be # +# accessed. # +############################################################################## + +{% if not librato_logging_use_log_file and not librato_logging_use_syslog and not librato_logging_use_logstash %} +### No logging configured +{% endif %} + +{% if librato_logging_use_log_file %} +LoadPlugin logfile + + + LogLevel {{ librato_logging_log_file_log_level }} + File "{{ librato_logging_log_file_filename }}" + Timestamp {{ librato_logging_log_file_timestamp }} + PrintSeverity {{ librato_logging_log_file_print_severity }} + +{% endif %} +{% if librato_logging_use_syslog %} +LoadPlugin syslog + + + LogLevel {{ librato_logging_syslog_log_level }} + +{% endif %} +{% if librato_logging_use_logstash %} +LoadPlugin log_logstash + + + LogLevel {{ librato_logging_logstash_log_level }} + File "{{ librato_logging_logstash_filename }}" + +{% endif %} diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja new file mode 100644 index 0000000..6460d2d --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja @@ -0,0 +1,7 @@ +LoadPlugin memcached + + + Host "{{ librato_memcached_host }}" + Port "{{ librato_memcached_port }}" + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja new file mode 100644 index 0000000..1078511 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja @@ -0,0 +1,23 @@ + + Globals true + + + + ModulePath "/opt/collectd/share/collectd" + ModulePath "/opt/collectd/share/collectd/pymongo.egg" + + Import "collectd-mongodb" + + + Host "{{ librato_mongodb_host }}" + Port "{{ librato_mongodb_port }}" + {% if librato_mongodb_user is defined %} + User "{{ librato_mongodb_user }}" + {% endif %} + {% if librato_mongodb_password is defined %} + Password "{{ librato_mongodb_password }}" + {% endif %} + Database "admin", {{ '\"' + librato_mongodb_databases|join('\", \"') + '\"' }} + Name "{{ librato_mongodb_name }}" + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja new file mode 100644 index 0000000..1854e77 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja @@ -0,0 +1,23 @@ +LoadPlugin mysql + +{% for db in librato_mysql_databases %} + + Host "{{ db.host }}" + User "{{ db.user }}" + {% if 'password' in db %} + Password "{{ db.password }}" + {% endif %} + {% if 'port' in db %} + Port {{ db.port }} + {% else %} + Port 3306 + {% endif %} + {% if 'innodb_stats' in db %} + InnodbStats {{ db.innodb_stats }} + {% endif %} + {% if 'slave_stats' in db %} + SlaveStats {{ db.slave_stats }} + {% endif %} + +{% endfor %} + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja new file mode 100644 index 0000000..e1c6bfd --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja @@ -0,0 +1,4 @@ +LoadPlugin nginx + + URL "{{ librato_nginx_protocol }}://{{ librato_nginx_host }}{{ librato_nginx_path }}" + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja new file mode 100644 index 0000000..17688d8 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja @@ -0,0 +1,11 @@ +LoadPlugin python + + ModulePath "/opt/collectd/share/collectd" + Import "collectd-nginx_plus" + LogTraces true + + + URL "{{ librato_nginx_plus_protocol }}://{{ librato_nginx_plus_host }}{{ librato_nginx_plus_path }}" + Verbose {{ librato_nginx_plus_verbose }} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja new file mode 100644 index 0000000..fd30dfd --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja @@ -0,0 +1,34 @@ +LoadPlugin postgresql + + + Host "{{ librato_postgresql_socket_file }}" + User "{{ librato_postgresql_user }}" + Query connections + Query transactions + Query queries + Query query_plans + Query table_states + Query disk_io + Query disk_usage + +{% for db in librato_postgresql_databases %} + + Instance "{{ db.instance }}" + Host "{{ db.host }}" + {% if 'port' in db %} + Port "{{ db.port }}" + {% else %} + Port 5432 + {% endif %} + User "{{ db.user }}" + {% if 'password' in db %} + Password "{{ db.password }}" + {% endif %} + {% if 'ssl_mode' in db %} + SSLMode "{{ db.ssl_mode }}" + {% else %} + SSLMode "Prefer" + {% endif %} + +{% endfor %} + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja new file mode 100644 index 0000000..7b238c8 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja @@ -0,0 +1,8 @@ +LoadPlugin redis + + + Host "{{ librato_redis_host }}" + Port "{{ librato_redis_port }}" + Timeout {{ librato_redis_timeout }} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja new file mode 100644 index 0000000..7b27c2f --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja @@ -0,0 +1,14 @@ +LoadPlugin varnish + + + CollectBackend true + CollectCache true + CollectConnections true + CollectFetch true + CollectObjects true + CollectSession true + CollectSHM true + CollectTotals true + CollectWorkers true + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja new file mode 100644 index 0000000..bcbd12f --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja @@ -0,0 +1,5 @@ +LoadPlugin zookeeper + + Host "{{ librato_zookeeper_host }}" + Port "{{ librato_zookeeper_port }}" + diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml new file mode 100644 index 0000000..7265c37 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml @@ -0,0 +1,55 @@ +--- +- hosts: all + + vars: + librato_email: 'foo@bar.baz' + librato_token: '1234abcd' + librato_logging_use_log_file: true + librato_enabled_plugins: ['docker', 'haproxy', 'memcached', 'mysql', 'nginx', 'redis', 'varnish', 'apache', 'elasticsearch', 'jvm', 'mongodb', 'nginx_plus', 'postgresql', 'zookeeper'] + librato_jvm_mbeans: [ + { + 'name': 'foo', + 'object_name': 'objname', + 'instance_prefix': 'prefix', + 'instance_from': 'from', + 'values': [ + { + 'type': 'type', + 'table': true, + 'attribute': 'att' + }, + { + 'type': 'type', + 'table': true, + 'attribute': 'att' + } + ] + } + ] + librato_mongodb_databases: ['foo', 'bar', 'baz'] + librato_nginx_path: '/test-status' + librato_nginx_plus_path: '/test-status' + librato_postgresql_databases: [ + { + name: 'mydb', + instance: 'baz', + host: 'localhost', + port: 5432, + user: 'foo', + password: 'baz', + } + ] + librato_mysql_databases: [ + { + name: 'mydb', + host: 'localhost', + port: 3306, + user: 'foo', + password: 'baz', + innodb_stats: true, + } + ] + librato_apache_path: '/test-status' + + roles: + - ansible-librato diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb new file mode 100644 index 0000000..38d7092 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb @@ -0,0 +1,28 @@ +require 'serverspec' +set :backend, :exec + +describe package('collectd-core') do + it { should be_installed } +end + +describe service('collectd') do + it { should be_running } +end + +describe file('/opt/collectd/etc/collectd.conf.d/librato.conf') do + it { should exist } + it { should contain 'User "foo@bar.baz"' } + it { should contain 'Password "1234abcd"' } +end + +describe file('/opt/collectd/var/log/collectd.log') do + it { should exist } + it { should contain 'Initialization complete' } +end + +%w(cpu df disk swap memory load).each do |plugin| + describe file("/opt/collectd/etc/collectd.conf.d/#{plugin}.conf") do + it { should exist } + it { should contain "LoadPlugin #{plugin}" } + end +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb new file mode 100644 index 0000000..4c802d4 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/apache.conf') do + it { should exist } + it { should contain 'LoadPlugin "apache"' } + it { should contain 'URL "http://localhost/test-status?auto"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb new file mode 100644 index 0000000..47181f7 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/docker.conf') do + it { should exist } + it { should contain 'LoadPlugin exec' } + it { should contain 'Exec "nobody" "/opt/collectd/share/collectd/collectd-docker.py" "http://localhost:2735"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb new file mode 100644 index 0000000..5e6a1e3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/elasticsearch.conf') do + it { should exist } + it { should contain 'LoadPlugin "python"' } + it { should contain 'Url "http://localhost:9200"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb new file mode 100644 index 0000000..35dfc4f --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/haproxy.conf') do + it { should exist } + it { should contain 'LoadPlugin python' } + it { should contain 'Socket "/run/haproxy/admin.sock"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb new file mode 100644 index 0000000..c501ef5 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb @@ -0,0 +1,10 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/jvm.conf') do + it { should exist } + it { should contain 'LoadPlugin java' } + it { should contain 'ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:17264/jmxrmi"' } + it { should contain 'Collect "foo"' } + it { should contain '' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb new file mode 100644 index 0000000..9f85802 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb @@ -0,0 +1,9 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/memcached.conf') do + it { should exist } + it { should contain 'LoadPlugin memcached' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "11211"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb new file mode 100644 index 0000000..f9c9ab4 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb @@ -0,0 +1,10 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/mongodb.conf') do + it { should exist } + it { should contain 'LoadPlugin python' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "27017"' } + it { should contain 'Database "admin", "foo", "bar", "baz"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb new file mode 100644 index 0000000..93209ce --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb @@ -0,0 +1,10 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/mysql.conf') do + it { should exist } + it { should contain 'LoadPlugin mysql' } + it { should contain 'Database "mydb"' } + it { should contain 'User "foo"' } + it { should contain 'Password "baz"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb new file mode 100644 index 0000000..0e7ae2e --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/nginx_plus.conf') do + it { should exist } + it { should contain 'LoadPlugin python' } + it { should contain 'URL "http://localhost/test-status"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb new file mode 100644 index 0000000..2ccd9f1 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/nginx.conf') do + it { should exist } + it { should contain 'LoadPlugin nginx' } + it { should contain 'URL "http://localhost/test-status"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb new file mode 100644 index 0000000..fe45180 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb @@ -0,0 +1,11 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/postgresql.conf') do + it { should exist } + it { should contain 'LoadPlugin postgresql' } + it { should contain 'Database mydb' } + it { should contain 'Database postgres' } + it { should contain 'Instance "baz"' } + it { should contain 'Host "/var/run/postgresql"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb new file mode 100644 index 0000000..df3b965 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb @@ -0,0 +1,9 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/redis.conf') do + it { should exist } + it { should contain 'LoadPlugin redis' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "6379"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb new file mode 100644 index 0000000..1a6b679 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb @@ -0,0 +1 @@ +require 'serverspec' diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb new file mode 100644 index 0000000..afc6781 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb @@ -0,0 +1,7 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/varnish.conf') do + it { should exist } + it { should contain 'LoadPlugin varnish' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb new file mode 100644 index 0000000..3fd238a --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb @@ -0,0 +1,9 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/zookeeper.conf') do + it { should exist } + it { should contain 'LoadPlugin zookeeper' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "2181"' } +end diff --git a/clojurians_log/playbooks/vars/clojurians_log_secrets.yml.gpg b/clojurians_log/playbooks/vars/clojurians_log_secrets.yml.gpg index 7f2f64c472d4145e2ee70946368b355130c59287..973da12f900c6e6251deb26bcdc5d2d2360faa0d 100644 GIT binary patch literal 5558 zcmV;n6-nxa0t^EJzqJ9RjRdd(5C3A5#On;MBg^o3D)4~^U+H|T9arcK?WkYTFRlF9 z9v7`B!S`E*3sEF)CZ)a8I`&ypegDva`b=%IPwV4}KTx|ib9tPys=1AOcyLKbw=P;1 za;Sp;RzP|^xVZ(%okA*804$x4g4`StIly?^MWZeJW$mlJEuyWE&nmK9i^_XovaPA* z2xjBl7oRL7!fncg#G1T~`=})@9RsOvtSC!>n_FYanU8H`mxi}cRl@okxk^1;r5Jwy zWgpVj%eezw-=z~9>kFNv)3*5fQef$Z@@F0|O%p~92^aEQn#*k!EoOvCwlK&>a{*V+ zc?`v9Uob`2|KyWyj304>3%!zQI2$#=}*H5lMO z?VUJXQ%TDOhG65m@Kk{tvw>r0hW|uO(>`93KRjzpPD6Pa2&<%}Io=;{uW*#eS_)24 zM;a3N;pf9JTy~z_xK??%Ef}XGt|BFh3yo98=kI!dICo`Sz^&T-&Z*-WjH8xY1Xcu_TF1wZMqtZamY*trNl= z={7u3XIy;4NDW155ZncNT$|{#dBhs&Uf;J-H{?WqEinsJ*0gRmhz+Hp&$a;FVjd0pceBf$S$ky;6w-Oj*JglaKewd!%UYuR+4d}nBUMDX>ARln|KJ8gA5hDmtj>u0wN}d(l@6G$x=;G4^jjwoSCTG*%Fa| z_UtL0O+31rig%N!T)V=rbW0oe@c7UzwUa$fDBgm2--qTob%7&6&`nZde!xh=W^?Kj z9-_Sp3`E3KoAm?n+z#+A%wKT4s^1b;B>4NfwMr$Y4}>5Q+A8rf`7#289_7l~Nj4~` z<6!zW(+O+9;_b37t!;fl{*ZeHk`H@bZ!Io3yU05}GL}b^lEIIbkJy>M%YUokg`@q* zw(BiA5zb)FNkh~Lu@DM|ZdzFhri5Mt48j%(Hv#F#5&;K542-B1ee&gvG_TfRf%@y^ zI3XVWFI&TM7R8eQb`Q!@Fg4m81iAw5^S5EPk*2slxRdx>$;mtIri3plC2imfozu#E zrq<7A7qwuxfykgboP`1m13-7ibb76Ds{s%F7V5u)+Q-ry9xl930dX<~JN-13=|&6` zj~5T~1dAzz?@7%q4bNk;7MD#0Z?Q+$(4#q5T4ci%F~if_cmlFCFLR{D2XejaPr}L8 zF9H4q@ZyO`2#+ISZ<$*#Hdr6ChXpP$r#s<-TP!I0+XqyJzb=KyMDO6%tYW?YiK-j(m`)j&CJy7QzqwhP?yMQt=eNTEJ5|)hYGd3!;Mn(zBZ3!o zbSy7T+S20ZJmDhq84iT)jF(;W5|wr&z(SkwC(O_8)^LF=+PHXeciPwk=IE<4bE!zmf=v*X*}46Z3J1EpuZ1M1re7q#|nN3mxCr2G1Q#=vA^HXG?Jzc=V0oyPu~ESF3oLaco0QedqEY% z*{w80^PfS{-LDAjkvWb&M|>Irl6MK>Gl@*J1l zx^;NT=xHRUhFg3=mwr1#fJL&;_aJ_;y zQqd;^NAx!r&9Z}->BI!rFGT9d((3^wPaWHMz^WYbdBh)`Go|YNI){xM*bg$IyQxb7 z=#X7(KV@_F8=1d%WPF0^X|xNhOb;qRg=Qd=AA#Vt>GD~b!TDTHfs>EhYf#k$tcT`h ziXdBnnI@F_lM*5AD$@}ydD`fZbipi3(zg`2QmNxGblAr*O4zrBpJe`Rb8(n7yv>I$ za?+*28+u~2H?&i~p;z;W%y8d|8QS8fulxdmz0&KNjq&yyH z{tZ*{#!A>fy%ZG?<#rnbycGANL6=t*@$Tj@X=vno-*Mt)`X3Q0x4ffj=K)S^FU+5G zEm1CkvaEm6dL3P%Im#4Br1ducOa6#Yf`LVK1wh-vN|}Gqx=rPOA z$)noy-ZxgY{I+LdcYL;!NC4lw{o2u1$xeDQ5e=qn!ve-FJ0gc(tp!ZZzFg2#=Ky3MwcE*}} zk0?UofC7pvPos zOkWm?G|y)^!$r{;RfuZ>5W6BU(nfD3_yvU_hBW8cF0v&$z^|CZf(TNC$6C7Aiq~R{ zXL4YonF=f7FHL*T1TsylF@0z7)D3k-ZrK=@1k$>zjBx1*Mu&39`b5(W>OFn|-N&UiNNr?5CNHrGy(xA4w@;<-KaX1BzaR_TPfjK1d8fe}Egr&(GZC9?M<1 zgUAO<_)|vjezWQeIdUp~dg104ic<`(y_Pu$bRk z$)aUSs&b*y^D&_OHSq%DvKfQYgSU$*lw!glVd!>O(nLM919>D!Qx64QNo%mxrhj2Y0qJSk&k*fh^4I76lAd_s@=EgUb zD`6+*CRfr13wa;@Rt@AIt{<@OU1__IJ01XbXSHc!055CTJL%XgJpNYrZ8I^tk45N+jp9V z^i!0s2kxnMN=n%lqNo_G4#o3|I6X9m#50+7dXT4xIv1UHuGabbypTps+PFG@_kFlh zg2Y>K=YI&TiCav*%Epc~4t%~_Nr;HGd%+%PH3Ixws!Y;$cUt0~)!XW{$wy*Ucm_~` zv7LSs#OUIpmXp5C#xfjB*p=hH!416v-TYMCcXD%=9NJ}{9*F6QNmvO(9VxJ)7KfJ* zlxgARS;Dsw6Vgnn>?&kGGR{>v4I)2RFu}18^@I=mD6UZvk_%%C|IHc!%YdT%)1>4% zo4OBE^btu~r~JHF zMOV_DH`X0#zb&Y5u&xHV%0jiR--88evxx#S_J*gZC@-0HtQ0i>Yce@0XBt?LY;abA ze=j83hi+AWFnB#C(9I`D!OxN*J0;ZuaIv}O;zb9G8^#A||i z6nveQn3bM4ZO-^4XfJa&D0R%&dd@9q z4CJGMS=rW zwyGrQ5}f5AL?3%<-_r!PR86FyS!O9PXOUd&rYsrd(kf}^*{7qd8?lgiNZplsYC8 zRuL%nD{;S|9eYZDn0M49MPABT$EW!FbU627B~Cv{g81E8WThHJwiss$7Y5}?Adko<0TUN@RqGVa(4oV@bk* zM#~jSxx_tZ4H-8TMc~ReLGAh|W0Db?`Z+q=`s+WF-r>8zmb{5p0xHcVsHJp5Hwmt> z%m|5{IZbM*GL@tJFGy>U4#loPu|;_pR%vi40P(Lqqc+r5cQzME{NCdRoD=->o;d6a zLRKG$S+R7#qAMASW1gYgk{OWCHM@g8U8djZ0yVvFqhCz*OvvYgG^TX%>3vD~9l&*o z>-AFc_ePL8cWU!@w6u^a*W@@s7Bbj6+y1lX`n`4UM$;`?1Rp9M_RT5%;E~u(nXeoX z`+DyE9#abhW3Z(6l}WMubDG2_N9P;w?$@@V(i)DaOE zr_~6itib?nn}+8IWGxa~Xvp~Yz~28%&jDX;;*U}2fT15l04`&OniOg_5GAMoTX|z# znJJ>Gl)oYh-c=D0?Wc11Bc4WDmAa=di9vfs)yU(tIER}49W z{Mq1)rRhVCjq3v>VCZ#DH#xA_z+U<8>%d@1B@bhV`D`0%io&k?2A?mSe)@qFG#)0g_wUjFOZ}Wfo z$9X;4uzV1cO=Vo;0P8_%{6*nDU?8|r=?x literal 5480 zcmV-u6_@ISUIPrm76~^2>BkZQ2S5#a?s4-!3U6lLYu<4j&qVW;fo6PTEsDei%@p3} zHAgUq=$ZS_3u%hDWGJ}Ilf?4YnIM6M$fSJ_bz&NcfeVP3A;qq1FQoPu0;Sqz z&OAOKl8HbVDf{NweK0QBEf5Mv22)pvSbXBKJ|w`g{D2L;I9f01E22b(<2)s%%{HxB zIF`LHV!!|1{sD?>Rrx-mW!iay-Y#=ioSf!_pFm{s(;DJ*S8+ zx0;z=@^-SOGmCb-hiaEI9~ag36%2G?Xi-wC* zx(ZHwMeXpNq;1^v_CI%2nzyW4s*+_}_88?6=Q8o`84Lj-Dc3V9npRWBxqbBmD+VbE zo`}<5&%kDb1O%RLJ35y=OooBBCg2Gyf0MZkStKH0XB1|{*zX^%PHf{aaQazq+zBxN zllxn#+5EZcuSlu3RK%_i5)x-L>K+xB(Tpl+y4`gkj{x@eWaHTP!h`gc%lVbW>jB~D znoAD@;xy$~O||1g;fa;nF_}g3g?j5JP(EHelKz|{ic@%iE8}1~SqQs=fm)#)0|x&7 z{m|2YtDU8%pKR-!3#mtr!QEplp}#!QBK}n|OBq$_)qR~TTkq$xSmO+GO?C! z>Bq1{(at8nmJ0qu4+!@=aaGMW52l885Fy6vum3@L74sRRC32b@AN1D|{*3_o5p>%G zlv@xk^^K}=&aV$Cg1h}LE+dId>6*OA9BMwE1W5R{ldhI7Hhs27>I!LB(l-}8afved zbohs8R@dG?*Y{~e4`+jTP!^PVeX9NeNyqR=^6&g*uU~3)t+ZAFvQ@-tjA1UUCMk}E zR^){|AFKAT<6!C32m@;$S}{k;ie;tj%*oF z0{xVrR_2dVv0fKsq=f;;$QvUh9AP0R!p6A+4exub^5d)xoLe{yZ zw1zo669gx*9V(2jcv|~()THhX-yi52p3XuLh#~$whypBo`XH8?%ZR^Htt?G$TPGmn za<&$@I*Jt`t&R)ggMiQ724n7KZmYK~P)!d|+L;c)gPSQw9%1NNiNcSKPRR=wwl=T$ zgFWd1GSZPj`{QreU8pq|XkfdKSkzoAyEpz@E6scy{7&U4aB8@&VcM!t4I*r0e+Pa#oMXg{^i?f2tN`b3*812LCtg(eMYJ(n1*rZ1h~8y}KX8i~ zFA9eWk!u2dsgDj_uq{{;4)N+BsPN#uHbbWyY*kO)pSW+fxFRFj>U;`=8KT!wZWG*f zAVz?5t24udSZbI$LMayhZ6E~~wM!=Tku##`l&1dq)%~E>0_Z*wb0v0eba`>*Xdot+ zt2{U6CQ->#5WyqV{>3-KK`w(q<2DQ!=&E>)|9&%l1qfMDEbQxd@=+`Kv!^ry{bjAalb6~Q{{B7Y2l-!UCkl(C z2N&59b(V5NEyuu83>B#1`*sC+5Dz(;D3gRZ9d{hAB0#{~!_)W;9>Y?1r-)riqvx_k zL}tq`^4N}TG$TOjk?xrGX(XEtyIlPEv>FD`Q*lj5#jGAAwlnV+&mtiPxLAs3wwOkf z2(%0^D#-^V;tn>5U0`xt!$}q$<9P6UR=^hX=y{4Epelh8zHjo6KZ1K6&2^>bJWhiP zzftb!P>^8A-ikWNC#pot?s>lhJahQjbrwHJ7>Dc!006&{Y9}3`^fIDck2Tv7Os(qU zmzyj7yJD^(U@aBS;7oz;Gyu)6`%X^I*ksnpvP))OssO`I{* zX2o#BYR_HLwYaV{j_vuQdO1{3`%*akr`TFS9pE4XN0^+dI-GeHX{ZWP4C|Xe><4AG zSsM1GqHEExiVOIO<{h5q`^7?0cgj~Trzg>c@t=`R8z}K45dqeVEbR||*dlqWdbaUO z@%1hI&V4RrmX`k;4!SmPp(n`D9Ve-7J9x(Pwr=b5hy2%tMIWe%r<+Xo%$C$aB(19H~;S#`W7M=b+`mNe}`&b ztf9}50vd4m);E$#gj27YbOICbgilp&18psp#UCvUOqlv6FrVn?XHIbz=>0aDmWY|y z)Z4*+fIzoC`-2Zg>@03I$zBAo;6Tj$cymfCMf`Pse4w$w;_UItQ>C+uCrM963$ocN zXL~v8ePq1^O&Htx&Zy^13th#1*?v}SU9-pZ0&Nd8yM^JyVb8`cx zs~Z?R5G- z!&Xf6#kFX&6;SAvv?gZ3-!H=yUDf3VZ8Lo2Tg60}xy(p6!ZW#enaW-nmY#a(Iwv&Lu z-!!>z^OaX(w*`0Bc%ppx77REWz?w3%5);Xts5@h}%1z=aNXtHgACO^i@C|yy{ak{= zq%G-qG=`tA_Q35pUBpJypQhi0vuJyyJCAmOf>?L)U_{?=Jbvu=)4B8TTAg%%mt8YP zZ5hEDh~g|m&@digmAF`AzxM2>h&K6U-9hnG<+umqB&0FRB%R&qd7gcyX&IYwPK1-t zqoDisVrwln2KkkV%1|J3}?7mawJ>mS;b0*X<=JQiA!+m7XygUTxx6`h94aW9) zD#T>;u*pAM6}8Y`Z)`16bfFy}yp?h(a`NJ<{WzUTCdXEk0S-ITExFHsWD)(K`+$-r z#_@H)34xgy9BwTNi9nh7k?L%tnNP>N^r=r&oEW(~7IgO?PKM&>g+DNPGs+j%o&IrI z!9Lqa$6bh(9vWC!rMvCm+V-uzv`4=Z1j>o3?AT~L0xKbKb^H;{{)w6rJZXF(7>lcv<^<;Usn{8a&;FDTP(%e$NQi)m$kgeN*IyMb0doR8S?=>qE5uco?)n^_x+v#zH90B>F zzfV~}_8tc2F=)&MAgD{|YZhAT5v6J`?2_4zpT^MY7;JUZoYO1rUc`Kk%CoU$J5Nkt zqWE-XJ1v+fNxEiiK{QHTnnrid^gk5sr*`_6&BrlAsB%s z38Daaj<%AtLML#$fZBf@y1VY{y6It|_aaFbN{4PqXy00Wi-Y>AoD=FE2y0zvBk0g` zP#QAHEzdI3jV&(w#6gz&Q)#4GyZ$>x+Eo25%M6&wro=}H&y|G5tz9-0JwovHe?1Ml z+F&9qbRp<9Gg_)!&tD^O{6&NLztKjWa0&lD=}U~NN4~=)LaneOgma}T4jta$9+&{^ z=;((6Ug(o;cw=^N_ zbubN&?h)(9BefVUxv27GtUc-(9Cilf5{_u>Mb8gntz&Y~_;}=7pzVs% zb{pO2a}TVxb9&s(V{W2H+T>TXGsM!Eqf_iys!_jsiSC<+d<2V<+7t2s;Ehmr)^tzP z4Vdg!MJ$d~Jb*W*P_)$SO0R*A2YVn4(9&2;PHNz!iKkk@E4mtU5pT+@xhu zW??!Ok2aIuMWF{CfFKFGl7CRc2EFk<&~2_C4DAzO3W9yjmR`(*ngPRuD>!Nhwy~oV zdkOQH2A}U98Y}~qc`sv1g-C3v1MahS7xweQW@7}~G8vuG0-OwvYW+T@7v&yq>*lRn z6Y`4zy9_hP6yNP}Ui^t9b@pfo{=5!W9WdG9`3o9qytdq}B3eNVPj6W+BPdEEKX&FEn zVxF%yU-1dJgK}lq<*BX&AZU4Q&qHqfukkN$XOwy1`xml{VPNhOp5?2{12A;+yb6_9 zO+~Au{Kpas9Ijtj$^a%19Jzg7!9;=AY2WF5D819VYLhb>QTxQY1@`uyM-aIRh`@p% z0Zwa@naf~M*35+{TP|`8VQ!2rjM3w~CUGQlH%#3z`YXdiMzGy?JXPEAjN?se1IZ4s z=|YHiX1FKf~Hy*vK6w`+s{_g>6y z&JgLXv+HVIU4nEg`H1*?Iw#M4PH#WEg6Gqqi2IoMvuCqW5eh$0sKBs!1a$uqybj4R z@y5y9`IY26yaQ10sA9Dr$h`-$TfAm}CCwSu6-njWzJqbO_);8bwVS9V<`{7Y?54!5 zk0R*D^RWTP9YAO5)~EJL_<3U7Sbtp?I59q@gN3Okw_%JQKlDvzQ;@w5*BZ{KFRcdM z)q%)GNkG}~o!YXMiT-Q=F13izz!khht-NTMddrS(_%Bl?_v-4F1lszSrmLFYE}ex| zoeJZ-_Pza3~1%U}cdU4!>25m{}G(n4wjRFq@`7>f3 zB0m35#ueZd`SRe<>x?U=z89uhZM$5oxbDASGNi#l(PwaT*ft0SRJ!P<+%_Wy*qv4- emo;;jFMFUOXO^O#yPzP8cAQ Date: Fri, 23 Aug 2019 11:08:45 +0200 Subject: [PATCH 16/26] Correctly import librato role --- clojurians_log/playbooks/clojurians-log.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 3d760c1..fb59e9e 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -28,7 +28,8 @@ tasks: - name: Set up librato first so we have eyes - import_role: librato.librato + import_role: + name: librato.librato vars: librato_enabled_plugins: ['nginx', 'jvm', 'memcached'] From b3c9cbfdf10e2269e11dba747b1ed01a5c38e7c4 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 11:14:01 +0200 Subject: [PATCH 17/26] Whitespace --- clojurians_log/playbooks/clojurians-log.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index fb59e9e..299e48d 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -24,7 +24,7 @@ datomic_object_cache_max: 2g datomic_transactor_max_heap: 3584m clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211 -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264 " + JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211 -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264" tasks: - name: Set up librato first so we have eyes From b317d3d47646257116c8e11c33ef67704e000776 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 11:44:27 +0200 Subject: [PATCH 18/26] Fix librato package install, disable jmx JMX seems to be causing issues both on the app side, and on the collectd side. --- clojurians_log/playbooks/clojurians-log.yml | 5 +++-- .../playbooks/roles/librato.librato/tasks/repo.yml | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 299e48d..f2965d1 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -24,14 +24,15 @@ datomic_object_cache_max: 2g datomic_transactor_max_heap: 3584m clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211 -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264" + JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" + # -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264 tasks: - name: Set up librato first so we have eyes import_role: name: librato.librato vars: - librato_enabled_plugins: ['nginx', 'jvm', 'memcached'] + librato_enabled_plugins: ['nginx', 'memcached'] #jvm, postgresql # Keys first, then repos, then the dist-upgrade. - name: Add repository for ansible diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml index 96260a6..c3fc214 100644 --- a/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml @@ -8,8 +8,8 @@ - name: Install gpg key apt_key: - url: http://packagecloud.io/gpg.key - id: 418A7F2FB0E1E6E7EABF6FE8C2E73424D59097AB + url: https://packagecloud.io/librato/librato-collectd/gpgkey + id: B83E13B25EA683E71C17FFA0BDBDCC081E77728D state: present when: ansible_os_family == 'Debian' @@ -21,7 +21,9 @@ - name: Create apt repo apt_repository: - repo: 'deb {{ librato_repo_url }}{{ librato_repo_base }}/{{ ansible_distribution|lower }}/ {{ ansible_distribution_release }} main' + # no packages (currently) for bionic, hard coding to use xenial instead + # repo: 'deb {{ librato_repo_url }}{{ librato_repo_base }}/{{ ansible_distribution|lower }}/ {{ ansible_distribution_release }} main' + repo: 'deb {{ librato_repo_url }}{{ librato_repo_base }}/{{ ansible_distribution|lower }}/ xenial main' state: present when: ansible_os_family == 'Debian' From 7f14e4e0ed99fbfbf75aab1945125916f7094368 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 13:00:02 +0200 Subject: [PATCH 19/26] Import logs as part of bootstrapping --- clojurians_log/playbooks/clojurians-log.yml | 4 +++- .../playbooks/roles/plexus.clojurians-log/tasks/main.yml | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index f2965d1..1bee2a9 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -23,8 +23,9 @@ datomic_pro_version: 0.9.5561.56 datomic_object_cache_max: 2g datomic_transactor_max_heap: 3584m + clojure_socket_repl_port: 50505 clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" + JVM_OPTS="-Dclojure.server.myrepl={:port,{{ clojure_socket_repl_port }},:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" # -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264 tasks: @@ -33,6 +34,7 @@ name: librato.librato vars: librato_enabled_plugins: ['nginx', 'memcached'] #jvm, postgresql + librato_logging_use_syslog: true # Keys first, then repos, then the dist-upgrade. - name: Add repository for ansible diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml index 37ea9f7..100d260 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml @@ -62,3 +62,7 @@ accept_hostkey: yes key_file: "{{ clojure_app_home_dir }}/.ssh/id_rsa" tags: git-checkout + +- name: Import logs + command: "nc localhost {{ clojure_socket_repl_port }} < {{ clojure_app_app_dir }}/repl/production_import.clj > {{ clojure_app_app_dir }}/initial_import.txt" + creates: "{{ clojure_app_app_dir }}/initial_import.txt" From 1d00a94d25a5d9051328fb38884595e83df2d454 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 13:07:38 +0200 Subject: [PATCH 20/26] Doh --- .../playbooks/roles/plexus.clojurians-log/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml index 100d260..1e82860 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml @@ -65,4 +65,5 @@ - name: Import logs command: "nc localhost {{ clojure_socket_repl_port }} < {{ clojure_app_app_dir }}/repl/production_import.clj > {{ clojure_app_app_dir }}/initial_import.txt" - creates: "{{ clojure_app_app_dir }}/initial_import.txt" + args: + creates: "{{ clojure_app_app_dir }}/initial_import.txt" From 0eca0e2eca448872f7e7f72eef88367f05fdcf1f Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 14:49:11 +0200 Subject: [PATCH 21/26] Close nc connection so the REPL starts doing stuff --- .../playbooks/roles/plexus.clojurians-log/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml index 1e82860..9bc8eac 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml @@ -64,6 +64,6 @@ tags: git-checkout - name: Import logs - command: "nc localhost {{ clojure_socket_repl_port }} < {{ clojure_app_app_dir }}/repl/production_import.clj > {{ clojure_app_app_dir }}/initial_import.txt" + command: "nc -N localhost {{ clojure_socket_repl_port }} < {{ clojure_app_app_dir }}/repl/production_import.clj > {{ clojure_app_app_dir }}/initial_import.txt" args: creates: "{{ clojure_app_app_dir }}/initial_import.txt" From 210c95ad5867d94353c9f305a2ee154968cbed5e Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 17:19:01 +0200 Subject: [PATCH 22/26] Use a datacenter that runs on renewables --- clojurians_log/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index f7de8b7..9a479bb 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -34,7 +34,7 @@ data "template_file" "userdata" { resource "exoscale_compute" "clojurians_log" { display_name = "${local.instance_name}" template = "Linux Ubuntu 18.04 LTS 64-bit" - zone = "de-fra-1" + zone = "ch-gva-2" size = "Large" disk_size = 50 key_pair = "${var.exoscale_ssh_keypair_name}" From 3539c489c3ef3cd15017bd2ee0a087445a407c00 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 23 Aug 2019 17:19:17 +0200 Subject: [PATCH 23/26] Save the ansible invocation to a script for later reuse --- clojurians_log/userdata.sh.tmpl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/clojurians_log/userdata.sh.tmpl b/clojurians_log/userdata.sh.tmpl index 35a3cd1..7517abd 100755 --- a/clojurians_log/userdata.sh.tmpl +++ b/clojurians_log/userdata.sh.tmpl @@ -34,5 +34,10 @@ base64 --decode < vars/clojurians_log_secrets.yml.b64 > vars/clojurians_log_secr # Bootstrap the instance ansible-playbook bootstrap.yml -i hosts >> ~/ansible.log +# Create a script for ansible, so it's easy to re-run at a later date +echo '#/bin/sh' >> ~/run_ansible +echo 'HOME=/root ansible-playbook clojurians-log.yml -i hosts ${ansible_playbook_params} "$@"' >> ~/run_ansible +chmod +x ~/run_ansible + # Setup clojurians_log -HOME=/root ansible-playbook clojurians-log.yml -i hosts ${ansible_playbook_params} >> ~/ansible.log +~/run_ansible From 763c2ecddbf68d3cb8f61da59acb13bda9bd6cc2 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 30 Aug 2019 13:37:30 +0200 Subject: [PATCH 24/26] Distinguish between a proxied and non-proxied domain This way you can ssh to clojurians-log-internal.clojureverse.org. --- clojurians_log/main.tf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index 9a479bb..2e2d353 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -53,9 +53,17 @@ output "username" { value = exoscale_compute.clojurians_log.username } -resource "cloudflare_record" "clojurians_log" { +resource "cloudflare_record" "clojurians_log_internal" { domain = "clojureverse.org" - name = "${local.instance_name}" + name = "${local.instance_name}-internal" value = "${exoscale_compute.clojurians_log.ip_address}" type = "A" } + +resource "cloudflare_record" "clojurians_log" { + domain = "clojureverse.org" + name = "${local.instance_name}" + value = "${exoscale_compute.clojurians_log.ip_address}" + type = "A" + proxied = true +} From 57c580db440fdd492527b1143bfd7d5032bef090 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 30 Aug 2019 13:47:50 +0200 Subject: [PATCH 25/26] Pipe output of run_ansible to ansible.log --- clojurians_log/userdata.sh.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/userdata.sh.tmpl b/clojurians_log/userdata.sh.tmpl index 7517abd..894597d 100755 --- a/clojurians_log/userdata.sh.tmpl +++ b/clojurians_log/userdata.sh.tmpl @@ -40,4 +40,4 @@ echo 'HOME=/root ansible-playbook clojurians-log.yml -i hosts ${ansible_playbook chmod +x ~/run_ansible # Setup clojurians_log -~/run_ansible +~/run_ansible >> ~/ansible.log From 81532cb02b6f499eb8c4a47211bc697e8cba9c07 Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 30 Aug 2019 14:19:33 +0200 Subject: [PATCH 26/26] Prepare for merging exoscale-deploy back to master --- clojurians_log/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index 2e2d353..5d0d714 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -27,7 +27,7 @@ data "template_file" "userdata" { # git_clone_params = "--single-branch --branch exoscale-deploy" # The following can be used for production ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\" --extra-vars \"clojurians_app_fqdn=${local.instance_name}.clojureverse.org\"" - git_clone_params = "--single-branch --branch exoscale-deploy" + git_clone_params = "" } }