From b1c7d48f50818f52271512e482079a28bb3bcc34 Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Fri, 26 Oct 2018 09:21:03 -0500 Subject: [PATCH 01/10] Add logging and backups --- .env | 3 +++ docker-compose.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/.env b/.env index da695e4..28ad22c 100644 --- a/.env +++ b/.env @@ -47,3 +47,6 @@ POSTFIX_RELAYHOST= POSTFIX_RELAYHOST_PORT=587 POSTFIX_SASL_AUTH= POSTFIX_TLS= + +# Configure docker log driver - currently supports 'none', 'syslog' or 'journald' +LOG_DRIVER=syslog diff --git a/docker-compose.yml b/docker-compose.yml index 93eaf6f..6c1b9bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,12 +2,19 @@ version: '3.6' +x-linux-logging: + &common-logging + driver: "${LOG_DRIVER}" + options: + tag: "{{.ImageName}}/{{.Name}}/{{.ID}}" + services: nginx-php-moodle: networks: db: mail: image: ${DOCKER_REGISTRY_URL}/nginx-php-moodle:latest + logging: *common-logging depends_on: - postfix - postgres @@ -60,10 +67,12 @@ services: POSTFIX_RELAYHOST_PORT: POSTFIX_SASL_AUTH: POSTFIX_TLS: + logging: *common-logging postgres: networks: db: image: ${DOCKER_REGISTRY_URL}/postgres:9.6 + logging: *common-logging environment: POSTGRES_DB: "${PGSQL_DATABASE}" POSTGRES_USER: "${PGSQL_USER}" @@ -72,10 +81,27 @@ services: - type: volume source: db target: /var/lib/postgresql/data + postgres-backup: + image: ${DOCKER_REGISTRY_URL}/postgres-backup:latest + depends_on: + - "postgres" + volumes: + - ./backup:/backup:Z + environment: + POSTGRES_PASSWORD: "${PGSQL_PASSWORD}" + POSTGRES_USER: "${PGSQL_USER}" + POSTGRES_DB: "${PGSQL_DATABASE}" + DUMPPREFIX: "backup" + POSTGRES_HOST: "db" + POSTGRES_PORT: "5432" + logging: *common-logging + networks: + db: certbot: networks: certbot: image: ${DOCKER_REGISTRY_URL}/sc-certbot:latest + logging: *common-logging depends_on: - nginx-php-moodle environment: From 763c23c958ab2f5cd50433280473c8b54ead44a5 Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Wed, 7 Nov 2018 13:17:24 -0600 Subject: [PATCH 02/10] compose adjustments --- docker-compose.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6c1b9bd..ac7ebc0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,15 @@ version: '3.6' -x-linux-logging: +x-logging: &common-logging driver: "${LOG_DRIVER}" - options: - tag: "{{.ImageName}}/{{.Name}}/{{.ID}}" + +x-secopts: + &sec-opts + security_opt: + - label:disable + services: nginx-php-moodle: @@ -15,6 +19,7 @@ services: mail: image: ${DOCKER_REGISTRY_URL}/nginx-php-moodle:latest logging: *common-logging + <<: *sec-opts depends_on: - postfix - postgres @@ -68,11 +73,13 @@ services: POSTFIX_SASL_AUTH: POSTFIX_TLS: logging: *common-logging + <<: *sec-opts postgres: networks: db: image: ${DOCKER_REGISTRY_URL}/postgres:9.6 logging: *common-logging + <<: *sec-opts environment: POSTGRES_DB: "${PGSQL_DATABASE}" POSTGRES_USER: "${PGSQL_USER}" @@ -92,9 +99,11 @@ services: POSTGRES_USER: "${PGSQL_USER}" POSTGRES_DB: "${PGSQL_DATABASE}" DUMPPREFIX: "backup" - POSTGRES_HOST: "db" + POSTGRES_HOST: "postgres" POSTGRES_PORT: "5432" logging: *common-logging + <<: *sec-opts + command: /bin/true networks: db: certbot: @@ -102,6 +111,7 @@ services: certbot: image: ${DOCKER_REGISTRY_URL}/sc-certbot:latest logging: *common-logging + <<: *sec-opts depends_on: - nginx-php-moodle environment: From 88fb9a9d5867089ff30c6c43bb80b41f57ddeef7 Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Wed, 7 Nov 2018 13:26:56 -0600 Subject: [PATCH 03/10] Switch logging driver back to json-file by default --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 28ad22c..e6141ec 100644 --- a/.env +++ b/.env @@ -49,4 +49,4 @@ POSTFIX_SASL_AUTH= POSTFIX_TLS= # Configure docker log driver - currently supports 'none', 'syslog' or 'journald' -LOG_DRIVER=syslog +LOG_DRIVER=json-file From 3c47a712467f4e6c4bf4bd61b1e9dc992a90a36c Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Wed, 7 Nov 2018 14:57:41 -0600 Subject: [PATCH 04/10] Add backup cron job and bump version --- .env | 3 +++ CHANGELOG.md | 7 +++++++ dc.deb.yml | 1 + docker-compose.yml | 2 +- systemd/postgres-backup | 1 + systemd/postinst.sh | 1 + 6 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 systemd/postgres-backup diff --git a/.env b/.env index e6141ec..0dc7cf9 100644 --- a/.env +++ b/.env @@ -50,3 +50,6 @@ POSTFIX_TLS= # Configure docker log driver - currently supports 'none', 'syslog' or 'journald' LOG_DRIVER=json-file + +# Configure the moodle backup path +MOODLE_BACKUP_ROOT=./backup diff --git a/CHANGELOG.md b/CHANGELOG.md index dffa0e1..f61b605 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [v0.1.15] - 2018-11-19 +### Added +- Add customizable log driver option +- Add postgres-backup container +- Add host cron job for postgres backups +- Add security label disable for selinux based systems + ## [v0.1.14] - 2018-11-08 ### Added - Bake in 'postfix' as default `smtphost` diff --git a/dc.deb.yml b/dc.deb.yml index 721b0b6..c97538f 100644 --- a/dc.deb.yml +++ b/dc.deb.yml @@ -70,6 +70,7 @@ services: - "/package/dc.prod-dbonly.yml=/etc/moodle-docker/" - "/package/docker-compose.yml=/etc/moodle-docker/" - "/package/systemd/moodle-docker.service=/lib/systemd/system/" + - "/package/systemd/postgres-backup=/etc/cron.d/" environment: BUILD_VERSION: "${BUILD_VERSION}" MOODLE_VERSION: "${MOODLE_VERSION}" diff --git a/docker-compose.yml b/docker-compose.yml index ac7ebc0..b94af7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: depends_on: - "postgres" volumes: - - ./backup:/backup:Z + - ${MOODLE_BACKUP_ROOT}:/backup:Z environment: POSTGRES_PASSWORD: "${PGSQL_PASSWORD}" POSTGRES_USER: "${PGSQL_USER}" diff --git a/systemd/postgres-backup b/systemd/postgres-backup new file mode 100644 index 0000000..4401123 --- /dev/null +++ b/systemd/postgres-backup @@ -0,0 +1 @@ +0 * * * * root cd /etc/moodle-docker && /usr/local/bin/docker-compose --file /etc/moodle-docker/docker-compose.yml run postgres-backup /bin/backup_db diff --git a/systemd/postinst.sh b/systemd/postinst.sh index 1875521..d57fc1f 100644 --- a/systemd/postinst.sh +++ b/systemd/postinst.sh @@ -44,6 +44,7 @@ if [ ! -f /etc/moodle-docker/.env ]; then sed -i "s/MOODLE_ADMIN_PASS=.*/MOODLE_ADMIN_PASS=${moodle_admin_pass}/g" /etc/moodle-docker/.env sed -i "s/MOODLE_UPGRADE_KEY=.*/MOODLE_UPGRADE_KEY=${moodle_upgrade_key}/g" /etc/moodle-docker/.env sed -i "s/PGSQL_PASSWORD=.*/PGSQL_PASSWORD=${pgsql_password}/g" /etc/moodle-docker/.env + sed -i "s#MOODLE_BACKUP_ROOT=.*#MOODLE_BACKUP_ROOT=/backups#g" /etc/moodle-docker/.env else echo "Nothing to do, .env already exists" fi From ebf503ac86415e317690b7a38adf2c7275202779 Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Thu, 8 Nov 2018 11:03:17 -0600 Subject: [PATCH 05/10] Ensure permissions on cronjob are correct --- systemd/postinst.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/systemd/postinst.sh b/systemd/postinst.sh index d57fc1f..39e0941 100644 --- a/systemd/postinst.sh +++ b/systemd/postinst.sh @@ -32,6 +32,9 @@ else echo "Moodle data directory already exists" fi +echo "Adjust cron job permissions" +chmod 644 /etc/cron.d/postgres-backup + echo "Setting up .env file" if [ ! -f /etc/moodle-docker/.env ]; then echo "Creating .env from example file" From a56a99aaf443081d254e769576bcc2ce9018a42b Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Thu, 8 Nov 2018 12:12:14 -0600 Subject: [PATCH 06/10] Update README with restore instructions --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/README.md b/README.md index 8adf252..eba3aea 100644 --- a/README.md +++ b/README.md @@ -87,3 +87,48 @@ The [Moodle eMailTest](https://moodle.org/plugins/local_mailtest) plugin is bake ```bash openssl req -x509 -newkey rsa:4096 -keyout moodle.key -out moodle.crt -days 365 -nodes -subj "/C=CA/ST=ON/L=Toronto/O=SC/OU=Org/CN=www.example.com" ``` + +## Restoring a database backup + +First, ensure you have a backup and delta to restore from! On systems installed from .deb package, the files are stored on the docker host under /backups by default. + +Pause the backup cron on the docker host by commenting the job: + +``` + # sed -i 's/^\([^#]\)/#\1/g' /etc/cron.d/postgres-backup +``` + +Switch to the moodle config folder: + +``` + # cd /etc/moodle-docker +``` + +Enable the site maintenance page: + +``` + # docker-compose exec nginx-php-moodle /usr/bin/php /opt/moodle/app/admin/cli/maintenance.php --enable +``` + +Drop and re-create the database: + +``` + # docker-compose run postgres-backup /bin/drop_and_create +``` + +Restore the DB by running the script and answering the interactive prompts regarding which backup to restore: + +``` + # docker-compose run postgres-backup /bin/restore_db_from_delta +``` + +Disable the site maintenance page: + +``` + # docker-compose exec nginx-php-moodle /usr/bin/php /opt/moodle/app/admin/cli/maintenance.php --disable +``` +Unpause the backup: + +``` + # sed -i 's/^#//g' /etc/cron.d/postgres-backup +``` From a52987460d71acfef7f6092f080c7aaa1096afba Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Thu, 8 Nov 2018 12:21:26 -0600 Subject: [PATCH 07/10] Minor adjustments --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index eba3aea..d988b33 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,14 @@ The [Moodle eMailTest](https://moodle.org/plugins/local_mailtest) plugin is bake openssl req -x509 -newkey rsa:4096 -keyout moodle.key -out moodle.crt -days 365 -nodes -subj "/C=CA/ST=ON/L=Toronto/O=SC/OU=Org/CN=www.example.com" ``` +## Manual backups + +On systems installed from .deb package, a cronjob should be automatically setup to perform backups. On a development system, you can manually run the same process: + +``` +docker-compose run postgres-backup /bin/backup_db +``` + ## Restoring a database backup First, ensure you have a backup and delta to restore from! On systems installed from .deb package, the files are stored on the docker host under /backups by default. @@ -95,40 +103,40 @@ First, ensure you have a backup and delta to restore from! On systems installed Pause the backup cron on the docker host by commenting the job: ``` - # sed -i 's/^\([^#]\)/#\1/g' /etc/cron.d/postgres-backup +sed -i 's/^\([^#]\)/#\1/g' /etc/cron.d/postgres-backup ``` Switch to the moodle config folder: ``` - # cd /etc/moodle-docker +cd /etc/moodle-docker ``` Enable the site maintenance page: ``` - # docker-compose exec nginx-php-moodle /usr/bin/php /opt/moodle/app/admin/cli/maintenance.php --enable +docker-compose exec nginx-php-moodle /usr/bin/php /opt/moodle/app/admin/cli/maintenance.php --enable ``` Drop and re-create the database: ``` - # docker-compose run postgres-backup /bin/drop_and_create +docker-compose run postgres-backup /bin/drop_and_create ``` Restore the DB by running the script and answering the interactive prompts regarding which backup to restore: ``` - # docker-compose run postgres-backup /bin/restore_db_from_delta +docker-compose run postgres-backup /bin/restore_db_from_delta ``` Disable the site maintenance page: ``` - # docker-compose exec nginx-php-moodle /usr/bin/php /opt/moodle/app/admin/cli/maintenance.php --disable +docker-compose exec nginx-php-moodle /usr/bin/php /opt/moodle/app/admin/cli/maintenance.php --disable ``` Unpause the backup: ``` - # sed -i 's/^#//g' /etc/cron.d/postgres-backup +sed -i 's/^#//g' /etc/cron.d/postgres-backup ``` From dd358dbfe6a7bd2bd8e5701b6077c071325a7b22 Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Thu, 8 Nov 2018 12:33:11 -0600 Subject: [PATCH 08/10] Adjust env comment for all supported options --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 0dc7cf9..175586c 100644 --- a/.env +++ b/.env @@ -48,7 +48,7 @@ POSTFIX_RELAYHOST_PORT=587 POSTFIX_SASL_AUTH= POSTFIX_TLS= -# Configure docker log driver - currently supports 'none', 'syslog' or 'journald' +# Configure docker log driver - currently supports 'none', 'syslog' or 'journald' or 'json-file' (assuming no tags) LOG_DRIVER=json-file # Configure the moodle backup path From 8cf9db49440beb1a9f0f8b3107da8c1913d48c92 Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Fri, 9 Nov 2018 09:26:57 -0600 Subject: [PATCH 09/10] Various adjustments --- CHANGELOG.md | 8 ++++---- README.md | 6 ++++-- {systemd => cron}/postgres-backup | 0 dc.deb.yml | 2 +- docker-compose.yml | 5 +++-- 5 files changed, 12 insertions(+), 9 deletions(-) rename {systemd => cron}/postgres-backup (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index f61b605..1160fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v0.1.15] - 2018-11-19 ### Added -- Add customizable log driver option -- Add postgres-backup container -- Add host cron job for postgres backups -- Add security label disable for selinux based systems +- Customizable log driver option +- Postgres-backup container +- Host cron job for postgres backups +- Security label disable for selinux based systems ## [v0.1.14] - 2018-11-08 ### Added diff --git a/README.md b/README.md index d988b33..aeb76e6 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,9 @@ The [Moodle eMailTest](https://moodle.org/plugins/local_mailtest) plugin is bake openssl req -x509 -newkey rsa:4096 -keyout moodle.key -out moodle.crt -days 365 -nodes -subj "/C=CA/ST=ON/L=Toronto/O=SC/OU=Org/CN=www.example.com" ``` -## Manual backups +## Backup and Restore + +### Manual backups On systems installed from .deb package, a cronjob should be automatically setup to perform backups. On a development system, you can manually run the same process: @@ -96,7 +98,7 @@ On systems installed from .deb package, a cronjob should be automatically setup docker-compose run postgres-backup /bin/backup_db ``` -## Restoring a database backup +### Restoring a database backup First, ensure you have a backup and delta to restore from! On systems installed from .deb package, the files are stored on the docker host under /backups by default. diff --git a/systemd/postgres-backup b/cron/postgres-backup similarity index 100% rename from systemd/postgres-backup rename to cron/postgres-backup diff --git a/dc.deb.yml b/dc.deb.yml index c97538f..e9b2f04 100644 --- a/dc.deb.yml +++ b/dc.deb.yml @@ -70,7 +70,7 @@ services: - "/package/dc.prod-dbonly.yml=/etc/moodle-docker/" - "/package/docker-compose.yml=/etc/moodle-docker/" - "/package/systemd/moodle-docker.service=/lib/systemd/system/" - - "/package/systemd/postgres-backup=/etc/cron.d/" + - "/package/cron/postgres-backup=/etc/cron.d/" environment: BUILD_VERSION: "${BUILD_VERSION}" MOODLE_VERSION: "${MOODLE_VERSION}" diff --git a/docker-compose.yml b/docker-compose.yml index b94af7b..8c2b3ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,6 @@ x-secopts: security_opt: - label:disable - services: nginx-php-moodle: networks: @@ -93,7 +92,9 @@ services: depends_on: - "postgres" volumes: - - ${MOODLE_BACKUP_ROOT}:/backup:Z + - type: bind + source: ${MOODLE_BACKUP_ROOT} + target: /backup environment: POSTGRES_PASSWORD: "${PGSQL_PASSWORD}" POSTGRES_USER: "${PGSQL_USER}" From 52ff5d382b8ed847c673c5ca1fae34066d5dc3bc Mon Sep 17 00:00:00 2001 From: Adam Gilbert Date: Mon, 19 Nov 2018 14:38:15 -0600 Subject: [PATCH 10/10] Rebase and bump version --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 175586c..268f75a 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ # Build version -BUILD_VERSION=v0.1.14 +BUILD_VERSION=v0.1.15 # Moodle version MOODLE_VERSION=3.5.2