From fabb7df154c75a4efa1f5176c9d89b9f49e0b953 Mon Sep 17 00:00:00 2001 From: SilverFire - Dmitry Naumenko Date: Mon, 13 Jan 2020 22:26:15 +0200 Subject: [PATCH 1/5] Implemented segregation by network --- app/letsencrypt_service_data.tmpl | 102 +++++++++++------- .../networks_segregation/expected-std-out.txt | 11 ++ test/tests/networks_segregation/run.sh | 71 ++++++++++++ 3 files changed, 146 insertions(+), 38 deletions(-) create mode 100644 test/tests/networks_segregation/expected-std-out.txt create mode 100755 test/tests/networks_segregation/run.sh diff --git a/app/letsencrypt_service_data.tmpl b/app/letsencrypt_service_data.tmpl index e3205afa..23e7c7ac 100644 --- a/app/letsencrypt_service_data.tmpl +++ b/app/letsencrypt_service_data.tmpl @@ -1,14 +1,38 @@ +{{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} +{{ $scopedContainersString := "" }} + +{{ range $hosts, $containers := groupBy $ "Env.LETSENCRYPT_HOST" }} + {{ if trim $hosts }} + {{ range $container := $containers }} + {{ $cid := printf "%.12s" $container.ID }} + {{ if $CurrentContainer.Env.MUST_BE_CONNECTED_WITH_NETWORK }} + {{ range $containerNetwork := $container.Networks }} + {{ if eq $CurrentContainer.Env.MUST_BE_CONNECTED_WITH_NETWORK $containerNetwork.Name }} + {{ $scopedContainersString = (printf "%s %s" $scopedContainersString $cid) }} + {{ end }} + {{ end }} + {{ else }} + {{ $scopedContainersString = (printf "%s %s" $scopedContainersString $cid) }} + {{ end }} + {{ end }} + {{ end }} +{{ end }} + +{{ $scopedContainersSlice := split (trim $scopedContainersString) " " }} + LETSENCRYPT_CONTAINERS=( {{ range $hosts, $containers := groupBy $ "Env.LETSENCRYPT_HOST" }} {{ if trim $hosts }} {{ range $container := $containers }} - {{ if parseBool (coalesce $container.Env.LETSENCRYPT_SINGLE_DOMAIN_CERTS "false") }} - {{ range $host := split $hosts "," }} - {{ $host := trim $host }} - {{- "\t"}}'{{ printf "%.12s" $container.ID }}_{{ sha1 $host }}' + {{ $cid := printf "%.12s" $container.ID }} + {{ if intersect $scopedContainersSlice (split $cid " ") }} + {{ if parseBool (coalesce $container.Env.LETSENCRYPT_SINGLE_DOMAIN_CERTS "false") }} + {{ range $host := split $hosts "," }} + {{- "\t"}}'{{ printf "%s_%s" $cid (sha1 (trim $host)) }}' + {{ end }} + {{ else }} + '{{ $cid }}' {{ end }} - {{ else }} - {{- "\t"}}'{{ printf "%.12s" $container.ID }}' {{ end }} {{ end }} {{ end }} @@ -19,39 +43,41 @@ LETSENCRYPT_CONTAINERS=( {{ $hosts := trimSuffix "," $hosts }} {{ range $container := $containers }} {{ $cid := printf "%.12s" $container.ID }} - {{ if parseBool (coalesce $container.Env.LETSENCRYPT_SINGLE_DOMAIN_CERTS "false") }} - {{ range $host := split $hosts "," }} - {{ $host := trim $host }} - {{ $host := trimSuffix "." $host }} - {{ $hostHash := sha1 $host }} - {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_HOST=('{{ $host }}') - {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_KEYSIZE="{{ $container.Env.LETSENCRYPT_KEYSIZE }}" - {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_TEST="{{ $container.Env.LETSENCRYPT_TEST }}" - {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_EMAIL="{{ $container.Env.LETSENCRYPT_EMAIL }}" - {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_CA_URI="{{ $container.Env.ACME_CA_URI }}" - {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_PREFERRED_CHAIN="{{ $container.Env.ACME_PREFERRED_CHAIN }}" - {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_EAB_KID="{{ $container.Env.ACME_EAB_KID }}" - {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_EAB_HMAC_KEY="{{ $container.Env.ACME_EAB_HMAC_KEY }}" - {{- "\n" }}ZEROSSL_{{ $cid }}_{{ $hostHash }}_API_KEY="{{ $container.Env.ZEROSSL_API_KEY }}" - {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_RESTART_CONTAINER="{{ $container.Env.LETSENCRYPT_RESTART_CONTAINER }}" + {{ if intersect $scopedContainersSlice (split $cid " ") }} + {{ if parseBool (coalesce $container.Env.LETSENCRYPT_SINGLE_DOMAIN_CERTS "false") }} + {{ range $host := split $hosts "," }} + {{ $host := trim $host }} + {{ $host := trimSuffix "." $host }} + {{ $hostHash := sha1 $host }} + {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_HOST=('{{ $host }}') + {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_KEYSIZE="{{ $container.Env.LETSENCRYPT_KEYSIZE }}" + {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_TEST="{{ $container.Env.LETSENCRYPT_TEST }}" + {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_EMAIL="{{ $container.Env.LETSENCRYPT_EMAIL }}" + {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_CA_URI="{{ $container.Env.ACME_CA_URI }}" + {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_PREFERRED_CHAIN="{{ $container.Env.ACME_PREFERRED_CHAIN }}" + {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_EAB_KID="{{ $container.Env.ACME_EAB_KID }}" + {{- "\n" }}ACME_{{ $cid }}_{{ $hostHash }}_EAB_HMAC_KEY="{{ $container.Env.ACME_EAB_HMAC_KEY }}" + {{- "\n" }}ZEROSSL_{{ $cid }}_{{ $hostHash }}_API_KEY="{{ $container.Env.ZEROSSL_API_KEY }}" + {{- "\n" }}LETSENCRYPT_{{ $cid }}_{{ $hostHash }}_RESTART_CONTAINER="{{ $container.Env.LETSENCRYPT_RESTART_CONTAINER }}" + {{ end }} + {{ else }} + {{- "\n" }}LETSENCRYPT_{{ $cid }}_HOST=( + {{- range $host := split $hosts "," }} + {{- $host := trim $host }} + {{- $host := trimSuffix "." $host -}} + '{{ $host }}'{{ " " }} + {{- end -}} + ) + {{- "\n" }}LETSENCRYPT_{{ $cid }}_KEYSIZE="{{ $container.Env.LETSENCRYPT_KEYSIZE }}" + {{- "\n" }}LETSENCRYPT_{{ $cid }}_TEST="{{ $container.Env.LETSENCRYPT_TEST }}" + {{- "\n" }}LETSENCRYPT_{{ $cid }}_EMAIL="{{ $container.Env.LETSENCRYPT_EMAIL }}" + {{- "\n" }}ACME_{{ $cid }}_CA_URI="{{ $container.Env.ACME_CA_URI }}" + {{- "\n" }}ACME_{{ $cid }}_PREFERRED_CHAIN="{{ $container.Env.ACME_PREFERRED_CHAIN }}" + {{- "\n" }}ACME_{{ $cid }}_EAB_KID="{{ $container.Env.ACME_EAB_KID }}" + {{- "\n" }}ACME_{{ $cid }}_EAB_HMAC_KEY="{{ $container.Env.ACME_EAB_HMAC_KEY }}" + {{- "\n" }}ZEROSSL_{{ $cid }}_API_KEY="{{ $container.Env.ZEROSSL_API_KEY }}" + {{- "\n" }}LETSENCRYPT_{{ $cid }}_RESTART_CONTAINER="{{ $container.Env.LETSENCRYPT_RESTART_CONTAINER }}" {{ end }} - {{ else }} - {{- "\n" }}LETSENCRYPT_{{ $cid }}_HOST=( - {{- range $host := split $hosts "," }} - {{- $host := trim $host }} - {{- $host := trimSuffix "." $host -}} - '{{ $host }}'{{ " " }} - {{- end -}} - ) - {{- "\n" }}LETSENCRYPT_{{ $cid }}_KEYSIZE="{{ $container.Env.LETSENCRYPT_KEYSIZE }}" - {{- "\n" }}LETSENCRYPT_{{ $cid }}_TEST="{{ $container.Env.LETSENCRYPT_TEST }}" - {{- "\n" }}LETSENCRYPT_{{ $cid }}_EMAIL="{{ $container.Env.LETSENCRYPT_EMAIL }}" - {{- "\n" }}ACME_{{ $cid }}_CA_URI="{{ $container.Env.ACME_CA_URI }}" - {{- "\n" }}ACME_{{ $cid }}_PREFERRED_CHAIN="{{ $container.Env.ACME_PREFERRED_CHAIN }}" - {{- "\n" }}ACME_{{ $cid }}_EAB_KID="{{ $container.Env.ACME_EAB_KID }}" - {{- "\n" }}ACME_{{ $cid }}_EAB_HMAC_KEY="{{ $container.Env.ACME_EAB_HMAC_KEY }}" - {{- "\n" }}ZEROSSL_{{ $cid }}_API_KEY="{{ $container.Env.ZEROSSL_API_KEY }}" - {{- "\n" }}LETSENCRYPT_{{ $cid }}_RESTART_CONTAINER="{{ $container.Env.LETSENCRYPT_RESTART_CONTAINER }}" {{ end }} {{ end }} {{ end }} diff --git a/test/tests/networks_segregation/expected-std-out.txt b/test/tests/networks_segregation/expected-std-out.txt new file mode 100644 index 00000000..6e234e5f --- /dev/null +++ b/test/tests/networks_segregation/expected-std-out.txt @@ -0,0 +1,11 @@ +Started letsencrypt container for test networks_segregation +Started test web server for le1.wtf in net boulder_bluenet +Started test web server for le2.wtf in net le_test_other_net1 +Started test web server for le3.wtf in net le_test_other_net2 +le1.wtf is in boulder_bluenet, cert should be generated +Symlink to le1.wtf certificate has been generated. +The link is pointing to the file ./le1.wtf/fullchain.pem +le2.wtf is not in boulder_bluenet, cert should not be generated +Domain le2.wtf was not included in the service_data. +le3.wtf is not in boulder_bluenet, cert should not be generated +Domain le3.wtf was not included in the service_data. diff --git a/test/tests/networks_segregation/run.sh b/test/tests/networks_segregation/run.sh new file mode 100755 index 00000000..c393b909 --- /dev/null +++ b/test/tests/networks_segregation/run.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +## Test for single domain certificates. + +if [[ -z $TRAVIS ]]; then + le_container_name="$(basename ${0%/*})_$(date "+%Y-%m-%d_%H.%M.%S")" +else + le_container_name="$(basename ${0%/*})" +fi +desired_network="boulder_bluenet" +run_le_container ${1:?} "$le_container_name" "--env MUST_BE_CONNECTED_WITH_NETWORK=$desired_network" + +# Create the $domains array from comma separated domains in TEST_DOMAINS. +IFS=',' read -r -a domains <<< "$TEST_DOMAINS" + +# Cleanup function with EXIT trap +function cleanup { + # Remove any remaining Nginx container(s) silently. + for domain in "${domains[@]}"; do + docker rm --force "$domain" > /dev/null 2>&1 + done + # Cleanup the files created by this run of the test to avoid foiling following test(s). + docker exec "$le_container_name" bash -c 'rm -rf /etc/nginx/certs/le?.wtf*' + # Stop the LE container + docker stop "$le_container_name" > /dev/null + # Drop temp network + docker network rm "le_test_other_net1" > /dev/null + docker network rm "le_test_other_net2" > /dev/null +} +trap cleanup EXIT + +docker network create "le_test_other_net1" > /dev/null +docker network create "le_test_other_net2" > /dev/null + +networks_map=("$desired_network" le_test_other_net1 le_test_other_net2) + +# Run a separate nginx container for each domain in the $domains array. +# Start all the containers in a row so that docker-gen debounce timers fire only once. +i=0 +for domain in "${domains[@]}"; do + docker run --rm -d \ + --name "$domain" \ + -e "VIRTUAL_HOST=${domain}" \ + -e "LETSENCRYPT_HOST=${domain}" \ + --network "${networks_map[i]}" \ + nginx:alpine > /dev/null && echo "Started test web server for $domain in net ${networks_map[${i}]}" + + i=$(( $i + 1 )) +done + +i=0 +for domain in "${domains[@]}"; do + if [ "${networks_map[i]}" != "$desired_network" ]; then + echo "$domain is not in $desired_network, cert should not be generated"; + + service_data="$(docker exec "$le_container_name" cat /app/letsencrypt_service_data)" + if grep -q "$domain" <<< "$service_data"; then + echo "Domain $domain is on data list, but MUST not!" + else + echo "Domain $domain was not included in the service_data." + fi + else + echo "$domain is in $desired_network, cert should be generated"; + + # Wait for a symlink at /etc/nginx/certs/$domain.crt + wait_for_symlink "$domain" "$le_container_name" + fi + # Stop the Nginx container silently. + docker stop "$domain" > /dev/null + i=$(( $i + 1 )) +done From b10f924048a8bed3ad8adf6bd45cc4a7c81c696d Mon Sep 17 00:00:00 2001 From: SilverFire - Dmitry Naumenko Date: Mon, 15 Jun 2020 14:27:58 +0300 Subject: [PATCH 2/5] Renamed MUST_BE_CONNECTED_WITH_NETWORK to NETWORK_SCOPE --- app/letsencrypt_service_data.tmpl | 4 ++-- test/tests/networks_segregation/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/letsencrypt_service_data.tmpl b/app/letsencrypt_service_data.tmpl index 23e7c7ac..6fa8bbde 100644 --- a/app/letsencrypt_service_data.tmpl +++ b/app/letsencrypt_service_data.tmpl @@ -5,9 +5,9 @@ {{ if trim $hosts }} {{ range $container := $containers }} {{ $cid := printf "%.12s" $container.ID }} - {{ if $CurrentContainer.Env.MUST_BE_CONNECTED_WITH_NETWORK }} + {{ if $CurrentContainer.Env.NETWORK_SCOPE }} {{ range $containerNetwork := $container.Networks }} - {{ if eq $CurrentContainer.Env.MUST_BE_CONNECTED_WITH_NETWORK $containerNetwork.Name }} + {{ if eq $CurrentContainer.Env.NETWORK_SCOPE $containerNetwork.Name }} {{ $scopedContainersString = (printf "%s %s" $scopedContainersString $cid) }} {{ end }} {{ end }} diff --git a/test/tests/networks_segregation/run.sh b/test/tests/networks_segregation/run.sh index c393b909..accb56ac 100755 --- a/test/tests/networks_segregation/run.sh +++ b/test/tests/networks_segregation/run.sh @@ -8,7 +8,7 @@ else le_container_name="$(basename ${0%/*})" fi desired_network="boulder_bluenet" -run_le_container ${1:?} "$le_container_name" "--env MUST_BE_CONNECTED_WITH_NETWORK=$desired_network" +run_le_container ${1:?} "$le_container_name" "--env NETWORK_SCOPE=$desired_network" # Create the $domains array from comma separated domains in TEST_DOMAINS. IFS=',' read -r -a domains <<< "$TEST_DOMAINS" From 58552d4c2591cec8fc3c5e77e11b5c57cd81b999 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Thu, 10 Dec 2020 10:11:36 +0100 Subject: [PATCH 3/5] networks_segregation test : output on error only --- .../networks_segregation/expected-std-out.txt | 12 +------- test/tests/networks_segregation/run.sh | 29 +++++++++++-------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/test/tests/networks_segregation/expected-std-out.txt b/test/tests/networks_segregation/expected-std-out.txt index 6e234e5f..8b137891 100644 --- a/test/tests/networks_segregation/expected-std-out.txt +++ b/test/tests/networks_segregation/expected-std-out.txt @@ -1,11 +1 @@ -Started letsencrypt container for test networks_segregation -Started test web server for le1.wtf in net boulder_bluenet -Started test web server for le2.wtf in net le_test_other_net1 -Started test web server for le3.wtf in net le_test_other_net2 -le1.wtf is in boulder_bluenet, cert should be generated -Symlink to le1.wtf certificate has been generated. -The link is pointing to the file ./le1.wtf/fullchain.pem -le2.wtf is not in boulder_bluenet, cert should not be generated -Domain le2.wtf was not included in the service_data. -le3.wtf is not in boulder_bluenet, cert should not be generated -Domain le3.wtf was not included in the service_data. + diff --git a/test/tests/networks_segregation/run.sh b/test/tests/networks_segregation/run.sh index accb56ac..7e49acd4 100755 --- a/test/tests/networks_segregation/run.sh +++ b/test/tests/networks_segregation/run.sh @@ -2,13 +2,13 @@ ## Test for single domain certificates. -if [[ -z $TRAVIS ]]; then - le_container_name="$(basename ${0%/*})_$(date "+%Y-%m-%d_%H.%M.%S")" +if [[ -z $GITHUB_ACTIONS ]]; then + le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else - le_container_name="$(basename ${0%/*})" + le_container_name="$(basename "${0%/*}")" fi desired_network="boulder_bluenet" -run_le_container ${1:?} "$le_container_name" "--env NETWORK_SCOPE=$desired_network" +run_le_container "${1:?}" "$le_container_name" "--env NETWORK_SCOPE=$desired_network" # Create the $domains array from comma separated domains in TEST_DOMAINS. IFS=',' read -r -a domains <<< "$TEST_DOMAINS" @@ -20,7 +20,7 @@ function cleanup { docker rm --force "$domain" > /dev/null 2>&1 done # Cleanup the files created by this run of the test to avoid foiling following test(s). - docker exec "$le_container_name" bash -c 'rm -rf /etc/nginx/certs/le?.wtf*' + docker exec "$le_container_name" /app/cleanup_test_artifacts # Stop the LE container docker stop "$le_container_name" > /dev/null # Drop temp network @@ -38,34 +38,39 @@ networks_map=("$desired_network" le_test_other_net1 le_test_other_net2) # Start all the containers in a row so that docker-gen debounce timers fire only once. i=0 for domain in "${domains[@]}"; do - docker run --rm -d \ + if ! docker run --rm -d \ --name "$domain" \ -e "VIRTUAL_HOST=${domain}" \ -e "LETSENCRYPT_HOST=${domain}" \ --network "${networks_map[i]}" \ - nginx:alpine > /dev/null && echo "Started test web server for $domain in net ${networks_map[${i}]}" + nginx:alpine > /dev/null; + then + echo "Could not start test web server for $domain" + elif [[ "${DRY_RUN:-}" == 1 ]]; then + echo "Started test web server for $domain" + fi - i=$(( $i + 1 )) + i=$(( i + 1 )) done i=0 for domain in "${domains[@]}"; do if [ "${networks_map[i]}" != "$desired_network" ]; then - echo "$domain is not in $desired_network, cert should not be generated"; + [[ "${DRY_RUN:-}" == 1 ]] && echo "$domain is not in $desired_network, cert should not be generated"; service_data="$(docker exec "$le_container_name" cat /app/letsencrypt_service_data)" if grep -q "$domain" <<< "$service_data"; then echo "Domain $domain is on data list, but MUST not!" else - echo "Domain $domain was not included in the service_data." + [[ "${DRY_RUN:-}" == 1 ]] && echo "Domain $domain was not included in the service_data." fi else - echo "$domain is in $desired_network, cert should be generated"; + [[ "${DRY_RUN:-}" == 1 ]] && echo "$domain is in $desired_network, cert should be generated"; # Wait for a symlink at /etc/nginx/certs/$domain.crt wait_for_symlink "$domain" "$le_container_name" fi # Stop the Nginx container silently. docker stop "$domain" > /dev/null - i=$(( $i + 1 )) + i=$(( i + 1 )) done From d52776fa22cfc5af122e99a00b05286a629e5a49 Mon Sep 17 00:00:00 2001 From: SilverFire - Dmitry Naumenko Date: Tue, 20 Oct 2020 13:35:11 +0300 Subject: [PATCH 4/5] Added documentation for the NETWORK_SCOPE env var --- docs/Container-configuration.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/Container-configuration.md b/docs/Container-configuration.md index ad499299..db61f765 100644 --- a/docs/Container-configuration.md +++ b/docs/Container-configuration.md @@ -22,3 +22,20 @@ You can also create test certificates per container (see [Test certificates](./L * `RENEW_PRIVATE_KEYS` - Set it to `false` to make `acme.sh` reuse previously generated private key for each certificate instead of creating a new one on certificate renewal. Reusing private keys can help if you intend to use [HPKP](https://developer.mozilla.org/en-US/docs/Web/HTTP/Public_Key_Pinning), but please note that HPKP has been deprecated by Google's Chrome and that it is therefore strongly discouraged to use it at all. * `DHPARAM_BITS` - Change the size of the Diffie-Hellman key generated by the container from the default value of 2048 bits. For example `--env DHPARAM_BITS=1024` to support some older clients like Java 6 and 7. + +* `NETWORK_SCOPE` – The network name, that the container requesting a certificate MUST be connected to, in order to be discovered. You may find this option useful, when the host machine has multiple public IP addresses and you want to run separate nginx-proxy containers that will handle separate services with a proper networking isolation. + +If you set this environment variable, you MUST connect the nginx-proxy container to the same network. For example: + +```bash +$ docker run --detach \ + --name nginx-proxy-letsencrypt \ + --volumes-from nginx-proxy \ + --volume /path/to/certs:/etc/nginx/certs:rw \ + --volume /var/run/docker.sock:/var/run/docker.sock:ro \ + --network domains_group_a + --env "NETWORK_SCOPE=domains_group_a" \ + jrcs/letsencrypt-nginx-proxy-companion +``` + +The created companion will discover only the containers, that are also connected to the `domains_group_a` network. \ No newline at end of file From 4af23734f18d35783b2648cddf6d071b7eea5385 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Wed, 23 Dec 2020 21:02:15 +0100 Subject: [PATCH 5/5] Run networks_segregation test --- .github/workflows/test.yml | 1 + test/config.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d46e5c1d..0d3d972d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,6 +115,7 @@ jobs: container_restart, permissions_default, permissions_custom, + networks_segregation, symlinks, ] runs-on: ubuntu-latest diff --git a/test/config.sh b/test/config.sh index 6bfd204b..b947dfa5 100755 --- a/test/config.sh +++ b/test/config.sh @@ -15,5 +15,6 @@ globalTests+=( container_restart permissions_default permissions_custom + networks_segregation symlinks )