From 4c0e7b38453601dd77d23a235fc29a2f4f43bf3d Mon Sep 17 00:00:00 2001 From: z8674558 Date: Fri, 11 Dec 2020 17:26:29 +0900 Subject: [PATCH 1/2] test(sentinel): add docker-compose for sentinel, modify github actions --- .ci/docker-compose-cluster-tls.yml | 2 +- .ci/docker-compose-cluster.yml | 2 +- .ci/docker-compose-sentinel.yml | 38 ++++++++++++ .ci/redis/redis.conf | 3 - .ci/redis/redis.sh | 83 ++++++++++++++++++++------- .ci/redis/sentinel.conf | 3 + .github/workflows/run_test_cases.yaml | 25 +++++++- test/emqx_auth_redis_SUITE.erl | 8 +-- 8 files changed, 132 insertions(+), 32 deletions(-) create mode 100644 .ci/docker-compose-sentinel.yml delete mode 100644 .ci/redis/redis.conf create mode 100644 .ci/redis/sentinel.conf diff --git a/.ci/docker-compose-cluster-tls.yml b/.ci/docker-compose-cluster-tls.yml index cde1875..ee9e6ea 100644 --- a/.ci/docker-compose-cluster-tls.yml +++ b/.ci/docker-compose-cluster-tls.yml @@ -19,7 +19,7 @@ services: volumes: - ../test/emqx_auth_redis_SUITE_data/certs:/tls - ./redis/:/data/conf - command: bash -c "/bin/bash /data/conf/redis.sh -t && while true; do echo 1; sleep 1; done" + command: bash -c "/bin/bash /data/conf/redis.sh -t -n cluster && while true; do echo 1; sleep 1; done" networks: app_net: # Assign a public address. Erlang container cannot find cluster nodes by network-scoped alias (redis_cluster). diff --git a/.ci/docker-compose-cluster.yml b/.ci/docker-compose-cluster.yml index e2d0231..ee543f0 100644 --- a/.ci/docker-compose-cluster.yml +++ b/.ci/docker-compose-cluster.yml @@ -18,7 +18,7 @@ services: container_name: redis-cluster volumes: - ./redis/:/data/conf - command: bash -c "/bin/bash /data/conf/redis.sh && while true; do echo 1; sleep 1; done" + command: bash -c "/bin/bash /data/conf/redis.sh -n cluster && while true; do echo 1; sleep 1; done" networks: app_net: # Assign a public address. Erlang container cannot find cluster nodes by network-scoped alias (redis_cluster). diff --git a/.ci/docker-compose-sentinel.yml b/.ci/docker-compose-sentinel.yml new file mode 100644 index 0000000..84a067a --- /dev/null +++ b/.ci/docker-compose-sentinel.yml @@ -0,0 +1,38 @@ +version: '2.4' +# network configuration is limited in version 3 +# https://github.com/docker/compose/issues/4958 + +services: + erlang: + image: erlang:22.3 + volumes: + - ../:/emqx_auth_redis + networks: + - app_net + depends_on: + - redis_cluster + tty: true + + redis_cluster: + image: redis:${REDIS_TAG} + container_name: redis-cluster + volumes: + - ./redis/:/data/conf + command: bash -c "/bin/bash /data/conf/redis.sh -n sentinel && while true; do echo 1; sleep 1; done" + networks: + app_net: + # Assign a public address. Erlang container cannot find cluster nodes by network-scoped alias (redis_cluster). + ipv4_address: 172.16.239.10 + ipv6_address: 2001:3200:3200::20 + +networks: + app_net: + driver: bridge + enable_ipv6: true + ipam: + driver: default + config: + - subnet: 172.16.239.0/24 + gateway: 172.16.239.1 + - subnet: 2001:3200:3200::/64 + gateway: 2001:3200:3200::1 \ No newline at end of file diff --git a/.ci/redis/redis.conf b/.ci/redis/redis.conf deleted file mode 100644 index ceb9969..0000000 --- a/.ci/redis/redis.conf +++ /dev/null @@ -1,3 +0,0 @@ -cluster-enabled yes -cluster-node-timeout 10000 -bind 0.0.0.0 :: \ No newline at end of file diff --git a/.ci/redis/redis.sh b/.ci/redis/redis.sh index 44da4ab..61d611a 100755 --- a/.ci/redis/redis.sh +++ b/.ci/redis/redis.sh @@ -1,9 +1,11 @@ #!/bin/bash tls=false; -while getopts t OPT +while getopts n:t OPT do case $OPT in + n) nodes=$OPTARG + ;; t) tls=true ;; \?) exit @@ -19,26 +21,58 @@ rm -f \ /data/conf/nodes.7001.conf \ /data/conf/nodes.7002.conf ; -if $tls ; then - redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes \ - --tls-port 8000 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt - redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes \ - --tls-port 8001 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt - redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes \ - --tls-port 8002 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt -else - redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes ; - redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes ; - redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes ; +if [ ${nodes} = "cluster" ] ; then + if $tls ; then + redis-server --cluster-enabled yes --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes \ + --tls-port 8000 \ + --tls-cert-file /tls/redis.crt \ + --tls-key-file /tls/redis.key \ + --tls-ca-cert-file /tls/ca.crt \ + --bind 0.0.0.0 :: + redis-server --cluster-enabled yes --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes \ + --tls-port 8001 \ + --tls-cert-file /tls/redis.crt \ + --tls-key-file /tls/redis.key \ + --tls-ca-cert-file /tls/ca.crt \ + --bind 0.0.0.0 :: + redis-server --cluster-enabled yes --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes \ + --tls-port 8002 \ + --tls-cert-file /tls/redis.crt \ + --tls-key-file /tls/redis.key \ + --tls-ca-cert-file /tls/ca.crt \ + --bind 0.0.0.0 :: + else + redis-server --cluster-enabled yes --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes --bind 0.0.0.0 :: + redis-server --cluster-enabled yes --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes --bind 0.0.0.0 :: + redis-server --cluster-enabled yes --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes --bind 0.0.0.0 :: + fi +elif [ ${nodes} = "sentinel" ] ; then + if $tls ; then + redis-server --cluster-enabled no --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes \ + --tls-port 8000 \ + --tls-cert-file /tls/redis.crt \ + --tls-key-file /tls/redis.key \ + --tls-ca-cert-file /tls/ca.crt \ + --bind 0.0.0.0 :: + redis-server --cluster-enabled no --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes \ + --tls-port 8001 \ + --tls-cert-file /tls/redis.crt \ + --tls-key-file /tls/redis.key \ + --tls-ca-cert-file /tls/ca.crt \ + --bind 0.0.0.0 :: \ + --slaveof 172.16.239.10 7000 + redis-server --cluster-enabled no --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes \ + --tls-port 8002 \ + --tls-cert-file /tls/redis.crt \ + --tls-key-file /tls/redis.key \ + --tls-ca-cert-file /tls/ca.crt \ + --bind 0.0.0.0 :: \ + --slaveof 172.16.239.10 7000 + else + redis-server --cluster-enabled no --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes --bind 0.0.0.0 :: + redis-server --cluster-enabled no --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes --bind 0.0.0.0 :: --slaveof 172.16.239.10 7000 + redis-server --cluster-enabled no --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes --bind 0.0.0.0 :: --slaveof 172.16.239.10 7000 + fi fi REDIS_LOAD_FLG=true; @@ -64,7 +98,12 @@ do else continue; fi - yes "yes" | redis-cli --cluster create 172.16.239.10:7000 172.16.239.10:7001 172.16.239.10:7002; + if [ ${nodes} = "cluster" ] ; then + yes "yes" | redis-cli --cluster create 172.16.239.10:7000 172.16.239.10:7001 172.16.239.10:7002; + elif [ ${nodes} = "sentinel" ] ; then + cp /data/conf/sentinel.conf /_sentinel.conf + redis-server /_sentinel.conf --sentinel; + fi REDIS_LOAD_FLG=false; done diff --git a/.ci/redis/sentinel.conf b/.ci/redis/sentinel.conf new file mode 100644 index 0000000..c3f96c1 --- /dev/null +++ b/.ci/redis/sentinel.conf @@ -0,0 +1,3 @@ +port 26379 +dir /tmp +sentinel monitor mymaster 172.16.239.10 7000 1 diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d8aa10b..334372b 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -20,6 +20,7 @@ jobs: node_type: - single - cluster + - sentinel steps: - name: install docker-compose @@ -111,8 +112,30 @@ jobs: echo 'auth.redis.cafile = /emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt' >> ./etc/emqx_auth_redis.conf echo 'auth.redis.certfile = /emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt' >> ./etc/emqx_auth_redis.conf echo 'auth.redis.keyfile = /emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key' >> ./etc/emqx_auth_redis.conf + - name: setup + if: matrix.connect_type == 'tcp' && matrix.network_type == 'ipv4' && matrix.node_type == 'sentinel' + env: + REDIS_TAG: ${{ matrix.redis_tag}} + run: | + set -e -u -x + docker-compose -f ./.ci/docker-compose-sentinel.yml -p tests up -d + sed -i '/auth.redis.type/c auth.redis.type = sentinel' ./etc/emqx_auth_redis.conf + sed -i '/auth.redis.server/c auth.redis.server = 172.16.239.10:26379' ./etc/emqx_auth_redis.conf + echo '\n' >> ./etc/emqx_auth_redis.conf + echo 'auth.redis.sentinel = mymaster' >> ./etc/emqx_auth_redis.conf + - name: setup + if: matrix.connect_type == 'tcp' && matrix.network_type == 'ipv6' && matrix.node_type == 'sentinel' + env: + REDIS_TAG: ${{ matrix.redis_tag}} + run: | + set -e -u -x + docker-compose -f ./.ci/docker-compose-sentinel.yml -p tests up -d + sed -i '/auth.redis.type/c auth.redis.type = sentinel' ./etc/emqx_auth_redis.conf + sed -i '/auth.redis.server/c auth.redis.server = 2001:3200:3200::20:26379' ./etc/emqx_auth_redis.conf + echo '\n' >> ./etc/emqx_auth_redis.conf + echo 'auth.redis.sentinel = mymaster' >> ./etc/emqx_auth_redis.conf - name: run test cases - if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag == '6.0.9') + if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag == '6.0.9' && matrix.node_type != 'sentinel') run: | docker exec -i tests_erlang_1 sh -c "make -C /emqx_auth_redis xref" docker exec -i tests_erlang_1 sh -c "make -C /emqx_auth_redis eunit" diff --git a/test/emqx_auth_redis_SUITE.erl b/test/emqx_auth_redis_SUITE.erl index b66274c..b2fac9b 100644 --- a/test/emqx_auth_redis_SUITE.erl +++ b/test/emqx_auth_redis_SUITE.erl @@ -182,9 +182,9 @@ reload(Config) when is_list(Config) -> q(Cmd) -> {ok, Server} = application:get_env(?APP, server), case proplists:get_value(type, Server) of - single -> - {ok, Connection} = ?POOL(?APP), - eredis:q(Connection, Cmd); cluster -> - eredis_cluster:q(emqx_auth_redis, Cmd) + eredis_cluster:q(emqx_auth_redis, Cmd); + _ -> + {ok, Connection} = ?POOL(?APP), + eredis:q(Connection, Cmd) end. \ No newline at end of file From 0fb50696e3778696eccc075b50ff36b8c2eac0c0 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Sat, 12 Dec 2020 16:52:02 +0900 Subject: [PATCH 2/2] chore(ci): refactor redis.sh --- .ci/docker-compose-cluster-tls.yml | 2 +- .ci/docker-compose-cluster.yml | 2 +- .ci/docker-compose-sentinel.yml | 2 +- .ci/redis/redis-tls.conf | 5 ++ .ci/redis/redis.conf | 2 + .ci/redis/redis.sh | 101 +++++++++++------------------ 6 files changed, 49 insertions(+), 65 deletions(-) create mode 100644 .ci/redis/redis-tls.conf create mode 100644 .ci/redis/redis.conf diff --git a/.ci/docker-compose-cluster-tls.yml b/.ci/docker-compose-cluster-tls.yml index ee9e6ea..3b6656d 100644 --- a/.ci/docker-compose-cluster-tls.yml +++ b/.ci/docker-compose-cluster-tls.yml @@ -19,7 +19,7 @@ services: volumes: - ../test/emqx_auth_redis_SUITE_data/certs:/tls - ./redis/:/data/conf - command: bash -c "/bin/bash /data/conf/redis.sh -t -n cluster && while true; do echo 1; sleep 1; done" + command: bash -c "/bin/bash /data/conf/redis.sh --node cluster --tls-enabled && while true; do echo 1; sleep 1; done" networks: app_net: # Assign a public address. Erlang container cannot find cluster nodes by network-scoped alias (redis_cluster). diff --git a/.ci/docker-compose-cluster.yml b/.ci/docker-compose-cluster.yml index ee543f0..1406e67 100644 --- a/.ci/docker-compose-cluster.yml +++ b/.ci/docker-compose-cluster.yml @@ -18,7 +18,7 @@ services: container_name: redis-cluster volumes: - ./redis/:/data/conf - command: bash -c "/bin/bash /data/conf/redis.sh -n cluster && while true; do echo 1; sleep 1; done" + command: bash -c "/bin/bash /data/conf/redis.sh --node cluster && while true; do echo 1; sleep 1; done" networks: app_net: # Assign a public address. Erlang container cannot find cluster nodes by network-scoped alias (redis_cluster). diff --git a/.ci/docker-compose-sentinel.yml b/.ci/docker-compose-sentinel.yml index 84a067a..a1709c9 100644 --- a/.ci/docker-compose-sentinel.yml +++ b/.ci/docker-compose-sentinel.yml @@ -18,7 +18,7 @@ services: container_name: redis-cluster volumes: - ./redis/:/data/conf - command: bash -c "/bin/bash /data/conf/redis.sh -n sentinel && while true; do echo 1; sleep 1; done" + command: bash -c "/bin/bash /data/conf/redis.sh --node sentinel && while true; do echo 1; sleep 1; done" networks: app_net: # Assign a public address. Erlang container cannot find cluster nodes by network-scoped alias (redis_cluster). diff --git a/.ci/redis/redis-tls.conf b/.ci/redis/redis-tls.conf new file mode 100644 index 0000000..3ef09f3 --- /dev/null +++ b/.ci/redis/redis-tls.conf @@ -0,0 +1,5 @@ +daemonize yes +bind 0.0.0.0 :: +tls-cert-file /tls/redis.crt +tls-key-file /tls/redis.key +tls-ca-cert-file /tls/ca.crt \ No newline at end of file diff --git a/.ci/redis/redis.conf b/.ci/redis/redis.conf new file mode 100644 index 0000000..27eabde --- /dev/null +++ b/.ci/redis/redis.conf @@ -0,0 +1,2 @@ +daemonize yes +bind 0.0.0.0 :: \ No newline at end of file diff --git a/.ci/redis/redis.sh b/.ci/redis/redis.sh index 61d611a..7da422c 100755 --- a/.ci/redis/redis.sh +++ b/.ci/redis/redis.sh @@ -1,16 +1,26 @@ #!/bin/bash -tls=false; -while getopts n:t OPT +node=single +tls=false +while [[ $# -gt 0 ]] do - case $OPT in - n) nodes=$OPTARG - ;; - t) tls=true - ;; - \?) exit - ;; - esac +key="$1" + +case $key in + -n|--node) + node="$2" + shift # past argument + shift # past value + ;; + -t|--tls-enabled) + tls="$2" + shift # past argument + shift # past value + ;; + *) + shift # past argument + ;; +esac done rm -f \ @@ -21,60 +31,27 @@ rm -f \ /data/conf/nodes.7001.conf \ /data/conf/nodes.7002.conf ; -if [ ${nodes} = "cluster" ] ; then +if [ ${node} = "cluster" ] ; then if $tls ; then - redis-server --cluster-enabled yes --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes \ - --tls-port 8000 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt \ - --bind 0.0.0.0 :: - redis-server --cluster-enabled yes --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes \ - --tls-port 8001 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt \ - --bind 0.0.0.0 :: - redis-server --cluster-enabled yes --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes \ - --tls-port 8002 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt \ - --bind 0.0.0.0 :: + redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \ + --tls-port 8000 --cluster-enabled yes ; + redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \ + --tls-port 8001 --cluster-enabled yes; + redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \ + --tls-port 8002 --cluster-enabled yes; else - redis-server --cluster-enabled yes --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes --bind 0.0.0.0 :: - redis-server --cluster-enabled yes --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes --bind 0.0.0.0 :: - redis-server --cluster-enabled yes --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes --bind 0.0.0.0 :: - fi -elif [ ${nodes} = "sentinel" ] ; then - if $tls ; then - redis-server --cluster-enabled no --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes \ - --tls-port 8000 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt \ - --bind 0.0.0.0 :: - redis-server --cluster-enabled no --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes \ - --tls-port 8001 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt \ - --bind 0.0.0.0 :: \ - --slaveof 172.16.239.10 7000 - redis-server --cluster-enabled no --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes \ - --tls-port 8002 \ - --tls-cert-file /tls/redis.crt \ - --tls-key-file /tls/redis.key \ - --tls-ca-cert-file /tls/ca.crt \ - --bind 0.0.0.0 :: \ - --slaveof 172.16.239.10 7000 - else - redis-server --cluster-enabled no --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --daemonize yes --bind 0.0.0.0 :: - redis-server --cluster-enabled no --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --daemonize yes --bind 0.0.0.0 :: --slaveof 172.16.239.10 7000 - redis-server --cluster-enabled no --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --daemonize yes --bind 0.0.0.0 :: --slaveof 172.16.239.10 7000 + redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --cluster-enabled yes; + redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --cluster-enabled yes; + redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --cluster-enabled yes; fi +elif [ ${node} = "sentinel" ] ; then + redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \ + --cluster-enabled no; + redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \ + --cluster-enabled no --slaveof 172.16.239.10 7000; + redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \ + --cluster-enabled no --slaveof 172.16.239.10 7000; fi - REDIS_LOAD_FLG=true; while $REDIS_LOAD_FLG; @@ -98,9 +75,9 @@ do else continue; fi - if [ ${nodes} = "cluster" ] ; then + if [ ${node} = "cluster" ] ; then yes "yes" | redis-cli --cluster create 172.16.239.10:7000 172.16.239.10:7001 172.16.239.10:7002; - elif [ ${nodes} = "sentinel" ] ; then + elif [ ${node} = "sentinel" ] ; then cp /data/conf/sentinel.conf /_sentinel.conf redis-server /_sentinel.conf --sentinel; fi