From bd8b8eb662b90294e90a23d4fe7a81e9a34a597e Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Tue, 15 Aug 2023 15:13:32 +0300 Subject: [PATCH 01/12] Added verbose logs on failure --- tests/mr_test_module/pytests/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mr_test_module/pytests/run_tests.sh b/tests/mr_test_module/pytests/run_tests.sh index 5a1a22b..d3c161d 100755 --- a/tests/mr_test_module/pytests/run_tests.sh +++ b/tests/mr_test_module/pytests/run_tests.sh @@ -19,4 +19,4 @@ else fi -python3 -m RLTest --module $MODULE_PATH --clear-logs "$@" --module-args "password" --oss_password "password" +python3 -m RLTest --module $MODULE_PATH --clear-logs "$@" --module-args "password" --oss_password "password" --verbose-information-on-failure From 52badf3ff39fff66f8e59c98b49b6a5da137e545 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Wed, 16 Aug 2023 14:59:31 +0300 Subject: [PATCH 02/12] Install RLTest from master --- .github/workflows/linux.yml | 2 +- .github/workflows/macos.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 546c471..dfa2d59 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -26,7 +26,7 @@ jobs: - name: format run: cargo fmt -- --check - name: install rltest - run: python3 -m pip install RLTest gevent + run: python3 -m pip install git+https://github.com/RedisLabsModules/RLTest.git gevent - name: install redis run: git clone https://github.com/redis/redis; cd redis; git checkout ${{ matrix.redis_version }}; BUILD_TLS=yes make valgrind install - name: install valgrind diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c657132..4f62363 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -24,7 +24,7 @@ jobs: - name: format run: cargo fmt -- --check - name: install rltest - run: python3 -m pip install RLTest gevent + run: python3 -m pip install git+https://github.com/RedisLabsModules/RLTest.git gevent - name: install redis run: git clone https://github.com/redis/redis; cd redis; git checkout 7.0.5; BUILD_TLS=yes make install - name: install automake From 43ed768d811945136c3e4843fd27389142d649e2 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Wed, 16 Aug 2023 15:21:05 +0300 Subject: [PATCH 03/12] Wait for cluster to be initialised before running the tests --- tests/mr_test_module/pytests/common.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/mr_test_module/pytests/common.py b/tests/mr_test_module/pytests/common.py index d244440..4cdae96 100644 --- a/tests/mr_test_module/pytests/common.py +++ b/tests/mr_test_module/pytests/common.py @@ -77,6 +77,24 @@ def runSkipTests(): def waitBeforeTestStart(): return True if os.environ.get('HOLD', False) else False +def shardsConnections(env): + for s in range(1, env.shardsCount + 1): + yield env.getConnection(shardId=s) + +def verifyClusterInitialized(env): + for conn in shardsConnections(env): + allConnected = False + while not allConnected: + res = conn.execute_command('MRTESTS.INFOCLUSTER') + nodes = res[4] + allConnected = True + for n in nodes: + status = n[17] + if status != 'connected' and status != 'uninitialized': + allConnected = False + if not allConnected: + time.sleep(0.1) + def MRTestDecorator(skipTest=False, skipOnSingleShard=False, skipOnCluster=False, skipOnValgrind=False, envArgs={}): def test_func_generator(test_function): def test_func(): @@ -99,7 +117,10 @@ def test_func(): 'env': env, 'conn': conn } - env.broadcast('MRTESTS.REFRESHCLUSTER') + if env.isCluster(): + env.broadcast('MRTESTS.REFRESHCLUSTER') + with TimeLimit(2): + verifyClusterInitialized(env) if waitBeforeTestStart(): input('\tpress any button to continue test %s' % test_name) test_function(**args) From 3d1b269ac95bd29a765bed4f5430e94b78a375d3 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Wed, 16 Aug 2023 15:34:12 +0300 Subject: [PATCH 04/12] Avoid env reuse --- tests/mr_test_module/pytests/run_full_tests.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/mr_test_module/pytests/run_full_tests.sh b/tests/mr_test_module/pytests/run_full_tests.sh index 311c848..4b277aa 100755 --- a/tests/mr_test_module/pytests/run_full_tests.sh +++ b/tests/mr_test_module/pytests/run_full_tests.sh @@ -3,19 +3,19 @@ set -x set -e echo oss -DEBUG=$DEBUG ./run_tests.sh --env-reuse "$@" +DEBUG=$DEBUG ./run_tests.sh "$@" echo single shard cluster -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 1 "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 1 "$@" echo 2 shards cluster -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 "$@" echo 3 shards cluster -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 "$@" echo ssl certificates bash ../generate_tests_cert.sh echo 2 shards cluster ssl enabled -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" echo 3 shards cluster ssl enabled -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" From d1d0588d9a9cc4f836c38986732743e46028955b Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Wed, 16 Aug 2023 16:01:04 +0300 Subject: [PATCH 05/12] increase cluster timeout --- tests/mr_test_module/pytests/common.py | 1 + tests/mr_test_module/pytests/run_full_tests.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/mr_test_module/pytests/common.py b/tests/mr_test_module/pytests/common.py index 4cdae96..43f8bda 100644 --- a/tests/mr_test_module/pytests/common.py +++ b/tests/mr_test_module/pytests/common.py @@ -118,6 +118,7 @@ def test_func(): 'conn': conn } if env.isCluster(): + env.broadcast('CONFIG', 'set', 'cluster-node-timeout', '60000') env.broadcast('MRTESTS.REFRESHCLUSTER') with TimeLimit(2): verifyClusterInitialized(env) diff --git a/tests/mr_test_module/pytests/run_full_tests.sh b/tests/mr_test_module/pytests/run_full_tests.sh index 4b277aa..311c848 100755 --- a/tests/mr_test_module/pytests/run_full_tests.sh +++ b/tests/mr_test_module/pytests/run_full_tests.sh @@ -3,19 +3,19 @@ set -x set -e echo oss -DEBUG=$DEBUG ./run_tests.sh "$@" +DEBUG=$DEBUG ./run_tests.sh --env-reuse "$@" echo single shard cluster -DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 1 "$@" +DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 1 "$@" echo 2 shards cluster -DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 "$@" +DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 "$@" echo 3 shards cluster -DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 "$@" +DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 "$@" echo ssl certificates bash ../generate_tests_cert.sh echo 2 shards cluster ssl enabled -DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" +DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" echo 3 shards cluster ssl enabled -DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" +DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" From 311dc80e7f8ab441c54835d73180602cb07b7c29 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Wed, 16 Aug 2023 17:16:03 +0300 Subject: [PATCH 06/12] Increase max idle time on lmr_read_all_keys --- tests/mr_test_module/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index 6789cd2..0037e40 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -360,6 +360,7 @@ fn lmr_read_all_keys(ctx: &Context, _args: Vec) -> RedisResult { .collect() .create_execution() .map_err(|e| RedisError::String(e))?; + execution.set_max_idle(30000); let blocked_client = ctx.block_client(); execution.set_done_hanlder(|mut res, mut errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); From dac7e6615d194cb09870ffca29a8801ae89374b8 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Thu, 17 Aug 2023 10:48:45 +0300 Subject: [PATCH 07/12] Test fixes --- tests/mr_test_module/pytests/run_full_tests.sh | 12 ++++++------ tests/mr_test_module/src/lib.rs | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/mr_test_module/pytests/run_full_tests.sh b/tests/mr_test_module/pytests/run_full_tests.sh index 311c848..4b277aa 100755 --- a/tests/mr_test_module/pytests/run_full_tests.sh +++ b/tests/mr_test_module/pytests/run_full_tests.sh @@ -3,19 +3,19 @@ set -x set -e echo oss -DEBUG=$DEBUG ./run_tests.sh --env-reuse "$@" +DEBUG=$DEBUG ./run_tests.sh "$@" echo single shard cluster -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 1 "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 1 "$@" echo 2 shards cluster -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 "$@" echo 3 shards cluster -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 "$@" echo ssl certificates bash ../generate_tests_cert.sh echo 2 shards cluster ssl enabled -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" echo 3 shards cluster ssl enabled -DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" +DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@" diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index 0037e40..6789cd2 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -360,7 +360,6 @@ fn lmr_read_all_keys(ctx: &Context, _args: Vec) -> RedisResult { .collect() .create_execution() .map_err(|e| RedisError::String(e))?; - execution.set_max_idle(30000); let blocked_client = ctx.block_client(); execution.set_done_hanlder(|mut res, mut errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); From 6885961f55ed53e88cf8b6d1c00ca532c2e6dc85 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Thu, 17 Aug 2023 11:32:26 +0300 Subject: [PATCH 08/12] Increase max idle time on replace_keys_values --- tests/mr_test_module/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index 6789cd2..349985e 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -267,6 +267,8 @@ fn replace_keys_values(ctx: &Context, args: Vec) -> RedisResult { .create_execution() .map_err(|e| RedisError::String(e))?; + execution.set_max_idle(100000); + let blocked_client = ctx.block_client(); execution.set_done_hanlder(|mut res, mut errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); From b656a6d504190ef9143994a9835cc46f1d0b03d7 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Thu, 17 Aug 2023 12:03:43 +0300 Subject: [PATCH 09/12] Fix tests --- tests/mr_test_module/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index 349985e..c017d44 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -121,6 +121,7 @@ fn lmr_accumulate_error(ctx: &Context, _args: Vec) -> RedisResult { .accumulate(CountAccumulator) .create_execution() .map_err(RedisError::String)?; + execution.set_max_idle(100000); let blocked_client = ctx.block_client(); execution.set_done_hanlder(|res, errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); From 073829e47c97ea258f403fd89f27523bd25d44f5 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Thu, 17 Aug 2023 12:51:11 +0300 Subject: [PATCH 10/12] Fix tests --- tests/mr_test_module/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index c017d44..70f9fe7 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -293,6 +293,7 @@ fn lmr_read_string_keys(ctx: &Context, _args: Vec) -> RedisResult { .collect() .create_execution() .map_err(|e| RedisError::String(e))?; + execution.set_max_idle(100000); let blocked_client = ctx.block_client(); execution.set_done_hanlder(|mut res, mut errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); From 3c58d9ca6edc4db78ef18f966c607fa857eb7df0 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Thu, 17 Aug 2023 13:24:15 +0300 Subject: [PATCH 11/12] Tests fixes --- tests/mr_test_module/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index 70f9fe7..d4db925 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -169,6 +169,7 @@ fn lmr_read_error(ctx: &Context, _args: Vec) -> RedisResult { .accumulate(CountAccumulator) .create_execution() .map_err(|e| RedisError::String(e))?; + execution.set_max_idle(100000); let blocked_client = ctx.block_client(); execution.set_done_hanlder(|res, errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); From e00acb3a1f2cabd068da248f116cec7fb76c2812 Mon Sep 17 00:00:00 2001 From: Meir Shpilraien Date: Thu, 17 Aug 2023 13:46:33 +0300 Subject: [PATCH 12/12] Fix tests --- tests/mr_test_module/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mr_test_module/src/lib.rs b/tests/mr_test_module/src/lib.rs index d4db925..90e3bb1 100644 --- a/tests/mr_test_module/src/lib.rs +++ b/tests/mr_test_module/src/lib.rs @@ -142,7 +142,7 @@ fn lmr_uneven_work(ctx: &Context, _args: Vec) -> RedisResult { .map(UnevenWorkMapper::new()) .create_execution() .map_err(RedisError::String)?; - execution.set_max_idle(2000); + execution.set_max_idle(10000); let blocked_client = ctx.block_client(); execution.set_done_hanlder(|mut res, mut errs| { let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client); @@ -642,7 +642,7 @@ impl MapStep for UnevenWorkMapper { fn map(&self, r: Self::InRecord) -> Result { if !self.is_initiator { - let millis = time::Duration::from_millis(30000 as u64); + let millis = time::Duration::from_millis(60000 as u64); thread::sleep(millis); } Ok(r)