Added verbose logs on failure

.github/workflows/linux.yml

@@ -26,7 +26,7 @@ jobs:
     - name: format
       run: cargo fmt -- --check
     - name: install rltest
-      run: python3 -m pip install RLTest gevent
+      run: python3 -m pip install git+https://github.com/RedisLabsModules/RLTest.git gevent
     - name: install redis
       run: git clone https://github.com/redis/redis; cd redis; git checkout ${{ matrix.redis_version }}; BUILD_TLS=yes make valgrind install
     - name: install valgrind

.github/workflows/macos.yml

@@ -24,7 +24,7 @@ jobs:
     - name: format
       run: cargo fmt -- --check
     - name: install rltest
-      run: python3 -m pip install RLTest gevent
+      run: python3 -m pip install git+https://github.com/RedisLabsModules/RLTest.git gevent
     - name: install redis
       run: git clone https://github.com/redis/redis; cd redis; git checkout 7.0.5; BUILD_TLS=yes make install
     - name: install automake

tests/mr_test_module/pytests/common.py

@@ -77,6 +77,24 @@ def runSkipTests():
 def waitBeforeTestStart():
     return True if os.environ.get('HOLD', False) else False
 
+def shardsConnections(env):
+    for s in range(1, env.shardsCount + 1):
+        yield env.getConnection(shardId=s)
+
+def verifyClusterInitialized(env):
+    for conn in shardsConnections(env):
+        allConnected = False
+        while not allConnected:
+            res = conn.execute_command('MRTESTS.INFOCLUSTER')
+            nodes = res[4]
+            allConnected = True
+            for n in nodes:
+                status = n[17]
+                if status != 'connected' and status != 'uninitialized':
+                    allConnected = False
+            if not allConnected:
+                time.sleep(0.1)
+
 def MRTestDecorator(skipTest=False, skipOnSingleShard=False, skipOnCluster=False, skipOnValgrind=False, envArgs={}):
     def test_func_generator(test_function):
         def test_func():
@@ -99,7 +117,11 @@ def test_func():
                 'env': env,
                 'conn': conn
             }
-            env.broadcast('MRTESTS.REFRESHCLUSTER')
+            if env.isCluster():
+                env.broadcast('CONFIG', 'set', 'cluster-node-timeout', '60000')
+                env.broadcast('MRTESTS.REFRESHCLUSTER')
+                with TimeLimit(2):
+                    verifyClusterInitialized(env)
             if waitBeforeTestStart():
                 input('\tpress any button to continue test %s' % test_name)
             test_function(**args)

tests/mr_test_module/pytests/run_full_tests.sh

@@ -3,19 +3,19 @@ set -x
 set -e
 
 echo oss
-DEBUG=$DEBUG ./run_tests.sh --env-reuse "$@"
+DEBUG=$DEBUG ./run_tests.sh "$@"
 echo single shard cluster
-DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 1 "$@"
+DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 1 "$@"
 echo 2 shards cluster
-DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 "$@"
+DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 "$@"
 echo 3 shards cluster
-DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 "$@"
+DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 "$@"
 
 echo ssl certificates
 bash ../generate_tests_cert.sh
 
 echo 2 shards cluster ssl enabled
-DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@"
+DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 2 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@"
 
 echo 3 shards cluster ssl enabled
-DEBUG=$DEBUG ./run_tests.sh --env-reuse --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@"
+DEBUG=$DEBUG ./run_tests.sh --env oss-cluster --shards-count 3 --tls --tls-cert-file ../tests/tls/redis.crt --tls-key-file ../tests/tls/redis.key --tls-ca-cert-file ../tests/tls/ca.crt --tls-passphrase foobar "$@"

tests/mr_test_module/pytests/run_tests.sh

@@ -19,4 +19,4 @@ else
 fi
 
 
-python3 -m RLTest --module $MODULE_PATH --clear-logs "$@" --module-args "password" --oss_password "password"
+python3 -m RLTest --module $MODULE_PATH --clear-logs "$@" --module-args "password" --oss_password "password" --verbose-information-on-failure

tests/mr_test_module/src/lib.rs

@@ -121,6 +121,7 @@ fn lmr_accumulate_error(ctx: &Context, _args: Vec<RedisString>) -> RedisResult {
         .accumulate(CountAccumulator)
         .create_execution()
         .map_err(RedisError::String)?;
+    execution.set_max_idle(100000);
     let blocked_client = ctx.block_client();
     execution.set_done_hanlder(|res, errs| {
         let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client);
@@ -141,7 +142,7 @@ fn lmr_uneven_work(ctx: &Context, _args: Vec<RedisString>) -> RedisResult {
         .map(UnevenWorkMapper::new())
         .create_execution()
         .map_err(RedisError::String)?;
-    execution.set_max_idle(2000);
+    execution.set_max_idle(10000);
     let blocked_client = ctx.block_client();
     execution.set_done_hanlder(|mut res, mut errs| {
         let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client);
@@ -168,6 +169,7 @@ fn lmr_read_error(ctx: &Context, _args: Vec<RedisString>) -> RedisResult {
         .accumulate(CountAccumulator)
         .create_execution()
         .map_err(|e| RedisError::String(e))?;
+    execution.set_max_idle(100000);
     let blocked_client = ctx.block_client();
     execution.set_done_hanlder(|res, errs| {
         let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client);
@@ -267,6 +269,8 @@ fn replace_keys_values(ctx: &Context, args: Vec<RedisString>) -> RedisResult {
         .create_execution()
         .map_err(|e| RedisError::String(e))?;
 
+    execution.set_max_idle(100000);
+
     let blocked_client = ctx.block_client();
     execution.set_done_hanlder(|mut res, mut errs| {
         let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client);
@@ -290,6 +294,7 @@ fn lmr_read_string_keys(ctx: &Context, _args: Vec<RedisString>) -> RedisResult {
         .collect()
         .create_execution()
         .map_err(|e| RedisError::String(e))?;
+    execution.set_max_idle(100000);
     let blocked_client = ctx.block_client();
     execution.set_done_hanlder(|mut res, mut errs| {
         let thread_ctx = ThreadSafeContext::with_blocked_client(blocked_client);
@@ -637,7 +642,7 @@ impl MapStep for UnevenWorkMapper {
 
     fn map(&self, r: Self::InRecord) -> Result<Self::OutRecord, RustMRError> {
         if !self.is_initiator {
-            let millis = time::Duration::from_millis(30000 as u64);
+            let millis = time::Duration::from_millis(60000 as u64);
             thread::sleep(millis);
         }
         Ok(r)