From c7c8e1cd049dd8f302166d0dae7fed9a9b5539df Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:09:32 +0100 Subject: [PATCH 01/10] refactor(tests): simplify percent math Don't ask how it got that bad. Signed-off-by: Egor Lazarchuk --- tests/integration_tests/performance/test_rate_limiter.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index e050a20c55d..c3526b2c9dd 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -428,13 +428,8 @@ def _run_iperf_on_host(iperf_cmd, test_microvm): def _get_percentage_difference(measured, base): """Return the percentage delta between the arguments.""" - if measured == base: - return 0 - try: - return (abs(measured - base) / base) * 100.0 - except ZeroDivisionError: - # It means base and only base is 0. - return 100.0 + assert base != 0 + return (abs(measured - base) / base) * 100.0 def _process_iperf_line(line): From 34f86e1d170a171931e10903c4643095cda10363 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:13:39 +0100 Subject: [PATCH 02/10] refactor(tests): remove retry attempt for bandwidth check There is no reason to do a single retry with specific time multiplier. If retry is indeed is needed it can be implemented with a simple loop inside or outside the function. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 42 ++----------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index c3526b2c9dd..a4952c71e4f 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -287,29 +287,12 @@ def _check_rx_rate_limit_patch(test_microvm): def _check_tx_bandwidth(test_microvm, ip, expected_kbps): - """Check that the rate-limited TX bandwidth is close to what we expect. - - At this point, a daemonized iperf3 server is expected to be running on - the host. - """ - print("Check guest TX rate-limit; expected kbps {}".format(expected_kbps)) + """Check that the rate-limited TX bandwidth is close to what we expect.""" observed_kbps = _get_tx_bandwidth_with_duration( test_microvm, ip, IPERF_TRANSMIT_TIME ) - diff_pc = _get_percentage_difference(observed_kbps, expected_kbps) - print("TX calculated diff percentage: {}\n".format(diff_pc)) - - if diff_pc >= MAX_BYTES_DIFF_PERCENTAGE: - print("Short duration test failed. Try another run with 10x duration.") - - observed_kbps = _get_tx_bandwidth_with_duration( - test_microvm, ip, 10 * IPERF_TRANSMIT_TIME - ) - diff_pc = _get_percentage_difference(observed_kbps, expected_kbps) - print("TX calculated diff percentage: {}\n".format(diff_pc)) - - assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE + assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): @@ -327,29 +310,12 @@ def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): def _check_rx_bandwidth(test_microvm, ip, expected_kbps): - """Check that the rate-limited RX bandwidth is close to what we expect. - - At this point, a daemonized iperf3 server is expected to be running on - the guest. - """ - print("Check guest RX rate-limit; expected kbps {}".format(expected_kbps)) + """Check that the rate-limited RX bandwidth is close to what we expect.""" observed_kbps = _get_rx_bandwidth_with_duration( test_microvm, ip, IPERF_TRANSMIT_TIME ) - diff_pc = _get_percentage_difference(observed_kbps, expected_kbps) - print("RX calculated diff percentage: {}\n".format(diff_pc)) - - if diff_pc >= MAX_BYTES_DIFF_PERCENTAGE: - print("Short duration test failed. Try another run with 10x duration.") - - observed_kbps = _get_rx_bandwidth_with_duration( - test_microvm, ip, 10 * IPERF_TRANSMIT_TIME - ) - diff_pc = _get_percentage_difference(observed_kbps, expected_kbps) - print("TX calculated diff percentage: {}\n".format(diff_pc)) - - assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE + assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE def _get_rx_bandwidth_with_duration(test_microvm, guest_ip, duration): From 2fe1501ec81ed1da39364a179a6e34553ded60d0 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:29:10 +0100 Subject: [PATCH 03/10] refactor(tests): don't reuse iperf in the guset Don't reuse iperf in the guest for RX tests since it was causing intermittent issues. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index a4952c71e4f..6f358229196 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -188,9 +188,6 @@ def _check_rx_rate_limiting(test_microvm): eth1 = test_microvm.iface["eth1"]["iface"] eth2 = test_microvm.iface["eth2"]["iface"] - # Start iperf server on guest. - _start_iperf_server_on_guest(test_microvm) - # First step: get the transfer rate when no rate limiting is enabled. # We are receiving the result in KBytes from iperf. print("Run guest RX iperf with no rate limiting") @@ -297,10 +294,10 @@ def _check_tx_bandwidth(test_microvm, ip, expected_kbps): def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): """Check that the rate-limited TX bandwidth is close to what we expect.""" + iperf_cmd = "{} -c {} -t {} -f KBytes -w {} -N".format( IPERF_BINARY, host_ip, duration, IPERF_TCP_WINDOW ) - iperf_out = _run_iperf_on_guest(test_microvm, iperf_cmd) print(iperf_out) @@ -320,6 +317,9 @@ def _check_rx_bandwidth(test_microvm, ip, expected_kbps): def _get_rx_bandwidth_with_duration(test_microvm, guest_ip, duration): """Check that the rate-limited RX bandwidth is close to what we expect.""" + + _start_iperf_server_on_guest(test_microvm) + iperf_cmd = "{} {} -c {} -t {} -f KBytes -w {} -N".format( test_microvm.netns.cmd_prefix(), IPERF_BINARY, @@ -351,10 +351,13 @@ def _patch_iface_bw(test_microvm, iface_id, rx_or_tx, new_bucket_size, new_refil def _start_iperf_server_on_guest(test_microvm): """Start iperf in server mode through an SSH connection.""" + kill_cmd = f"pkill {IPERF_BINARY}" + test_microvm.ssh.run(kill_cmd) + iperf_cmd = f"{IPERF_BINARY} -sD -f KBytes --logfile {GUEST_IPERF_SERVER_LOG}" test_microvm.ssh.run(iperf_cmd) - # Wait for the iperf daemon to start. + # Wait for the iperf to start. time.sleep(1) From 8e982ca60385187caad5aa970634853e0e2516b8 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:32:36 +0100 Subject: [PATCH 04/10] refactor(tests): rename _get_percentage_difference into _diff With shorter name we can inline some functions in the next commit. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index 6f358229196..2fd86db14a9 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -157,7 +157,7 @@ def _check_tx_rate_limiting(test_microvm): # Sanity check that bandwidth with no rate limiting is at least double # than the one expected when rate limiting is in place. - assert _get_percentage_difference(rate_no_limit_kbps, expected_kbps) > 100 + assert _diff(rate_no_limit_kbps, expected_kbps) > 100 # Second step: check bandwidth when rate limiting is on. print("Run guest TX iperf for rate limiting without burst") @@ -176,7 +176,7 @@ def _check_tx_rate_limiting(test_microvm): _, burst_kbps = _process_iperf_output(iperf_out) print("TX burst_kbps: {}".format(burst_kbps)) # Test that the burst bandwidth is at least as two times the rate limit. - assert _get_percentage_difference(burst_kbps, expected_kbps) > 100 + assert _diff(burst_kbps, expected_kbps) > 100 # Since the burst should be consumed, check rate limit is in place. _check_tx_bandwidth(test_microvm, eth2.host_ip, expected_kbps) @@ -203,7 +203,7 @@ def _check_rx_rate_limiting(test_microvm): # Sanity check that bandwidth with no rate limiting is at least double # than the one expected when rate limiting is in place. - assert _get_percentage_difference(rate_no_limit_kbps, expected_kbps) > 100 + assert _diff(rate_no_limit_kbps, expected_kbps) > 100 # Second step: check bandwidth when rate limiting is on. print("Run guest RX iperf for rate limiting without burst") @@ -225,7 +225,7 @@ def _check_rx_rate_limiting(test_microvm): _, burst_kbps = _process_iperf_output(iperf_out) print("RX burst_kbps: {}".format(burst_kbps)) # Test that the burst bandwidth is at least as two times the rate limit. - assert _get_percentage_difference(burst_kbps, expected_kbps) > 100 + assert _diff(burst_kbps, expected_kbps) > 100 # Since the burst should be consumed, check rate limit is in place. _check_rx_bandwidth(test_microvm, eth2.guest_ip, expected_kbps) @@ -254,7 +254,7 @@ def _check_tx_rate_limit_patch(test_microvm): ) # Check that bandwidth when rate-limit disabled is at least 1.5x larger # than the one when rate limiting was enabled. - assert _get_percentage_difference(rate_no_limit_kbps, expected_kbps) > 50 + assert _diff(rate_no_limit_kbps, expected_kbps) > 50 def _check_rx_rate_limit_patch(test_microvm): @@ -280,7 +280,7 @@ def _check_rx_rate_limit_patch(test_microvm): ) # Check that bandwidth when rate-limit disabled is at least 1.5x larger # than the one when rate limiting was enabled. - assert _get_percentage_difference(rate_no_limit_kbps, expected_kbps) > 50 + assert _diff(rate_no_limit_kbps, expected_kbps) > 50 def _check_tx_bandwidth(test_microvm, ip, expected_kbps): @@ -288,7 +288,7 @@ def _check_tx_bandwidth(test_microvm, ip, expected_kbps): observed_kbps = _get_tx_bandwidth_with_duration( test_microvm, ip, IPERF_TRANSMIT_TIME ) - diff_pc = _get_percentage_difference(observed_kbps, expected_kbps) + diff_pc = _diff(observed_kbps, expected_kbps) assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE @@ -311,7 +311,7 @@ def _check_rx_bandwidth(test_microvm, ip, expected_kbps): observed_kbps = _get_rx_bandwidth_with_duration( test_microvm, ip, IPERF_TRANSMIT_TIME ) - diff_pc = _get_percentage_difference(observed_kbps, expected_kbps) + diff_pc = _diff(observed_kbps, expected_kbps) assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE @@ -395,7 +395,7 @@ def _run_iperf_on_host(iperf_cmd, test_microvm): return stdout -def _get_percentage_difference(measured, base): +def _diff(measured, base): """Return the percentage delta between the arguments.""" assert base != 0 return (abs(measured - base) / base) * 100.0 From 711fcf00d4be255e87acbf1480e45cd044c4c112 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:38:01 +0100 Subject: [PATCH 05/10] refactor(tests): inline _check_rx_bandwidth The function is too small, so inline it. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index 2fd86db14a9..b623fb75670 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -207,7 +207,10 @@ def _check_rx_rate_limiting(test_microvm): # Second step: check bandwidth when rate limiting is on. print("Run guest RX iperf for rate limiting without burst") - _check_rx_bandwidth(test_microvm, eth1.guest_ip, expected_kbps) + observed_kbps = _get_rx_bandwidth_with_duration( + test_microvm, eth1.guest_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Third step: get the number of bytes when rate limiting is on and there is # an initial burst size from where to consume. @@ -228,7 +231,10 @@ def _check_rx_rate_limiting(test_microvm): assert _diff(burst_kbps, expected_kbps) > 100 # Since the burst should be consumed, check rate limit is in place. - _check_rx_bandwidth(test_microvm, eth2.guest_ip, expected_kbps) + observed_kbps = _get_rx_bandwidth_with_duration( + test_microvm, eth2.guest_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE def _check_tx_rate_limit_patch(test_microvm): @@ -267,11 +273,17 @@ def _check_rx_rate_limit_patch(test_microvm): # Check that an RX rate limiter can be applied to a previously unlimited # interface. _patch_iface_bw(test_microvm, "eth0", "RX", bucket_size, REFILL_TIME_MS) - _check_rx_bandwidth(test_microvm, eth0.guest_ip, expected_kbps) + observed_kbps = _get_rx_bandwidth_with_duration( + test_microvm, eth0.guest_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that an RX rate limiter can be updated. _patch_iface_bw(test_microvm, "eth1", "RX", bucket_size, REFILL_TIME_MS) - _check_rx_bandwidth(test_microvm, eth1.guest_ip, expected_kbps) + observed_kbps = _get_rx_bandwidth_with_duration( + test_microvm, eth1.guest_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that an RX rate limiter can be removed. _patch_iface_bw(test_microvm, "eth0", "RX", 0, 0) @@ -306,15 +318,6 @@ def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): return observed_kbps -def _check_rx_bandwidth(test_microvm, ip, expected_kbps): - """Check that the rate-limited RX bandwidth is close to what we expect.""" - observed_kbps = _get_rx_bandwidth_with_duration( - test_microvm, ip, IPERF_TRANSMIT_TIME - ) - diff_pc = _diff(observed_kbps, expected_kbps) - assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE - - def _get_rx_bandwidth_with_duration(test_microvm, guest_ip, duration): """Check that the rate-limited RX bandwidth is close to what we expect.""" From 290d714f005373b6ba17ac885f37f760157540f4 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:45:54 +0100 Subject: [PATCH 06/10] refactor(tests): remove duration from _get_rx_bandwidth_with_duration The duration was always the same, so no need to have it as an argument. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index b623fb75670..c79be02afb4 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -191,9 +191,7 @@ def _check_rx_rate_limiting(test_microvm): # First step: get the transfer rate when no rate limiting is enabled. # We are receiving the result in KBytes from iperf. print("Run guest RX iperf with no rate limiting") - rate_no_limit_kbps = _get_rx_bandwidth_with_duration( - test_microvm, eth0.guest_ip, IPERF_TRANSMIT_TIME - ) + rate_no_limit_kbps = _get_rx_bandwidth(test_microvm, eth0.guest_ip) print("RX rate_no_limit_kbps: {}".format(rate_no_limit_kbps)) # Calculate the number of bytes that are expected to be sent @@ -207,9 +205,7 @@ def _check_rx_rate_limiting(test_microvm): # Second step: check bandwidth when rate limiting is on. print("Run guest RX iperf for rate limiting without burst") - observed_kbps = _get_rx_bandwidth_with_duration( - test_microvm, eth1.guest_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_rx_bandwidth(test_microvm, eth1.guest_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Third step: get the number of bytes when rate limiting is on and there is @@ -231,9 +227,7 @@ def _check_rx_rate_limiting(test_microvm): assert _diff(burst_kbps, expected_kbps) > 100 # Since the burst should be consumed, check rate limit is in place. - observed_kbps = _get_rx_bandwidth_with_duration( - test_microvm, eth2.guest_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_rx_bandwidth(test_microvm, eth2.guest_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE @@ -273,23 +267,17 @@ def _check_rx_rate_limit_patch(test_microvm): # Check that an RX rate limiter can be applied to a previously unlimited # interface. _patch_iface_bw(test_microvm, "eth0", "RX", bucket_size, REFILL_TIME_MS) - observed_kbps = _get_rx_bandwidth_with_duration( - test_microvm, eth0.guest_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_rx_bandwidth(test_microvm, eth0.guest_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that an RX rate limiter can be updated. _patch_iface_bw(test_microvm, "eth1", "RX", bucket_size, REFILL_TIME_MS) - observed_kbps = _get_rx_bandwidth_with_duration( - test_microvm, eth1.guest_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_rx_bandwidth(test_microvm, eth1.guest_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that an RX rate limiter can be removed. _patch_iface_bw(test_microvm, "eth0", "RX", 0, 0) - rate_no_limit_kbps = _get_rx_bandwidth_with_duration( - test_microvm, eth0.guest_ip, IPERF_TRANSMIT_TIME - ) + rate_no_limit_kbps = _get_rx_bandwidth(test_microvm, eth0.guest_ip) # Check that bandwidth when rate-limit disabled is at least 1.5x larger # than the one when rate limiting was enabled. assert _diff(rate_no_limit_kbps, expected_kbps) > 50 @@ -318,7 +306,7 @@ def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): return observed_kbps -def _get_rx_bandwidth_with_duration(test_microvm, guest_ip, duration): +def _get_rx_bandwidth(test_microvm, guest_ip): """Check that the rate-limited RX bandwidth is close to what we expect.""" _start_iperf_server_on_guest(test_microvm) @@ -327,7 +315,7 @@ def _get_rx_bandwidth_with_duration(test_microvm, guest_ip, duration): test_microvm.netns.cmd_prefix(), IPERF_BINARY, guest_ip, - duration, + IPERF_TRANSMIT_TIME, IPERF_TCP_WINDOW, ) iperf_out = _run_iperf_on_host(iperf_cmd, test_microvm) From 1e77dcf50847afd97ddc8a6ecabe82b7b3d602b2 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:54:06 +0100 Subject: [PATCH 07/10] refactor(tests): inline _check_tx_bandwidth The function is too small, so inline it. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index c79be02afb4..1c1191a2a3c 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -161,7 +161,10 @@ def _check_tx_rate_limiting(test_microvm): # Second step: check bandwidth when rate limiting is on. print("Run guest TX iperf for rate limiting without burst") - _check_tx_bandwidth(test_microvm, eth1.host_ip, expected_kbps) + observed_kbps = _get_tx_bandwidth_with_duration( + test_microvm, eth1.host_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Third step: get the number of bytes when rate limiting is on and there is # an initial burst size from where to consume. @@ -179,7 +182,10 @@ def _check_tx_rate_limiting(test_microvm): assert _diff(burst_kbps, expected_kbps) > 100 # Since the burst should be consumed, check rate limit is in place. - _check_tx_bandwidth(test_microvm, eth2.host_ip, expected_kbps) + observed_kbps = _get_tx_bandwidth_with_duration( + test_microvm, eth2.host_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE def _check_rx_rate_limiting(test_microvm): @@ -241,11 +247,17 @@ def _check_tx_rate_limit_patch(test_microvm): # Check that a TX rate limiter can be applied to a previously unlimited # interface. _patch_iface_bw(test_microvm, "eth0", "TX", bucket_size, REFILL_TIME_MS) - _check_tx_bandwidth(test_microvm, eth0.host_ip, expected_kbps) + observed_kbps = _get_tx_bandwidth_with_duration( + test_microvm, eth0.host_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that a TX rate limiter can be updated. _patch_iface_bw(test_microvm, "eth1", "TX", bucket_size, REFILL_TIME_MS) - _check_tx_bandwidth(test_microvm, eth1.host_ip, expected_kbps) + observed_kbps = _get_tx_bandwidth_with_duration( + test_microvm, eth1.host_ip, IPERF_TRANSMIT_TIME + ) + assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that a TX rate limiter can be removed. _patch_iface_bw(test_microvm, "eth0", "TX", 0, 0) @@ -283,15 +295,6 @@ def _check_rx_rate_limit_patch(test_microvm): assert _diff(rate_no_limit_kbps, expected_kbps) > 50 -def _check_tx_bandwidth(test_microvm, ip, expected_kbps): - """Check that the rate-limited TX bandwidth is close to what we expect.""" - observed_kbps = _get_tx_bandwidth_with_duration( - test_microvm, ip, IPERF_TRANSMIT_TIME - ) - diff_pc = _diff(observed_kbps, expected_kbps) - assert diff_pc < MAX_BYTES_DIFF_PERCENTAGE - - def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): """Check that the rate-limited TX bandwidth is close to what we expect.""" From 1135450537bbb67a6cb712db80a20dc43e9b3d9a Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 17:55:56 +0100 Subject: [PATCH 08/10] refactor(tests): remove duration from _get_tx_bandwidth_with_duration The duration was always the same, so no need to have it as an argument. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index 1c1191a2a3c..d8a795ad3f9 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -145,9 +145,7 @@ def _check_tx_rate_limiting(test_microvm): # First step: get the transfer rate when no rate limiting is enabled. # We are receiving the result in KBytes from iperf. print("Run guest TX iperf for no rate limiting") - rate_no_limit_kbps = _get_tx_bandwidth_with_duration( - test_microvm, eth0.host_ip, IPERF_TRANSMIT_TIME - ) + rate_no_limit_kbps = _get_tx_bandwidth(test_microvm, eth0.host_ip) print("TX rate_no_limit_kbps: {}".format(rate_no_limit_kbps)) # Calculate the number of bytes that are expected to be sent @@ -161,9 +159,7 @@ def _check_tx_rate_limiting(test_microvm): # Second step: check bandwidth when rate limiting is on. print("Run guest TX iperf for rate limiting without burst") - observed_kbps = _get_tx_bandwidth_with_duration( - test_microvm, eth1.host_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_tx_bandwidth(test_microvm, eth1.host_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Third step: get the number of bytes when rate limiting is on and there is @@ -182,9 +178,7 @@ def _check_tx_rate_limiting(test_microvm): assert _diff(burst_kbps, expected_kbps) > 100 # Since the burst should be consumed, check rate limit is in place. - observed_kbps = _get_tx_bandwidth_with_duration( - test_microvm, eth2.host_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_tx_bandwidth(test_microvm, eth2.host_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE @@ -247,23 +241,17 @@ def _check_tx_rate_limit_patch(test_microvm): # Check that a TX rate limiter can be applied to a previously unlimited # interface. _patch_iface_bw(test_microvm, "eth0", "TX", bucket_size, REFILL_TIME_MS) - observed_kbps = _get_tx_bandwidth_with_duration( - test_microvm, eth0.host_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_tx_bandwidth(test_microvm, eth0.host_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that a TX rate limiter can be updated. _patch_iface_bw(test_microvm, "eth1", "TX", bucket_size, REFILL_TIME_MS) - observed_kbps = _get_tx_bandwidth_with_duration( - test_microvm, eth1.host_ip, IPERF_TRANSMIT_TIME - ) + observed_kbps = _get_tx_bandwidth(test_microvm, eth1.host_ip) assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE # Check that a TX rate limiter can be removed. _patch_iface_bw(test_microvm, "eth0", "TX", 0, 0) - rate_no_limit_kbps = _get_tx_bandwidth_with_duration( - test_microvm, eth0.host_ip, IPERF_TRANSMIT_TIME - ) + rate_no_limit_kbps = _get_tx_bandwidth(test_microvm, eth0.host_ip) # Check that bandwidth when rate-limit disabled is at least 1.5x larger # than the one when rate limiting was enabled. assert _diff(rate_no_limit_kbps, expected_kbps) > 50 @@ -295,11 +283,11 @@ def _check_rx_rate_limit_patch(test_microvm): assert _diff(rate_no_limit_kbps, expected_kbps) > 50 -def _get_tx_bandwidth_with_duration(test_microvm, host_ip, duration): +def _get_tx_bandwidth(test_microvm, host_ip): """Check that the rate-limited TX bandwidth is close to what we expect.""" iperf_cmd = "{} -c {} -t {} -f KBytes -w {} -N".format( - IPERF_BINARY, host_ip, duration, IPERF_TCP_WINDOW + IPERF_BINARY, host_ip, IPERF_TRANSMIT_TIME, IPERF_TCP_WINDOW ) iperf_out = _run_iperf_on_guest(test_microvm, iperf_cmd) print(iperf_out) From 90c31c3cbc1fb694f95bca74d51c689ed7a5ab34 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 12 Sep 2025 18:04:58 +0100 Subject: [PATCH 09/10] refactor(tests): don't reuse iperf on the host Similar to the RX tests, in TX tests we can just use one shot servers on the host. This should prevent any instability with host iperf caused by reuse. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index d8a795ad3f9..969e53a2e41 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -139,9 +139,6 @@ def _check_tx_rate_limiting(test_microvm): eth1 = test_microvm.iface["eth1"]["iface"] eth2 = test_microvm.iface["eth2"]["iface"] - # Start iperf server on the host as this is the tx rate limiting test. - _start_iperf_server_on_host(test_microvm.netns.cmd_prefix()) - # First step: get the transfer rate when no rate limiting is enabled. # We are receiving the result in KBytes from iperf. print("Run guest TX iperf for no rate limiting") @@ -286,6 +283,8 @@ def _check_rx_rate_limit_patch(test_microvm): def _get_tx_bandwidth(test_microvm, host_ip): """Check that the rate-limited TX bandwidth is close to what we expect.""" + _start_iperf_server_on_host(test_microvm.netns.cmd_prefix()) + iperf_cmd = "{} -c {} -t {} -f KBytes -w {} -N".format( IPERF_BINARY, host_ip, IPERF_TRANSMIT_TIME, IPERF_TCP_WINDOW ) @@ -350,14 +349,10 @@ def _run_iperf_on_guest(test_microvm, iperf_cmd): def _start_iperf_server_on_host(netns_cmd_prefix): """Start iperf in server mode after killing any leftover iperf daemon.""" - iperf_cmd = "pkill {}\n".format(IPERF_BINARY) - - # Don't check the result of this command because it can fail if no iperf - # is running. - utils.run_cmd(iperf_cmd) + kill_cmd = f"pkill {IPERF_BINARY}" + utils.run_cmd(kill_cmd) iperf_cmd = "{} {} -sD -f KBytes\n".format(netns_cmd_prefix, IPERF_BINARY) - utils.check_output(iperf_cmd) # Wait for the iperf daemon to start. From dfd5113c7ad28d1ba9db0090308d62081cd7d81c Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Mon, 15 Sep 2025 14:22:53 +0100 Subject: [PATCH 10/10] refactor(tests): rename _diff into _relative_change Give a function more correct name and stop multiplying resulting value by 100. Signed-off-by: Egor Lazarchuk --- .../performance/test_rate_limiter.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/integration_tests/performance/test_rate_limiter.py b/tests/integration_tests/performance/test_rate_limiter.py index 969e53a2e41..5aef248b118 100644 --- a/tests/integration_tests/performance/test_rate_limiter.py +++ b/tests/integration_tests/performance/test_rate_limiter.py @@ -41,7 +41,7 @@ # Deltas that are accepted between expected values and achieved # values throughout the tests -MAX_BYTES_DIFF_PERCENTAGE = 10 +MAX_RELATIVE_KBPS_CHANGE = 0.1 MAX_TIME_DIFF = 25 @@ -152,12 +152,12 @@ def _check_tx_rate_limiting(test_microvm): # Sanity check that bandwidth with no rate limiting is at least double # than the one expected when rate limiting is in place. - assert _diff(rate_no_limit_kbps, expected_kbps) > 100 + assert _relative_change(rate_no_limit_kbps, expected_kbps) > 1.0 # Second step: check bandwidth when rate limiting is on. print("Run guest TX iperf for rate limiting without burst") observed_kbps = _get_tx_bandwidth(test_microvm, eth1.host_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE # Third step: get the number of bytes when rate limiting is on and there is # an initial burst size from where to consume. @@ -172,11 +172,11 @@ def _check_tx_rate_limiting(test_microvm): _, burst_kbps = _process_iperf_output(iperf_out) print("TX burst_kbps: {}".format(burst_kbps)) # Test that the burst bandwidth is at least as two times the rate limit. - assert _diff(burst_kbps, expected_kbps) > 100 + assert _relative_change(burst_kbps, expected_kbps) > 1.0 # Since the burst should be consumed, check rate limit is in place. observed_kbps = _get_tx_bandwidth(test_microvm, eth2.host_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE def _check_rx_rate_limiting(test_microvm): @@ -198,12 +198,12 @@ def _check_rx_rate_limiting(test_microvm): # Sanity check that bandwidth with no rate limiting is at least double # than the one expected when rate limiting is in place. - assert _diff(rate_no_limit_kbps, expected_kbps) > 100 + assert _relative_change(rate_no_limit_kbps, expected_kbps) > 1.0 # Second step: check bandwidth when rate limiting is on. print("Run guest RX iperf for rate limiting without burst") observed_kbps = _get_rx_bandwidth(test_microvm, eth1.guest_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE # Third step: get the number of bytes when rate limiting is on and there is # an initial burst size from where to consume. @@ -221,11 +221,11 @@ def _check_rx_rate_limiting(test_microvm): _, burst_kbps = _process_iperf_output(iperf_out) print("RX burst_kbps: {}".format(burst_kbps)) # Test that the burst bandwidth is at least as two times the rate limit. - assert _diff(burst_kbps, expected_kbps) > 100 + assert _relative_change(burst_kbps, expected_kbps) > 1.0 # Since the burst should be consumed, check rate limit is in place. observed_kbps = _get_rx_bandwidth(test_microvm, eth2.guest_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE def _check_tx_rate_limit_patch(test_microvm): @@ -239,19 +239,19 @@ def _check_tx_rate_limit_patch(test_microvm): # interface. _patch_iface_bw(test_microvm, "eth0", "TX", bucket_size, REFILL_TIME_MS) observed_kbps = _get_tx_bandwidth(test_microvm, eth0.host_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE # Check that a TX rate limiter can be updated. _patch_iface_bw(test_microvm, "eth1", "TX", bucket_size, REFILL_TIME_MS) observed_kbps = _get_tx_bandwidth(test_microvm, eth1.host_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE # Check that a TX rate limiter can be removed. _patch_iface_bw(test_microvm, "eth0", "TX", 0, 0) rate_no_limit_kbps = _get_tx_bandwidth(test_microvm, eth0.host_ip) # Check that bandwidth when rate-limit disabled is at least 1.5x larger # than the one when rate limiting was enabled. - assert _diff(rate_no_limit_kbps, expected_kbps) > 50 + assert _relative_change(rate_no_limit_kbps, expected_kbps) > 0.5 def _check_rx_rate_limit_patch(test_microvm): @@ -265,19 +265,19 @@ def _check_rx_rate_limit_patch(test_microvm): # interface. _patch_iface_bw(test_microvm, "eth0", "RX", bucket_size, REFILL_TIME_MS) observed_kbps = _get_rx_bandwidth(test_microvm, eth0.guest_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE # Check that an RX rate limiter can be updated. _patch_iface_bw(test_microvm, "eth1", "RX", bucket_size, REFILL_TIME_MS) observed_kbps = _get_rx_bandwidth(test_microvm, eth1.guest_ip) - assert _diff(observed_kbps, expected_kbps) < MAX_BYTES_DIFF_PERCENTAGE + assert _relative_change(observed_kbps, expected_kbps) < MAX_RELATIVE_KBPS_CHANGE # Check that an RX rate limiter can be removed. _patch_iface_bw(test_microvm, "eth0", "RX", 0, 0) rate_no_limit_kbps = _get_rx_bandwidth(test_microvm, eth0.guest_ip) # Check that bandwidth when rate-limit disabled is at least 1.5x larger # than the one when rate limiting was enabled. - assert _diff(rate_no_limit_kbps, expected_kbps) > 50 + assert _relative_change(rate_no_limit_kbps, expected_kbps) > 0.5 def _get_tx_bandwidth(test_microvm, host_ip): @@ -372,10 +372,10 @@ def _run_iperf_on_host(iperf_cmd, test_microvm): return stdout -def _diff(measured, base): +def _relative_change(measured, base): """Return the percentage delta between the arguments.""" assert base != 0 - return (abs(measured - base) / base) * 100.0 + return abs(measured - base) / base def _process_iperf_line(line):