From d37edc939293452e3eb87ff03442c609d89663c1 Mon Sep 17 00:00:00 2001 From: George Barnett Date: Fri, 11 Apr 2025 10:38:55 +0100 Subject: [PATCH 1/6] Speed up allocation counting tests Motivation: The allocation counting tests take around 18 minutes in CI, which is quite a while! The tests themselves don't take nearly so long, however each test is run 11 times but between each time the framework waits for allocations to become rebalanced (i.e. everything we allocated should be freed). The framework will wait up to 5 seconds between each iteration for this. About 15 or so tests take 55 seconds, meaning they're just waiting for the allocs to quiece. It turns out that dispatch doesn't always free blocks from `async` work items immediately. This is relevant because a number of tests create their own MTELGs, and so when they are shutdown we see allocations from the shutdown flow lingering for more than 5 seconds. Not only does this slow the tests down, if the allocations don't quiece, the framework ignores the result, so we lose some coverage too. Modifiications: - Add a pre-warmed singleton MTELG which tests can use, this ensures it has plenty of space in its queues so that intermediate allocs from running tests aren't seen as leaks - Cancel tasks which are scheduled to run after the test to avoid 'leaks'. - Avoid using `swift run` and just execute the test binary, this shaves off a few seconds for each test. - Print out how long each test run takes to make debugging easier Result: - Faster alloc tests --- .../run-allocation-counter.sh | 3 +- .../template/scaffolding.swift | 8 ++++ .../test_01_resources/shared.swift | 39 ++++++++++++++++++- .../test_1000_autoReadGetAndSet_sync.swift | 1 + .../test_1000_rst_connections.swift | 5 +-- ...isolated_scheduling_10000_executions.swift | 3 +- .../test_flat_schedule_10000_tasks.swift | 20 +++++++--- ...schedule_assume_isolated_10000_tasks.swift | 20 +++++++--- .../test_schedule_10000_tasks.swift | 20 +++++++--- .../test_schedule_and_run_10000_tasks.swift | 7 ++-- ...schedule_assume_isolated_10000_tasks.swift | 20 +++++++--- ...t_schedule_with_deadline_10000_tasks.swift | 20 +++++++--- ...deadline_assume_isolated_10000_tasks.swift | 19 ++++++--- .../test_scheduling_10000_executions.swift | 3 +- .../test_submit_10000_tasks.swift | 3 +- ...t_submit_assume_isolated_10000_tasks.swift | 3 +- 16 files changed, 140 insertions(+), 54 deletions(-) diff --git a/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh b/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh index 7a828934b5..dea2298968 100755 --- a/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh +++ b/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh @@ -296,8 +296,9 @@ build_package \ set -eu cd "$working_dir" swift build "${build_opts[@]}" +build_dir=$(swift build "${build_opts[@]}" --show-bin-path) for f in "${files[@]}"; do echo "- $f" - swift run "${build_opts[@]}" "$(module_name_from_path "$f")" + "${build_dir}/$(module_name_from_path "$f")" done ) diff --git a/IntegrationTests/allocation-counter-tests-framework/template/scaffolding.swift b/IntegrationTests/allocation-counter-tests-framework/template/scaffolding.swift index ca102aa1e8..924ad2bd38 100644 --- a/IntegrationTests/allocation-counter-tests-framework/template/scaffolding.swift +++ b/IntegrationTests/allocation-counter-tests-framework/template/scaffolding.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import AtomicCounter +import Dispatch import Foundation #if canImport(Darwin) @@ -139,7 +140,14 @@ func measureAll(trackFDs: Bool, _ fn: () -> Int) -> [Measurement] { } func measureAndPrint(desc: String, trackFDs: Bool, fn: () -> Int) { + let start = DispatchTime.now() let measurements = measureAll(trackFDs: trackFDs, fn) + + let end = DispatchTime.now() + let durationNanos = end.uptimeNanoseconds - start.uptimeNanoseconds + let durationSeconds = Double(durationNanos) / 1e9 + print("\(desc).duration: \(durationSeconds) (s)") + measurements.printTotalAllocations(description: desc) measurements.printRemainingAllocations(description: desc) measurements.printTotalAllocatedBytes(description: desc) diff --git a/IntegrationTests/tests_04_performance/test_01_resources/shared.swift b/IntegrationTests/tests_04_performance/test_01_resources/shared.swift index 6677fdc016..d5abc590ea 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/shared.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/shared.swift @@ -18,7 +18,44 @@ import NIOHTTP1 import NIOPosix let localhostPickPort = try! SocketAddress.makeAddressResolvingHost("127.0.0.1", port: 0) -let group = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) +let group: MultiThreadedEventLoopGroup = { + preHeatMTELGSingleton() + return MultiThreadedEventLoopGroup.singleton +}() + +extension MultiThreadedEventLoopGroup { + static var preheatedSingleton: MultiThreadedEventLoopGroup { + group + } +} + +private func preHeatMTELGSingleton() { + for loop in MultiThreadedEventLoopGroup.singleton.makeIterator() { + var futures = [EventLoopFuture]() + futures.reserveCapacity(10_000) + + for _ in 0..<10_000 { + let f = loop.submit {} + futures.append(f) + } + + for f in futures { + try! f.wait() + } + + var scheduleds = [Scheduled]() + scheduleds.reserveCapacity(10_000) + + for _ in 0..<10_000 { + let t = loop.scheduleTask(in: .hours(1)) {} + scheduleds.append(t) + } + + for t in scheduleds { + t.cancel() + } + } +} final class RepeatedRequests: ChannelInboundHandler { typealias InboundIn = HTTPClientResponsePart diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_1000_autoReadGetAndSet_sync.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_1000_autoReadGetAndSet_sync.swift index 45aa4d6a5d..3cc075d7b5 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/test_1000_autoReadGetAndSet_sync.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/test_1000_autoReadGetAndSet_sync.swift @@ -16,6 +16,7 @@ import NIOCore import NIOPosix func run(identifier: String) { + let group = MultiThreadedEventLoopGroup.preheatedSingleton MultiThreadedEventLoopGroup.withCurrentThreadAsEventLoop { loop in ServerBootstrap(group: group).bind(host: "127.0.0.1", port: 0).map { server in measure(identifier: identifier) { diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_1000_rst_connections.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_1000_rst_connections.swift index 3e8bf65af8..d0d60b7704 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/test_1000_rst_connections.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/test_1000_rst_connections.swift @@ -27,10 +27,7 @@ private final class CloseAfterTimeoutHandler: ChannelInboundHandler { } func run(identifier: String) { - let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) - defer { - try! group.syncShutdownGracefully() - } + let group = MultiThreadedEventLoopGroup.preheatedSingleton.next() let serverConnection = try! ServerBootstrap(group: group) .bind(host: "localhost", port: 0) diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_assume_isolated_scheduling_10000_executions.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_assume_isolated_scheduling_10000_executions.swift index 7087e17aa7..41df0a1559 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/test_assume_isolated_scheduling_10000_executions.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/test_assume_isolated_scheduling_10000_executions.swift @@ -17,7 +17,7 @@ import NIOPosix func run(identifier: String) { measure(identifier: identifier) { - let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + let group = MultiThreadedEventLoopGroup.preheatedSingleton.next() let loop = group.next() let dg = DispatchGroup() @@ -28,7 +28,6 @@ func run(identifier: String) { } }.wait() dg.wait() - try! group.syncShutdownGracefully() return 10_000 } } diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift index 1d80f8d43f..7b6c076e6a 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift @@ -18,24 +18,32 @@ import NIOPosix func run(identifier: String) { measure(identifier: identifier) { - let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + let group = MultiThreadedEventLoopGroup.preheatedSingleton let loop = group.next() - let counter = try! loop.submit { () -> Int in + let (counter, tasks) = try! loop.submit { () -> (Int, [Scheduled]) in + let iterations = 10_000 + var counter: Int = 0 + var tasks: [Scheduled] = [] + tasks.reserveCapacity(iterations) let deadline = NIODeadline.now() + .hours(1) - for _ in 0..<10000 { - loop.flatScheduleTask(deadline: deadline) { + for _ in 0.. Int in + let (counter, tasks) = try! loop.submit { () -> (Int, [Scheduled]) in + let iterations = 10_000 + var counter: Int = 0 + var tasks: [Scheduled] = [] + tasks.reserveCapacity(iterations) let deadline = NIODeadline.now() + .hours(1) - for _ in 0..<10000 { - loop.assumeIsolated().flatScheduleTask(deadline: deadline) { + for _ in 0.. Int in + let (counter, tasks) = try! loop.submit { () -> (Int, [Scheduled]) in + let iterations = 10_000 var counter: Int = 0 + var tasks = [Scheduled]() + tasks.reserveCapacity(iterations) - for _ in 0..<10000 { - loop.scheduleTask(in: .hours(1)) { + for _ in 0.. Int in + let (counter, tasks) = try! loop.submit { () -> (Int, [Scheduled]) in + let iterations = 10_000 var counter: Int = 0 + var tasks = [Scheduled]() + tasks.reserveCapacity(iterations) - for _ in 0..<10000 { - loop.assumeIsolated().scheduleTask(in: .hours(1)) { + for _ in 0.. Int in + let (counter, tasks) = try! loop.submit { () -> (Int, [Scheduled]) in + let iterations = 10_000 var counter: Int = 0 + var tasks = [Scheduled]() + tasks.reserveCapacity(iterations) + let deadline = NIODeadline.now() + .hours(1) - for _ in 0..<10000 { - loop.scheduleTask(deadline: deadline) { + for _ in 0.. Int in + let (counter, tasks) = try! loop.submit { () -> (Int, [Scheduled]) in + let iterations = 10_000 var counter: Int = 0 + var tasks = [Scheduled]() + tasks.reserveCapacity(iterations) let deadline = NIODeadline.now() + .hours(1) - for _ in 0..<10000 { - loop.assumeIsolated().scheduleTask(deadline: deadline) { + for _ in 0.. Int in var counter: Int = 0 @@ -32,7 +32,6 @@ func run(identifier: String) { return counter }.wait() - try! group.syncShutdownGracefully() return counter } } diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_submit_assume_isolated_10000_tasks.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_submit_assume_isolated_10000_tasks.swift index 47f16587ab..a5fbcb69d0 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/test_submit_assume_isolated_10000_tasks.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/test_submit_assume_isolated_10000_tasks.swift @@ -17,7 +17,7 @@ import NIOPosix func run(identifier: String) { measure(identifier: identifier) { - let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + let group = MultiThreadedEventLoopGroup.preheatedSingleton let loop = group.next() let counter = try! loop.submit { () -> Int in var counter: Int = 0 @@ -32,7 +32,6 @@ func run(identifier: String) { return counter }.wait() - try! group.syncShutdownGracefully() return counter } } From 95c4104c843e137b70c46eacfa637bc27e7e75c7 Mon Sep 17 00:00:00 2001 From: George Barnett Date: Fri, 11 Apr 2025 16:42:24 +0100 Subject: [PATCH 2/6] remove copy-pasta --- .../test_01_resources/test_flat_schedule_10000_tasks.swift | 2 +- .../test_01_resources/test_schedule_and_run_10000_tasks.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift index 7b6c076e6a..7098474724 100644 --- a/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift +++ b/IntegrationTests/tests_04_performance/test_01_resources/test_flat_schedule_10000_tasks.swift @@ -30,7 +30,7 @@ func run(identifier: String) { let deadline = NIODeadline.now() + .hours(1) for _ in 0.. Date: Fri, 11 Apr 2025 17:05:03 +0100 Subject: [PATCH 3/6] Update limits --- .../tests_04_performance/Thresholds/5.10.json | 22 +++++++-------- .../tests_04_performance/Thresholds/5.9.json | 20 ++++++------- .../tests_04_performance/Thresholds/6.0.json | 20 ++++++------- .../tests_04_performance/Thresholds/6.1.json | 22 +++++++-------- .../Thresholds/nightly-main.json | 28 +++++++++---------- .../Thresholds/nightly-next.json | 20 ++++++------- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.10.json b/IntegrationTests/tests_04_performance/Thresholds/5.10.json index 29a7bf1e7c..4c5a5572af 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/5.10.json +++ b/IntegrationTests/tests_04_performance/Thresholds/5.10.json @@ -19,8 +19,8 @@ "1000_udp_reqs": 6050, "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, - "1_reqs_1000_conn": 384050, - "assume_isolated_scheduling_10000_executions": 89, + "1_reqs_1000_conn": 384000, + "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, "decode_1000_ws_frames": 2050, @@ -33,8 +33,8 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 100100, - "flat_schedule_assume_isolated_10000_tasks": 90100, + "flat_schedule_10000_tasks": 110050, + "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, @@ -46,14 +46,14 @@ "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 314, "read_10000_chunks_from_file": 110050, - "schedule_10000_tasks": 40100, + "schedule_10000_tasks": 50050, "schedule_and_run_10000_tasks": 50050, - "schedule_assume_isolated_10000_tasks": 40100, - "schedule_with_deadline_10000_tasks": 40100, - "schedule_with_deadline_assume_isolated_10000_tasks": 40100, - "scheduling_10000_executions": 89, - "submit_10000_tasks": 20100, - "submit_assume_isolated_10000_tasks": 20100, + "schedule_assume_isolated_10000_tasks": 50050, + "schedule_with_deadline_10000_tasks": 50050, + "schedule_with_deadline_assume_isolated_10000_tasks": 50050, + "scheduling_10000_executions": 11, + "submit_10000_tasks": 20050, + "submit_assume_isolated_10000_tasks": 20050, "udp_1000_reqs_1_conn": 6200, "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.9.json b/IntegrationTests/tests_04_performance/Thresholds/5.9.json index 2d8cb07bff..6045869f09 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/5.9.json +++ b/IntegrationTests/tests_04_performance/Thresholds/5.9.json @@ -20,7 +20,7 @@ "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, "1_reqs_1000_conn": 393050, - "assume_isolated_scheduling_10000_executions": 89, + "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, "decode_1000_ws_frames": 2050, @@ -33,8 +33,8 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 100100, - "flat_schedule_assume_isolated_10000_tasks": 90100, + "flat_schedule_10000_tasks": 110050, + "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, @@ -46,14 +46,14 @@ "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 329, "read_10000_chunks_from_file": 110050, - "schedule_10000_tasks": 40100, + "schedule_10000_tasks": 50050, "schedule_and_run_10000_tasks": 50050, - "schedule_assume_isolated_10000_tasks": 40100, - "schedule_with_deadline_10000_tasks": 40100, - "schedule_with_deadline_assume_isolated_10000_tasks": 40100, - "scheduling_10000_executions": 89, - "submit_10000_tasks": 20100, - "submit_assume_isolated_10000_tasks": 20100, + "schedule_assume_isolated_10000_tasks": 50050, + "schedule_with_deadline_10000_tasks": 50050, + "schedule_with_deadline_assume_isolated_10000_tasks": 50050, + "scheduling_10000_executions": 11, + "submit_10000_tasks": 20050, + "submit_assume_isolated_10000_tasks": 20050, "udp_1000_reqs_1_conn": 6200, "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/6.0.json b/IntegrationTests/tests_04_performance/Thresholds/6.0.json index 0870e33436..4c5a5572af 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/6.0.json +++ b/IntegrationTests/tests_04_performance/Thresholds/6.0.json @@ -20,7 +20,7 @@ "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, "1_reqs_1000_conn": 384000, - "assume_isolated_scheduling_10000_executions": 89, + "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, "decode_1000_ws_frames": 2050, @@ -33,8 +33,8 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 100100, - "flat_schedule_assume_isolated_10000_tasks": 90100, + "flat_schedule_10000_tasks": 110050, + "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, @@ -46,14 +46,14 @@ "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 314, "read_10000_chunks_from_file": 110050, - "schedule_10000_tasks": 40100, + "schedule_10000_tasks": 50050, "schedule_and_run_10000_tasks": 50050, - "schedule_assume_isolated_10000_tasks": 40100, - "schedule_with_deadline_10000_tasks": 40100, - "schedule_with_deadline_assume_isolated_10000_tasks": 40100, - "scheduling_10000_executions": 89, - "submit_10000_tasks": 20100, - "submit_assume_isolated_10000_tasks": 20100, + "schedule_assume_isolated_10000_tasks": 50050, + "schedule_with_deadline_10000_tasks": 50050, + "schedule_with_deadline_assume_isolated_10000_tasks": 50050, + "scheduling_10000_executions": 11, + "submit_10000_tasks": 20050, + "submit_assume_isolated_10000_tasks": 20050, "udp_1000_reqs_1_conn": 6200, "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/6.1.json b/IntegrationTests/tests_04_performance/Thresholds/6.1.json index 0870e33436..08b2a6491d 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/6.1.json +++ b/IntegrationTests/tests_04_performance/Thresholds/6.1.json @@ -19,8 +19,8 @@ "1000_udp_reqs": 6050, "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, - "1_reqs_1000_conn": 384000, - "assume_isolated_scheduling_10000_executions": 89, + "1_reqs_1000_conn": 384050, + "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, "decode_1000_ws_frames": 2050, @@ -33,8 +33,8 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 100100, - "flat_schedule_assume_isolated_10000_tasks": 90100, + "flat_schedule_10000_tasks": 110050, + "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, @@ -46,14 +46,14 @@ "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 314, "read_10000_chunks_from_file": 110050, - "schedule_10000_tasks": 40100, + "schedule_10000_tasks": 50050, "schedule_and_run_10000_tasks": 50050, - "schedule_assume_isolated_10000_tasks": 40100, - "schedule_with_deadline_10000_tasks": 40100, - "schedule_with_deadline_assume_isolated_10000_tasks": 40100, - "scheduling_10000_executions": 89, - "submit_10000_tasks": 20100, - "submit_assume_isolated_10000_tasks": 20100, + "schedule_assume_isolated_10000_tasks": 50050, + "schedule_with_deadline_10000_tasks": 50050, + "schedule_with_deadline_assume_isolated_10000_tasks": 50050, + "scheduling_10000_executions": 11, + "submit_10000_tasks": 20050, + "submit_assume_isolated_10000_tasks": 20050, "udp_1000_reqs_1_conn": 6200, "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json index a5ce735741..08b2a6491d 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json @@ -20,7 +20,7 @@ "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, "1_reqs_1000_conn": 384050, - "assume_isolated_scheduling_10000_executions": 89, + "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, "decode_1000_ws_frames": 2050, @@ -33,27 +33,27 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 100100, - "flat_schedule_assume_isolated_10000_tasks": 80100, + "flat_schedule_10000_tasks": 110050, + "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, - "get_100000_headers_canonical_form": 500050, - "get_100000_headers_canonical_form_trimming_whitespace": 500050, - "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 500050, - "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 500050, + "get_100000_headers_canonical_form": 700050, + "get_100000_headers_canonical_form_trimming_whitespace": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, "modifying_1000_circular_buffer_elements": 0, "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 314, "read_10000_chunks_from_file": 110050, - "schedule_10000_tasks": 40100, + "schedule_10000_tasks": 50050, "schedule_and_run_10000_tasks": 50050, - "schedule_assume_isolated_10000_tasks": 40100, - "schedule_with_deadline_10000_tasks": 40100, - "schedule_with_deadline_assume_isolated_10000_tasks": 40100, - "scheduling_10000_executions": 89, - "submit_10000_tasks": 20100, - "submit_assume_isolated_10000_tasks": 20100, + "schedule_assume_isolated_10000_tasks": 50050, + "schedule_with_deadline_10000_tasks": 50050, + "schedule_with_deadline_assume_isolated_10000_tasks": 50050, + "scheduling_10000_executions": 11, + "submit_10000_tasks": 20050, + "submit_assume_isolated_10000_tasks": 20050, "udp_1000_reqs_1_conn": 6200, "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json index 29a7bf1e7c..08b2a6491d 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json @@ -20,7 +20,7 @@ "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, "1_reqs_1000_conn": 384050, - "assume_isolated_scheduling_10000_executions": 89, + "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, "decode_1000_ws_frames": 2050, @@ -33,8 +33,8 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 100100, - "flat_schedule_assume_isolated_10000_tasks": 90100, + "flat_schedule_10000_tasks": 110050, + "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, @@ -46,14 +46,14 @@ "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 314, "read_10000_chunks_from_file": 110050, - "schedule_10000_tasks": 40100, + "schedule_10000_tasks": 50050, "schedule_and_run_10000_tasks": 50050, - "schedule_assume_isolated_10000_tasks": 40100, - "schedule_with_deadline_10000_tasks": 40100, - "schedule_with_deadline_assume_isolated_10000_tasks": 40100, - "scheduling_10000_executions": 89, - "submit_10000_tasks": 20100, - "submit_assume_isolated_10000_tasks": 20100, + "schedule_assume_isolated_10000_tasks": 50050, + "schedule_with_deadline_10000_tasks": 50050, + "schedule_with_deadline_assume_isolated_10000_tasks": 50050, + "scheduling_10000_executions": 11, + "submit_10000_tasks": 20050, + "submit_assume_isolated_10000_tasks": 20050, "udp_1000_reqs_1_conn": 6200, "udp_1_reqs_1000_conn": 162050 } From ea290e3e685a3946cec750727dc45983c68dd53c Mon Sep 17 00:00:00 2001 From: George Barnett Date: Fri, 11 Apr 2025 17:21:38 +0100 Subject: [PATCH 4/6] update thresholds --- IntegrationTests/tests_04_performance/Thresholds/5.10.json | 2 +- IntegrationTests/tests_04_performance/Thresholds/5.9.json | 2 +- IntegrationTests/tests_04_performance/Thresholds/6.0.json | 2 +- IntegrationTests/tests_04_performance/Thresholds/6.1.json | 2 +- .../tests_04_performance/Thresholds/nightly-main.json | 2 +- .../tests_04_performance/Thresholds/nightly-next.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.10.json b/IntegrationTests/tests_04_performance/Thresholds/5.10.json index 4c5a5572af..2e40ec05a1 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/5.10.json +++ b/IntegrationTests/tests_04_performance/Thresholds/5.10.json @@ -33,7 +33,7 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 110050, + "flat_schedule_10000_tasks": 120050, "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.9.json b/IntegrationTests/tests_04_performance/Thresholds/5.9.json index 6045869f09..f230a3cace 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/5.9.json +++ b/IntegrationTests/tests_04_performance/Thresholds/5.9.json @@ -33,7 +33,7 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 110050, + "flat_schedule_10000_tasks": 120050, "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, diff --git a/IntegrationTests/tests_04_performance/Thresholds/6.0.json b/IntegrationTests/tests_04_performance/Thresholds/6.0.json index 4c5a5572af..2e40ec05a1 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/6.0.json +++ b/IntegrationTests/tests_04_performance/Thresholds/6.0.json @@ -33,7 +33,7 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 110050, + "flat_schedule_10000_tasks": 120050, "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, diff --git a/IntegrationTests/tests_04_performance/Thresholds/6.1.json b/IntegrationTests/tests_04_performance/Thresholds/6.1.json index 08b2a6491d..6ddcb66fc7 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/6.1.json +++ b/IntegrationTests/tests_04_performance/Thresholds/6.1.json @@ -33,7 +33,7 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 110050, + "flat_schedule_10000_tasks": 120050, "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json index 08b2a6491d..6ddcb66fc7 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json @@ -33,7 +33,7 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 110050, + "flat_schedule_10000_tasks": 120050, "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json index 08b2a6491d..6ddcb66fc7 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-next.json @@ -33,7 +33,7 @@ "encode_1000_ws_frames_new_buffer_with_space": 3050, "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "flat_schedule_10000_tasks": 110050, + "flat_schedule_10000_tasks": 120050, "flat_schedule_assume_isolated_10000_tasks": 110050, "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, From af19e4c236367955611d3e88c10d4a37f5726605 Mon Sep 17 00:00:00 2001 From: George Barnett Date: Fri, 11 Apr 2025 17:22:09 +0100 Subject: [PATCH 5/6] dont print all logs for each test --- .../tests_04_performance/test_01_allocation_counts.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh b/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh index 3c4a5a8840..dae1569268 100755 --- a/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh +++ b/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh @@ -45,8 +45,9 @@ observed_allocations="${observed_allocations} info "observed allocations: ${observed_allocations}" +cat "$tmp/output" # helps debugging + for test in "${all_tests[@]}"; do - cat "$tmp/output" # helps debugging while read -r test_case; do test_case=${test_case#test_*} @@ -55,7 +56,7 @@ for test in "${all_tests[@]}"; do leaked_fds=$(grep "^test_$test_case.leaked_fds:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case" max_allowed=$(jq '.'\""$test_case"\" "$here/Thresholds/$SWIFT_VERSION.json") - + assert_is_number "$max_allowed" "Malformed or nonexistent ${SWIFT_VERSION}.json thresholds file" info "$test_case: allocations not freed: $not_freed_allocations" From ad13f25046ab85610dfc3af842fcfa096036070c Mon Sep 17 00:00:00 2001 From: George Barnett Date: Fri, 11 Apr 2025 17:41:31 +0100 Subject: [PATCH 6/6] nightly main allocs --- .../tests_04_performance/Thresholds/nightly-main.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json index 6ddcb66fc7..d53f02417b 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json @@ -19,7 +19,7 @@ "1000_udp_reqs": 6050, "1000_udpbootstraps": 2050, "1000_udpconnections": 75050, - "1_reqs_1000_conn": 384050, + "1_reqs_1000_conn": 384000, "assume_isolated_scheduling_10000_executions": 11, "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, @@ -38,10 +38,10 @@ "future_assume_isolated_lots_of_callbacks": 74050, "future_erase_result": 4050, "future_lots_of_callbacks": 74050, - "get_100000_headers_canonical_form": 700050, - "get_100000_headers_canonical_form_trimming_whitespace": 700050, - "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, - "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, + "get_100000_headers_canonical_form": 500050, + "get_100000_headers_canonical_form_trimming_whitespace": 500050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 500050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 500050, "modifying_1000_circular_buffer_elements": 0, "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 314,