diff --git a/Cargo.lock b/Cargo.lock index fa8eb5c80..7789079ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,7 +235,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -246,7 +246,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -438,7 +438,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -556,7 +556,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -722,7 +722,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1156,7 +1156,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1170,7 +1170,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1181,7 +1181,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1192,7 +1192,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1223,7 +1223,7 @@ checksum = "6178a82cf56c836a3ba61a7935cdb1c49bfaa6fa4327cd5bf554a503087de26b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1255,7 +1255,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1266,7 +1266,7 @@ checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1286,7 +1286,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", "unicode-xid", ] @@ -1364,7 +1364,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1400,7 +1400,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1448,7 +1448,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1489,7 +1489,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -1821,7 +1821,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", "trybuild", ] @@ -1833,7 +1833,7 @@ checksum = "3357fc23a41e5eca883901009e0c509e9c500d66d87da970767a2ca9fd6ddeef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -2019,7 +2019,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -2262,7 +2262,7 @@ dependencies = [ "once_cell", "rand 0.8.5", "rustls 0.21.12", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "thiserror 1.0.69", "tinyvec", "tokio", @@ -3162,7 +3162,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3439,7 +3439,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3718,7 +3718,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3859,7 +3859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3895,7 +3895,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3937,7 +3937,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3950,7 +3950,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -3999,7 +3999,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -4178,7 +4178,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -4333,7 +4333,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -4427,6 +4427,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.13.0" @@ -4640,7 +4649,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -4716,7 +4725,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -4940,7 +4949,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -4963,7 +4972,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.110", + "syn 2.0.108", "tokio", "url", ] @@ -5152,9 +5161,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.110" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", @@ -5178,7 +5187,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -5295,7 +5304,7 @@ checksum = "451b374529930d7601b1eef8d32bc79ae870b6079b069401709c2a8bf9e75f36" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -5330,7 +5339,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -5341,7 +5350,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -5468,7 +5477,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -5746,7 +5755,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -5848,9 +5857,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "trybuild" -version = "1.0.114" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17e807bff86d2a06b52bca4276746584a78375055b6e45843925ce2802b335" +checksum = "559b6a626c0815c942ac98d434746138b4f89ddd6a1b8cbb168c6845fb3376c5" dependencies = [ "glob", "serde", @@ -5991,9 +6000,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.1.4" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" +checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" dependencies = [ "base64 0.22.1", "cookie_store", @@ -6001,6 +6010,7 @@ dependencies = [ "log", "percent-encoding", "rustls 0.23.34", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -6172,7 +6182,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", "wasm-bindgen-shared", ] @@ -6583,7 +6593,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -6594,7 +6604,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -7001,7 +7011,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", "synstructure", ] @@ -7022,7 +7032,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] [[package]] @@ -7042,7 +7052,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", "synstructure", ] @@ -7082,5 +7092,5 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.108", ] diff --git a/apps/freenet-ping/Cargo.toml b/apps/freenet-ping/Cargo.toml index 2c0d2e32f..79a379bde 100644 --- a/apps/freenet-ping/Cargo.toml +++ b/apps/freenet-ping/Cargo.toml @@ -4,7 +4,7 @@ members = ["contracts/ping", "app", "types"] [workspace.dependencies] # freenet-stdlib = { path = "./../../stdlib/rust", features = ["contract"] } -freenet-stdlib = { version = "0.1.14" } +freenet-stdlib = { version = "0.1.24" } freenet-ping-types = { path = "types", default-features = false } chrono = { version = "0.4", default-features = false } testresult = "0.4" @@ -19,4 +19,3 @@ debug = false codegen-units = 1 panic = 'abort' strip = true - diff --git a/apps/freenet-ping/app/Cargo.toml b/apps/freenet-ping/app/Cargo.toml index ef83d63ae..dd0b05bf8 100644 --- a/apps/freenet-ping/app/Cargo.toml +++ b/apps/freenet-ping/app/Cargo.toml @@ -10,7 +10,7 @@ testing = ["freenet-stdlib/testing", "freenet/testing"] anyhow = "1.0" chrono = { workspace = true, features = ["default"] } clap = { version = "4.5", features = ["derive"] } -freenet-stdlib = { version = "0.1.22", features = ["net"] } +freenet-stdlib = { version = "0.1.24", features = ["net"] } freenet-ping-types = { path = "../types", features = ["std", "clap"] } futures = "0.3.31" rand = "0.9.2" diff --git a/crates/core/src/client_events/session_actor.rs b/crates/core/src/client_events/session_actor.rs index a5152f341..cce77223e 100644 --- a/crates/core/src/client_events/session_actor.rs +++ b/crates/core/src/client_events/session_actor.rs @@ -48,17 +48,7 @@ use std::time::{Duration, Instant}; use tokio::sync::mpsc; use tracing::debug; -/// Time-to-live for cached pending results. Entries older than this duration are -/// eligible for removal during pruning (triggered on message processing). -/// -/// Note: Due to lazy evaluation, stale entries may persist beyond TTL during idle periods. const PENDING_RESULT_TTL: Duration = Duration::from_secs(60); - -/// Maximum number of cached pending results. When this limit is reached, LRU eviction -/// removes the oldest entry to make room for new ones. -/// -/// Note: Cache may temporarily exceed this limit between messages since enforcement -/// is lazy (triggered only during message processing). const MAX_PENDING_RESULTS: usize = 2048; /// Simple session actor for client connection refactor @@ -289,7 +279,6 @@ impl SessionActor { .or_insert_with(|| PendingResult::new(result.clone())); entry.result = result.clone(); entry.touch(); - if let Some(waiting_clients) = self.client_transactions.remove(&tx) { for client_id in waiting_clients { if entry.delivered_clients.insert(client_id) { @@ -400,23 +389,6 @@ impl SessionActor { self.client_request_ids.retain(|(_, c), _| *c != client_id); } - /// Prune stale pending results based on TTL and enforce capacity limits. - /// - /// This is the **only** cache cleanup mechanism - there is no background task. - /// Called on every message in `process_message()`. - /// - /// # Cleanup Strategy (Lazy Evaluation) - /// - /// 1. **Skip if empty**: Early return if no cached results - /// 2. **Identify active transactions**: Collect all transactions that still have waiting clients - /// 3. **TTL-based removal**: Remove inactive entries older than `PENDING_RESULT_TTL` - /// 4. **Capacity enforcement**: If still at/over `MAX_PENDING_RESULTS`, trigger LRU eviction - /// - /// # Lazy Evaluation Implications - /// - /// - During idle periods (no messages), stale entries persist in memory - /// - Cache cleanup happens only when actor receives messages - /// - Stale entries may remain beyond TTL until next message arrives fn prune_pending_results(&mut self) { if self.pending_results.is_empty() { return; @@ -460,18 +432,6 @@ impl SessionActor { } } - /// Enforce capacity limits using LRU (Least Recently Used) eviction. - /// - /// Removes the entry with the oldest `last_accessed` timestamp when the cache - /// reaches or exceeds `MAX_PENDING_RESULTS`. - /// - /// # Lazy Evaluation Note - /// - /// This is only called: - /// 1. At the end of `prune_pending_results()` if still at capacity - /// 2. Before inserting new entries when already at capacity - /// - /// Between messages, cache size may temporarily exceed the limit. fn enforce_pending_capacity(&mut self) { if self.pending_results.len() < MAX_PENDING_RESULTS { return; diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 4481ad204..babf7b6f7 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -367,6 +367,7 @@ pub(crate) enum NodeEvent { /// Register expectation for an inbound connection from the given peer. ExpectPeerConnection { peer: PeerId, + courtesy: bool, }, } @@ -444,8 +445,11 @@ impl Display for NodeEvent { "Local subscribe complete (tx: {tx}, key: {key}, subscribed: {subscribed})" ) } - NodeEvent::ExpectPeerConnection { peer } => { - write!(f, "ExpectPeerConnection (from {peer})") + NodeEvent::ExpectPeerConnection { peer, courtesy } => { + write!( + f, + "ExpectPeerConnection (from {peer}, courtesy: {courtesy})" + ) } } } diff --git a/crates/core/src/node/network_bridge/p2p_protoc.rs b/crates/core/src/node/network_bridge/p2p_protoc.rs index bd18f56f0..1e3b3e8a3 100644 --- a/crates/core/src/node/network_bridge/p2p_protoc.rs +++ b/crates/core/src/node/network_bridge/p2p_protoc.rs @@ -14,7 +14,8 @@ use std::{ sync::Arc, }; use tokio::net::UdpSocket; -use tokio::sync::mpsc::{self, error::TryRecvError, Receiver, Sender}; +use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::sync::mpsc::error::TryRecvError; use tokio::time::timeout; use tracing::Instrument; @@ -629,14 +630,14 @@ impl P2pConnManager { ) .await?; } - NodeEvent::ExpectPeerConnection { peer } => { - tracing::debug!(%peer, "ExpectPeerConnection event received; registering inbound expectation via handshake driver"); + NodeEvent::ExpectPeerConnection { peer, courtesy } => { + tracing::debug!(%peer, courtesy, "ExpectPeerConnection event received; registering inbound expectation via handshake driver"); state.outbound_handler.expect_incoming(peer.addr); if let Err(error) = handshake_cmd_sender .send(HandshakeCommand::ExpectInbound { peer: peer.clone(), transaction: None, - courtesy: false, + courtesy, }) .await { @@ -750,8 +751,12 @@ impl P2pConnManager { // Collect node information if config.include_node_info { - // Calculate location and adress if is set - let (addr, location) = if let Some(peer_id) = + // Prefer the runtime's current ring location; fall back to derivation from the peer's + // advertised address if we don't have one yet. + let current_location = + op_manager.ring.connection_manager.own_location().location; + + let (addr, fallback_location) = if let Some(peer_id) = op_manager.ring.connection_manager.get_peer_key() { let location = Location::from_address(&peer_id.addr); @@ -760,11 +765,15 @@ impl P2pConnManager { (None, None) }; + let location_str = current_location + .or(fallback_location) + .map(|loc| format!("{:.6}", loc.as_f64())); + // Always include basic node info, but only include address/location if available response.node_info = Some(NodeInfo { peer_id: ctx.key_pair.public().to_string(), is_gateway: self.is_gateway, - location: location.map(|loc| format!("{:.6}", loc.0)), + location: location_str, listening_address: addr .map(|peer_addr| peer_addr.to_string()), uptime_seconds: 0, // TODO: implement actual uptime tracking @@ -1258,6 +1267,12 @@ impl P2pConnManager { "connect_peer: registered new pending connection" ); state.outbound_handler.expect_incoming(peer_addr); + let loc_hint = Location::from_address(&peer.addr); + self.bridge + .op_manager + .ring + .connection_manager + .register_outbound_pending(&peer, Some(loc_hint)); } } @@ -1350,6 +1365,7 @@ impl P2pConnManager { } } + let mut derived_courtesy = courtesy; let peer_id = peer.unwrap_or_else(|| { tracing::info!( remote = %remote_addr, @@ -1369,15 +1385,31 @@ impl P2pConnManager { ) }); + if !derived_courtesy { + derived_courtesy = self + .bridge + .op_manager + .ring + .connection_manager + .take_pending_courtesy_by_addr(&remote_addr); + } + tracing::info!( remote = %peer_id.addr, - courtesy, + courtesy = derived_courtesy, transaction = ?transaction, "Inbound connection established" ); - self.handle_successful_connection(peer_id, connection, state, select_stream, None) - .await?; + self.handle_successful_connection( + peer_id, + connection, + state, + select_stream, + None, + derived_courtesy, + ) + .await?; } HandshakeEvent::OutboundEstablished { transaction, @@ -1391,8 +1423,15 @@ impl P2pConnManager { transaction = %transaction, "Outbound connection established" ); - self.handle_successful_connection(peer, connection, state, select_stream, None) - .await?; + self.handle_successful_connection( + peer, + connection, + state, + select_stream, + None, + courtesy, + ) + .await?; } HandshakeEvent::OutboundFailed { transaction, @@ -1507,6 +1546,7 @@ impl P2pConnManager { state: &mut EventListenerState, select_stream: &mut priority_select::ProductionPrioritySelectStream, remaining_checks: Option, + courtesy: bool, ) -> anyhow::Result<()> { let pending_txs = state .awaiting_connection_txs @@ -1582,18 +1622,41 @@ impl P2pConnManager { } if newly_inserted { - let pending_loc = self + let loc = self .bridge .op_manager .ring .connection_manager - .prune_in_transit_connection(&peer_id); - let loc = pending_loc.unwrap_or_else(|| Location::from_address(&peer_id.addr)); - self.bridge + .pending_location_hint(&peer_id) + .unwrap_or_else(|| Location::from_address(&peer_id.addr)); + let eviction_candidate = self + .bridge .op_manager .ring - .add_connection(loc, peer_id.clone(), false) + .add_connection(loc, peer_id.clone(), false, courtesy) .await; + if let Some(victim) = eviction_candidate { + if victim == peer_id { + tracing::debug!( + %peer_id, + "Courtesy eviction candidate matched current connection; skipping drop" + ); + } else { + tracing::info!( + %victim, + %peer_id, + courtesy_limit = true, + "Courtesy connection budget exceeded; dropping oldest courtesy peer" + ); + if let Err(error) = self.bridge.drop_connection(&victim).await { + tracing::warn!( + %victim, + ?error, + "Failed to drop courtesy connection after hitting budget" + ); + } + } + } } Ok(()) } @@ -1651,6 +1714,46 @@ impl P2pConnManager { } } + if let Some(sender_peer) = extract_sender_from_message(&peer_conn.msg) { + if sender_peer.peer.addr == remote_addr + || sender_peer.peer.addr.ip().is_unspecified() + { + let mut new_peer_id = sender_peer.peer.clone(); + if new_peer_id.addr.ip().is_unspecified() { + new_peer_id.addr = remote_addr; + if let Some(sender_mut) = + extract_sender_from_message_mut(&mut peer_conn.msg) + { + if sender_mut.peer.addr.ip().is_unspecified() { + sender_mut.peer.addr = remote_addr; + } + } + } + if let Some(existing_key) = self + .connections + .keys() + .find(|peer| { + peer.addr == remote_addr && peer.pub_key != new_peer_id.pub_key + }) + .cloned() + { + if let Some(channel) = self.connections.remove(&existing_key) { + tracing::info!( + remote = %remote_addr, + old_peer = %existing_key, + new_peer = %new_peer_id, + "Updating provisional peer identity after inbound message" + ); + self.bridge + .op_manager + .ring + .update_connection_identity(&existing_key, new_peer_id.clone()); + self.connections.insert(new_peer_id, channel); + } + } + } + } + // Check if we need to establish a connection back to the sender let should_connect = !self.connections.keys().any(|peer| peer.addr == remote_addr) && !state.awaiting_connection.contains_key(&remote_addr); diff --git a/crates/core/src/node/testing_impl.rs b/crates/core/src/node/testing_impl.rs index 75a49cbb9..c77a1aef0 100644 --- a/crates/core/src/node/testing_impl.rs +++ b/crates/core/src/node/testing_impl.rs @@ -940,8 +940,8 @@ where NodeEvent::QueryNodeDiagnostics { .. } => { unimplemented!() } - NodeEvent::ExpectPeerConnection { peer } => { - tracing::debug!(%peer, "ExpectPeerConnection ignored in testing impl"); + NodeEvent::ExpectPeerConnection { peer, courtesy } => { + tracing::debug!(%peer, courtesy, "ExpectPeerConnection ignored in testing impl"); continue; } }, diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index d33076c33..1c378a5b9 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -11,8 +11,8 @@ use std::time::{Duration, Instant}; use futures::{stream::FuturesUnordered, StreamExt}; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc; use tokio::task::{self, JoinHandle}; +use tokio::sync::mpsc; use crate::client_events::HostResult; use crate::dev_tool::Location; @@ -164,7 +164,7 @@ pub(crate) trait RelayContext { fn self_location(&self) -> &PeerKeyLocation; /// Determine whether we should accept the joiner immediately. - fn should_accept(&self, joiner: &PeerKeyLocation) -> bool; + fn should_accept(&self, joiner: &PeerKeyLocation, courtesy: bool) -> bool; /// Choose the next hop for the request, avoiding peers already visited. fn select_next_hop( @@ -178,10 +178,16 @@ pub(crate) trait RelayContext { } /// Result of processing a request at a relay. +#[derive(Debug, Clone)] +pub(crate) struct ExpectedConnection { + pub peer: PeerKeyLocation, + pub courtesy: bool, +} + #[derive(Debug, Default)] pub(crate) struct RelayActions { pub accept_response: Option, - pub expect_connection_from: Option, + pub expect_connection_from: Option, pub forward: Option<(PeerKeyLocation, ConnectRequest)>, pub observed_address: Option<(PeerKeyLocation, SocketAddr)>, } @@ -209,16 +215,20 @@ impl RelayState { actions.observed_address = Some((self.request.origin.clone(), observed_addr)); } - if !self.accepted_locally && ctx.should_accept(&self.request.origin) { + let acceptor = ctx.self_location().clone(); + let courtesy = ctx.courtesy_hint(&acceptor, &self.request.origin); + + if !self.accepted_locally && ctx.should_accept(&self.request.origin, courtesy) { self.accepted_locally = true; - let acceptor = ctx.self_location().clone(); - let courtesy = ctx.courtesy_hint(&acceptor, &self.request.origin); self.courtesy_hint = courtesy; actions.accept_response = Some(ConnectResponse { acceptor: acceptor.clone(), courtesy, }); - actions.expect_connection_from = Some(self.request.origin.clone()); + actions.expect_connection_from = Some(ExpectedConnection { + peer: self.request.origin.clone(), + courtesy, + }); } if self.forwarded_to.is_none() && self.request.ttl > 0 { @@ -273,14 +283,14 @@ impl RelayContext for RelayEnv<'_> { &self.self_location } - fn should_accept(&self, joiner: &PeerKeyLocation) -> bool { + fn should_accept(&self, joiner: &PeerKeyLocation, courtesy: bool) -> bool { let location = joiner .location .unwrap_or_else(|| Location::from_address(&joiner.peer.addr)); self.op_manager .ring .connection_manager - .should_accept(location, &joiner.peer) + .should_accept(location, &joiner.peer, courtesy) } fn select_next_hop( @@ -297,10 +307,7 @@ impl RelayContext for RelayEnv<'_> { } fn courtesy_hint(&self, _acceptor: &PeerKeyLocation, _joiner: &PeerKeyLocation) -> bool { - // Courtesy slots still piggyback on regular connections. Flag the first acceptance so the - // joiner can prioritise it, and keep the logic simple until dedicated courtesy tracking - // is wired in (see courtesy-connection-budget branch). - self.op_manager.ring.open_connections() == 0 + self.op_manager.ring.is_gateway() } } @@ -592,10 +599,11 @@ impl Operation for ConnectOp { .await?; } - if let Some(peer) = actions.expect_connection_from { + if let Some(expected) = actions.expect_connection_from { op_manager .notify_node_event(NodeEvent::ExpectPeerConnection { - peer: peer.peer.clone(), + peer: expected.peer.peer.clone(), + courtesy: expected.courtesy, }) .await?; } @@ -654,6 +662,7 @@ impl Operation for ConnectOp { .notify_node_event( crate::message::NodeEvent::ExpectPeerConnection { peer: new_acceptor.peer.peer.clone(), + courtesy: new_acceptor.courtesy, }, ) .await?; @@ -781,7 +790,7 @@ pub(crate) async fn join_ring_request( if !op_manager .ring .connection_manager - .should_accept(location, &gateway.peer) + .should_accept(location, &gateway.peer, false) { return Err(OpError::ConnError(ConnectionError::UnwantedConnection)); } @@ -992,7 +1001,7 @@ mod tests { &self.self_loc } - fn should_accept(&self, _joiner: &PeerKeyLocation) -> bool { + fn should_accept(&self, _joiner: &PeerKeyLocation, _courtesy: bool) -> bool { self.accept } @@ -1043,7 +1052,10 @@ mod tests { let response = actions.accept_response.expect("expected acceptance"); assert_eq!(response.acceptor.peer, self_loc.peer); assert!(response.courtesy); - assert_eq!(actions.expect_connection_from.unwrap().peer, joiner.peer); + assert_eq!( + actions.expect_connection_from.unwrap().peer.peer, + joiner.peer + ); assert!(actions.forward.is_none()); } diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 1963e87b3..3fcd6c689 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -82,6 +82,16 @@ pub(crate) async fn request_get( get_op: GetOp, skip_list: HashSet, ) -> Result<(), OpError> { + let mut skip_list = skip_list; + // Always avoid bouncing straight back to ourselves. + skip_list.insert( + op_manager + .ring + .connection_manager + .own_location() + .peer + .clone(), + ); let (mut candidates, id, key_val, _fetch_contract) = if let Some(GetState::PrepareRequest { key, id, @@ -1271,6 +1281,7 @@ async fn try_forward_or_return( let mut new_skip_list = skip_list.clone(); new_skip_list.insert(this_peer.peer.clone()); + new_skip_list.insert(sender.peer.clone()); let new_htl = htl.saturating_sub(1); diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index e4c2f96ed..33af3e728 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -1077,9 +1077,7 @@ async fn try_to_broadcast( key ); // means the whole tx finished so can return early - let upstream_for_completion = preserved_upstream - .clone() - .or_else(|| Some(upstream.clone())); + let upstream_for_completion = preserved_upstream.clone().or(Some(upstream.clone())); new_state = Some(PutState::AwaitingResponse { key, upstream: upstream_for_completion, diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index c8fab8952..c0162f5f1 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -20,7 +20,9 @@ use serde::{Deserialize, Serialize}; use tokio::time::{sleep, Duration}; const MAX_RETRIES: usize = 10; +/// Maximum time in milliseconds to wait for a locally fetched contract to become available. const LOCAL_FETCH_TIMEOUT_MS: u64 = 1_500; +/// Polling interval in milliseconds while waiting for a fetched contract to be stored locally. const LOCAL_FETCH_POLL_INTERVAL_MS: u64 = 25; fn subscribers_snapshot(op_manager: &OpManager, key: &ContractKey) -> Vec { @@ -35,6 +37,50 @@ fn subscribers_snapshot(op_manager: &OpManager, key: &ContractKey) -> Vec Result<(), ()> { + let before = subscribers_snapshot(op_manager, key); + tracing::debug!( + tx = %tx, + %key, + subscriber = %subscriber.peer, + stage, + subscribers_before = ?before, + "subscribe: attempting to register subscriber" + ); + + match op_manager.ring.add_subscriber(key, subscriber.clone()) { + Ok(()) => { + let after = subscribers_snapshot(op_manager, key); + tracing::debug!( + tx = %tx, + %key, + subscriber = %subscriber.peer, + stage, + subscribers_after = ?after, + "subscribe: registered subscriber" + ); + Ok(()) + } + Err(_) => { + tracing::warn!( + tx = %tx, + %key, + subscriber = %subscriber.peer, + stage, + subscribers_before = ?before, + "subscribe: subscriber registration failed (max subscribers reached)" + ); + Err(()) + } + } +} + /// Poll local storage for a short period until the fetched contract becomes available. async fn wait_for_local_contract( op_manager: &OpManager, @@ -646,37 +692,18 @@ impl Operation for SubscribeOp { // After fetch attempt we should now have the contract locally. } - let before_direct = subscribers_snapshot(op_manager, key); - tracing::info!( - tx = %id, - %key, - subscriber = %subscriber.peer, - subscribers_before = ?before_direct, - "subscribe: attempting to register direct subscriber" - ); - if op_manager - .ring - .add_subscriber(key, subscriber.clone()) - .is_err() + if register_subscriber_with_logging( + id, + op_manager, + key, + subscriber, + "direct subscriber", + ) + .is_err() { - tracing::warn!( - tx = %id, - %key, - subscriber = %subscriber.peer, - subscribers_before = ?before_direct, - "subscribe: direct registration failed (max subscribers reached)" - ); // max number of subscribers for this contract reached return Ok(return_not_subbed()); } - let after_direct = subscribers_snapshot(op_manager, key); - tracing::info!( - tx = %id, - %key, - subscriber = %subscriber.peer, - subscribers_after = ?after_direct, - "subscribe: registered direct subscriber" - ); match self.state { Some(SubscribeState::ReceivedRequest) => { @@ -768,7 +795,15 @@ impl Operation for SubscribeOp { upstream_subscriber, .. }) => { - fetch_contract_if_missing(op_manager, *key).await?; + if let Err(err) = fetch_contract_if_missing(op_manager, *key).await { + tracing::warn!( + tx = %id, + %key, + error = %err, + "Failed to fetch contract code after successful subscription" + ); + return Err(err); + } tracing::info!( tx = %id, @@ -787,63 +822,26 @@ impl Operation for SubscribeOp { "Handling ReturnSub (subscribed=true)" ); if let Some(upstream_subscriber) = upstream_subscriber.as_ref() { - let before_upstream = subscribers_snapshot(op_manager, key); - tracing::info!( - tx = %id, - %key, - upstream = %upstream_subscriber.peer, - subscribers_before = ?before_upstream, - "subscribe: attempting to register upstream link" + let _ = register_subscriber_with_logging( + id, + op_manager, + key, + upstream_subscriber, + "upstream link", ); - if op_manager - .ring - .add_subscriber(key, upstream_subscriber.clone()) - .is_err() - { - tracing::warn!( - tx = %id, - %key, - upstream = %upstream_subscriber.peer, - subscribers_before = ?before_upstream, - "subscribe: upstream registration failed (max subscribers reached)" - ); - } else { - let after_upstream = subscribers_snapshot(op_manager, key); - tracing::info!( - tx = %id, - %key, - upstream = %upstream_subscriber.peer, - subscribers_after = ?after_upstream, - "subscribe: registered upstream link" - ); - } } - let before_provider = subscribers_snapshot(op_manager, key); - tracing::info!( - tx = %id, - %key, - provider = %sender.peer, - subscribers_before = ?before_provider, - "subscribe: registering provider/subscription source" - ); - if op_manager.ring.add_subscriber(key, sender.clone()).is_err() { - // concurrently it reached max number of subscribers for this contract - tracing::debug!( - tx = %id, - %key, - "Max number of subscribers reached for contract" - ); + if register_subscriber_with_logging( + id, + op_manager, + key, + sender, + "provider link", + ) + .is_err() + { return Err(OpError::UnexpectedOpState); } - let after_provider = subscribers_snapshot(op_manager, key); - tracing::info!( - tx = %id, - %key, - provider = %sender.peer, - subscribers_after = ?after_provider, - "subscribe: registered provider/subscription source" - ); new_state = Some(SubscribeState::Completed { key: *key }); if let Some(upstream_subscriber) = upstream_subscriber { diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index 7743ce95b..f59aa9d10 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -4,7 +4,7 @@ use freenet_stdlib::client_api::{ErrorKind, HostResponse}; use freenet_stdlib::prelude::*; pub(crate) use self::messages::UpdateMsg; -use super::{OpEnum, OpError, OpInitialization, OpOutcome, Operation, OperationResult}; +use super::{get, OpEnum, OpError, OpInitialization, OpOutcome, Operation, OperationResult}; use crate::contract::{ContractHandlerEvent, StoreResponse}; use crate::message::{InnerMessage, NetMessage, NodeEvent, Transaction}; use crate::node::IsOperationCompleted; @@ -13,6 +13,7 @@ use crate::{ client_events::HostResult, node::{NetworkBridge, OpManager, PeerId}, }; +use std::collections::HashSet; pub(crate) struct UpdateOp { pub id: Transaction, @@ -248,9 +249,14 @@ impl Operation for UpdateOp { return_msg = None; } else { // Get broadcast targets for propagating UPDATE to subscribers - let broadcast_to = op_manager + let mut broadcast_to = op_manager .get_broadcast_targets_update(key, &request_sender.peer); + if broadcast_to.is_empty() { + broadcast_to = op_manager + .compute_update_fallback_targets(key, &request_sender.peer); + } + if broadcast_to.is_empty() { tracing::debug!( tx = %id, @@ -292,10 +298,21 @@ impl Operation for UpdateOp { } } else { // Contract not found locally - forward to another peer - let next_target = op_manager.ring.closest_potentially_caching( - key, - [&self_location.peer, &request_sender.peer].as_slice(), - ); + let skip_peers = [&self_location.peer, &request_sender.peer]; + let next_target = op_manager + .ring + .closest_potentially_caching(key, skip_peers.as_slice()) + .or_else(|| { + op_manager + .ring + .k_closest_potentially_caching( + key, + skip_peers.as_slice(), + 5, + ) + .into_iter() + .next() + }); if let Some(forward_target) = next_target { tracing::debug!( @@ -409,10 +426,14 @@ impl Operation for UpdateOp { return_msg = None; } else { // Get broadcast targets - let broadcast_to = + let mut broadcast_to = op_manager.get_broadcast_targets_update(key, &sender.peer); - // If no peers to broadcast to, nothing else to do + if broadcast_to.is_empty() { + broadcast_to = + op_manager.compute_update_fallback_targets(key, &sender.peer); + } + if broadcast_to.is_empty() { tracing::debug!( tx = %id, @@ -470,7 +491,12 @@ impl Operation for UpdateOp { }); new_state = None; } else { - // No more peers to try - capture context for diagnostics + tracing::warn!( + tx = %id, + %key, + "No forwarding targets for UPDATE SeekNode - attempting local fetch" + ); + // Capture additional context for diagnostics let skip_list = [&sender.peer, &self_location.peer]; let subscribers = op_manager .ring @@ -490,16 +516,104 @@ impl Operation for UpdateOp { .collect::>(); let connection_count = op_manager.ring.connection_manager.num_connections(); - tracing::error!( + tracing::debug!( tx = %id, %key, subscribers = ?subscribers, candidates = ?candidates, connection_count, sender = %sender.peer, - "Cannot handle UPDATE SeekNode: contract not found and no peers to forward to" + "SeekNode fallback context before attempting local fetch" ); - return Err(OpError::RingError(RingError::NoCachingPeers(*key))); + + let mut fetch_skip = HashSet::new(); + fetch_skip.insert(sender.peer.clone()); + + let get_op = get::start_op(*key, true, false); + if let Err(fetch_err) = + get::request_get(op_manager, get_op, fetch_skip).await + { + tracing::warn!( + tx = %id, + %key, + error = %fetch_err, + "Failed to fetch contract while handling UPDATE SeekNode" + ); + return Err(OpError::RingError(RingError::NoCachingPeers(*key))); + } + + if super::has_contract(op_manager, *key).await? { + tracing::info!( + tx = %id, + %key, + "Successfully fetched contract locally, applying UPDATE" + ); + let UpdateExecution { + value: updated_value, + summary: _summary, + changed, + } = update_contract( + op_manager, + *key, + value.clone(), + related_contracts.clone(), + ) + .await?; + + if !changed { + tracing::debug!( + tx = %id, + %key, + "Fetched contract apply produced no change during SeekNode fallback" + ); + new_state = None; + return_msg = None; + } else { + let mut broadcast_to = + op_manager.get_broadcast_targets_update(key, &sender.peer); + + if broadcast_to.is_empty() { + broadcast_to = op_manager + .compute_update_fallback_targets(key, &sender.peer); + } + + if broadcast_to.is_empty() { + tracing::debug!( + tx = %id, + %key, + "No broadcast targets after SeekNode fallback apply; finishing locally" + ); + new_state = None; + return_msg = None; + } else { + match try_to_broadcast( + *id, + true, + op_manager, + self.state, + (broadcast_to, sender.clone()), + *key, + updated_value.clone(), + false, + ) + .await + { + Ok((state, msg)) => { + new_state = state; + return_msg = msg; + } + Err(err) => return Err(err), + } + } + } + } else { + tracing::error!( + tx = %id, + %key, + "Contract still unavailable after fetch attempt during UPDATE SeekNode" + ); + return Err(OpError::RingError(RingError::NoCachingPeers(*key))); + } } } } @@ -533,9 +647,14 @@ impl Operation for UpdateOp { new_state = None; return_msg = None; } else { - let broadcast_to = + let mut broadcast_to = op_manager.get_broadcast_targets_update(key, &sender.peer); + if broadcast_to.is_empty() { + broadcast_to = + op_manager.compute_update_fallback_targets(key, &sender.peer); + } + tracing::debug!( "Successfully updated a value for contract {} @ {:?} - BroadcastTo - update", key, @@ -749,6 +868,40 @@ impl OpManager { subscribers } + + fn compute_update_fallback_targets( + &self, + key: &ContractKey, + sender: &PeerId, + ) -> Vec { + let mut skip: HashSet = HashSet::new(); + skip.insert(sender.clone()); + if let Some(self_peer) = self.ring.connection_manager.get_peer_key() { + skip.insert(self_peer); + } + + let candidates = self + .ring + .k_closest_potentially_caching(key, &skip, 3) + .into_iter() + .filter(|candidate| &candidate.peer != sender) + .collect::>(); + + if !candidates.is_empty() { + tracing::info!( + "UPDATE_PROPAGATION: contract={:.8} from={} using fallback targets={}", + key, + sender, + candidates + .iter() + .map(|c| format!("{:.8}", c.peer)) + .collect::>() + .join(",") + ); + } + + candidates + } } fn build_op_result( diff --git a/crates/core/src/ring/connection_manager.rs b/crates/core/src/ring/connection_manager.rs index 4f1d7023c..824d25093 100644 --- a/crates/core/src/ring/connection_manager.rs +++ b/crates/core/src/ring/connection_manager.rs @@ -1,6 +1,9 @@ +use dashmap::DashMap; use parking_lot::Mutex; use rand::prelude::IndexedRandom; -use std::collections::{btree_map::Entry, BTreeMap}; +use std::collections::{btree_map::Entry, BTreeMap, HashMap, HashSet, VecDeque}; +use std::net::SocketAddr; +use std::time::{Duration, Instant}; use crate::topology::{Limits, TopologyManager}; @@ -22,9 +25,136 @@ pub(crate) struct ConnectionManager { pub max_connections: usize, pub rnd_if_htl_above: usize, pub pub_key: Arc, + courtesy_links: Arc>>, + max_courtesy_links: usize, + pending_courtesy: Arc>>, + pending_courtesy_addr: Arc>>, + pending_connections: DashMap, +} + +#[derive(Clone)] +struct CourtesyLink { + peer: PeerId, +} + +const MAX_COURTESY_LINKS: usize = 10; +const PENDING_CONNECTION_TTL: Duration = Duration::from_secs(30); + +#[derive(Clone)] +struct PendingConnection { + inserted_at: Instant, + reserved: bool, } impl ConnectionManager { + fn cleanup_stale_pending(&self) { + let now = Instant::now(); + let mut expired = Vec::new(); + for entry in self.pending_connections.iter() { + if now.duration_since(entry.value().inserted_at) > PENDING_CONNECTION_TTL { + expired.push(entry.key().clone()); + } + } + if expired.is_empty() { + return; + } + let mut locations = self.location_for_peer.write(); + for peer in expired { + if let Some((peer_id, meta)) = self.pending_connections.remove(&peer) { + tracing::warn!(%peer_id, "pending connection timed out; releasing slot"); + if meta.reserved { + self.release_reserved_slot(Some(&peer_id), "pending_gc"); + } + locations.remove(&peer_id); + } + } + } + + fn register_pending_connection(&self, peer: &PeerId, reserved: bool) { + self.cleanup_stale_pending(); + let previous = self.pending_connections.insert( + peer.clone(), + PendingConnection { + inserted_at: Instant::now(), + reserved, + }, + ); + if let Some(prev) = previous { + tracing::debug!(%peer, reserved_previous = prev.reserved, reserved_new = reserved, "Replacing existing pending connection entry"); + if prev.reserved && !reserved { + self.release_reserved_slot(Some(peer), "pending_replaced"); + } + } + } + + fn take_pending_connection(&self, peer: &PeerId) -> Option { + self.pending_connections.remove(peer).map(|(_, meta)| meta) + } + + fn release_reserved_slot(&self, peer: Option<&PeerId>, context: &'static str) { + let mut current = self + .reserved_connections + .load(std::sync::atomic::Ordering::SeqCst); + loop { + if current == 0 { + tracing::warn!( + ?peer, + context, + "release_reserved_slot: counter already at zero" + ); + return; + } + match self.reserved_connections.compare_exchange( + current, + current - 1, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) { + Ok(_) => { + tracing::debug!( + ?peer, + previous = current, + context, + "release_reserved_slot: decremented reserved counter" + ); + return; + } + Err(actual) => current = actual, + } + } + } + + fn reserve_connection_slot(&self, peer_id: &PeerId) -> Option { + loop { + let current = self + .reserved_connections + .load(std::sync::atomic::Ordering::SeqCst); + if current == usize::MAX { + tracing::error!( + %peer_id, + "reserved connection counter overflowed; rejecting new connection" + ); + return None; + } + match self.reserved_connections.compare_exchange( + current, + current + 1, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) { + Ok(_) => return Some(current), + Err(actual) => { + tracing::debug!( + %peer_id, + expected = current, + actual, + "reserved connection counter changed concurrently; retrying" + ); + } + } + } + } + pub fn new(config: &NodeConfig) -> Self { let min_connections = if let Some(v) = config.min_number_conn { v @@ -111,6 +241,132 @@ impl ConnectionManager { max_connections, rnd_if_htl_above, pub_key: Arc::new(pub_key), + courtesy_links: Arc::new(Mutex::new(VecDeque::new())), + max_courtesy_links: if is_gateway { MAX_COURTESY_LINKS } else { 0 }, + pending_courtesy: Arc::new(Mutex::new(HashSet::new())), + pending_courtesy_addr: Arc::new(Mutex::new(HashSet::new())), + pending_connections: DashMap::new(), + } + } + + fn remember_courtesy_intent(&self, peer: &PeerId) { + if !self.is_gateway { + return; + } + let mut pending = self.pending_courtesy.lock(); + pending.insert(peer.clone()); + tracing::info!( + %peer, + pending = pending.len(), + "remember_courtesy_intent: recorded pending courtesy join" + ); + if !peer.addr.ip().is_unspecified() { + let mut addr_set = self.pending_courtesy_addr.lock(); + addr_set.insert(peer.addr); + tracing::info!( + %peer, + addr = %peer.addr, + pending = addr_set.len(), + "remember_courtesy_intent: tracking courtesy addr" + ); + } + } + + fn take_pending_courtesy(&self, peer: &PeerId) -> bool { + if !self.is_gateway { + return false; + } + let mut pending = self.pending_courtesy.lock(); + let removed = pending.remove(peer); + if removed { + tracing::debug!( + %peer, + pending = pending.len(), + "take_pending_courtesy: consuming pending courtesy flag" + ); + if !peer.addr.ip().is_unspecified() { + let mut addr_set = self.pending_courtesy_addr.lock(); + if addr_set.remove(&peer.addr) { + tracing::debug!( + %peer, + addr = %peer.addr, + remaining = addr_set.len(), + "take_pending_courtesy: removed pending courtesy addr" + ); + } + } + } + removed + } + + pub(crate) fn take_pending_courtesy_by_addr(&self, addr: &SocketAddr) -> bool { + if !self.is_gateway { + return false; + } + let mut addr_set = self.pending_courtesy_addr.lock(); + if addr_set.remove(addr) { + tracing::info!( + addr = %addr, + remaining = addr_set.len(), + "take_pending_courtesy_by_addr: consuming pending courtesy flag" + ); + true + } else { + false + } + } + + fn register_courtesy_connection(&self, peer: &PeerId) -> Option { + if !self.is_gateway || self.max_courtesy_links == 0 { + return None; + } + let mut links = self.courtesy_links.lock(); + if links.len() == self.max_courtesy_links && links.iter().all(|entry| entry.peer != *peer) { + tracing::debug!( + %peer, + max = self.max_courtesy_links, + "register_courtesy_connection: budget full before inserting" + ); + } + links.retain(|entry| entry.peer != *peer); + links.push_back(CourtesyLink { peer: peer.clone() }); + tracing::info!( + %peer, + len = links.len(), + max = self.max_courtesy_links, + "register_courtesy_connection: tracked courtesy link" + ); + if links.len() > self.max_courtesy_links { + let evicted = links.pop_front().map(|entry| entry.peer); + if let Some(ref victim) = evicted { + tracing::info!( + %victim, + %peer, + "register_courtesy_connection: evicting oldest courtesy link to stay under budget" + ); + } + evicted + } else { + None + } + } + + fn unregister_courtesy_connection(&self, peer: &PeerId) { + if !self.is_gateway { + return; + } + let mut links = self.courtesy_links.lock(); + if links.is_empty() { + return; + } + let before = links.len(); + links.retain(|entry| entry.peer != *peer); + if links.len() != before { + tracing::debug!( + %peer, + remaining = links.len(), + "unregister_courtesy_connection: removed courtesy tracking entry" + ); } } @@ -119,14 +375,28 @@ impl ConnectionManager { /// /// # Panic /// Will panic if the node checking for this condition has no location assigned. - pub fn should_accept(&self, location: Location, peer_id: &PeerId) -> bool { - tracing::info!("Checking if should accept connection"); + pub fn should_accept(&self, location: Location, peer_id: &PeerId, courtesy: bool) -> bool { + let courtesy_join = courtesy && self.is_gateway; + tracing::info!( + courtesy = courtesy_join, + "Checking if should accept connection" + ); let open = self .open_connections .load(std::sync::atomic::Ordering::SeqCst); - let reserved_before = self - .reserved_connections - .load(std::sync::atomic::Ordering::SeqCst); + + if self.location_for_peer.read().contains_key(peer_id) { + if courtesy_join { + self.remember_courtesy_intent(peer_id); + } + tracing::debug!(%peer_id, "Peer already pending/connected; acknowledging acceptance"); + return true; + } + + let reserved_before = match self.reserve_connection_slot(peer_id) { + Some(val) => val, + None => return false, + }; tracing::info!( %peer_id, @@ -148,35 +418,6 @@ impl ConnectionManager { ); } - let reserved_before = loop { - let current = self - .reserved_connections - .load(std::sync::atomic::Ordering::SeqCst); - if current == usize::MAX { - tracing::error!( - %peer_id, - "reserved connection counter overflowed; rejecting new connection" - ); - return false; - } - match self.reserved_connections.compare_exchange( - current, - current + 1, - std::sync::atomic::Ordering::SeqCst, - std::sync::atomic::Ordering::SeqCst, - ) { - Ok(_) => break current, - Err(actual) => { - tracing::debug!( - %peer_id, - expected = current, - actual, - "reserved connection counter changed concurrently; retrying" - ); - } - } - }; - let total_conn = match reserved_before .checked_add(1) .and_then(|val| val.checked_add(open)) @@ -189,19 +430,23 @@ impl ConnectionManager { open, "connection counters would overflow; rejecting connection" ); - self.reserved_connections - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + self.release_reserved_slot(Some(peer_id), "should_accept_overflow_guard"); return false; } }; if open == 0 { + if courtesy_join { + self.remember_courtesy_intent(peer_id); + } + self.record_pending_location(peer_id, location); + self.register_pending_connection(peer_id, true); tracing::debug!(%peer_id, "should_accept: first connection -> accepting"); return true; } - const GATEWAY_DIRECT_ACCEPT_LIMIT: usize = 2; - if self.is_gateway { + const GATEWAY_DIRECT_ACCEPT_LIMIT: usize = 10; + if self.is_gateway && !courtesy_join { let direct_total = open + reserved_before; if direct_total >= GATEWAY_DIRECT_ACCEPT_LIMIT { tracing::info!( @@ -211,19 +456,35 @@ impl ConnectionManager { limit = GATEWAY_DIRECT_ACCEPT_LIMIT, "Gateway reached direct-accept limit; forwarding join request instead" ); - self.reserved_connections - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + self.release_reserved_slot(Some(peer_id), "gateway_direct_accept_limit"); tracing::info!(%peer_id, "should_accept: gateway direct-accept limit hit, forwarding instead"); return false; } } if self.location_for_peer.read().get(peer_id).is_some() { + if courtesy_join { + self.remember_courtesy_intent(peer_id); + } // We've already accepted this peer (pending or active); treat as a no-op acceptance. tracing::debug!(%peer_id, "Peer already pending/connected; acknowledging acceptance"); return true; } + if courtesy_join { + tracing::info!(%peer_id, "should_accept: marking courtesy intent"); + self.remember_courtesy_intent(peer_id); + tracing::debug!( + %peer_id, + open, + reserved = reserved_before, + "should_accept: accepting courtesy connection despite topology limits" + ); + self.record_pending_location(peer_id, location); + self.register_pending_connection(peer_id, true); + return true; + } + let accepted = if total_conn < self.min_connections { tracing::info!(%peer_id, total_conn, "should_accept: accepted (below min connections)"); true @@ -256,11 +517,11 @@ impl ConnectionManager { "should_accept: final decision" ); if !accepted { - self.reserved_connections - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + self.release_reserved_slot(Some(peer_id), "should_accept_rejected"); } else { tracing::info!(%peer_id, total_conn, "should_accept: accepted (reserving spot)"); self.record_pending_location(peer_id, location); + self.register_pending_connection(peer_id, true); } accepted } @@ -350,21 +611,33 @@ impl ConnectionManager { self.prune_connection(peer, false) } - pub fn add_connection(&self, loc: Location, peer: PeerId, was_reserved: bool) { - tracing::info!(%peer, %loc, %was_reserved, "Adding connection to topology"); + pub fn add_connection( + &self, + loc: Location, + peer: PeerId, + was_reserved: bool, + courtesy: bool, + ) -> Option { + tracing::info!( + %peer, + %loc, + %was_reserved, + courtesy, + "Adding connection to topology" + ); debug_assert!(self.get_peer_key().expect("should be set") != peer); - if was_reserved { - let old = self - .reserved_connections - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); - #[cfg(debug_assertions)] - { - tracing::debug!(old, "Decremented reserved connections"); - if old == 0 { - panic!("Underflow of reserved connections"); - } - } - let _ = old; + let pending_meta = self.take_pending_connection(&peer); + let reserved_slot = pending_meta + .as_ref() + .map(|meta| meta.reserved) + .unwrap_or(was_reserved); + if reserved_slot { + self.release_reserved_slot(Some(&peer), "add_connection"); + } else if was_reserved { + tracing::warn!( + %peer, + "add_connection: expected reserved slot but pending entry missing" + ); } let mut lop = self.location_for_peer.write(); lop.insert(peer.clone(), loc); @@ -381,6 +654,32 @@ impl ConnectionManager { self.open_connections .fetch_add(1, std::sync::atomic::Ordering::SeqCst); std::mem::drop(lop); + + let courtesy = if courtesy { + // Clear any pending markers so they don't leak. + let _ = self.take_pending_courtesy(&peer); + true + } else { + self.take_pending_courtesy(&peer) + }; + + if courtesy { + self.register_courtesy_connection(&peer) + } else { + self.unregister_courtesy_connection(&peer); + None + } + } + + pub fn register_outbound_pending(&self, peer: &PeerId, location: Option) { + if let Some(loc) = location { + self.record_pending_location(peer, loc); + } + self.register_pending_connection(peer, false); + } + + pub fn pending_location_hint(&self, peer: &PeerId) -> Option { + self.location_for_peer.read().get(peer).copied() } pub fn update_peer_identity(&self, old_peer: &PeerId, new_peer: PeerId) -> bool { @@ -432,14 +731,24 @@ impl ConnectionManager { tracing::debug!(%peer, "Pruning {} connection", connection_type); let mut locations_for_peer = self.location_for_peer.write(); + let pending_meta = if is_alive { + None + } else { + self.take_pending_connection(peer) + }; let Some(loc) = locations_for_peer.remove(peer) else { if is_alive { tracing::debug!("no location found for peer, skip pruning"); return None; } else { - self.reserved_connections - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + if let Some(meta) = pending_meta { + if meta.reserved { + self.release_reserved_slot(Some(peer), "prune_missing_in_transit"); + } + } else { + tracing::warn!(%peer, "prune_missing_in_transit: no pending entry found while releasing"); + } } return None; }; @@ -452,11 +761,15 @@ impl ConnectionManager { } if is_alive { + self.unregister_courtesy_connection(peer); self.open_connections .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } else if let Some(meta) = pending_meta { + if meta.reserved { + self.release_reserved_slot(Some(peer), "prune_in_transit"); + } } else { - self.reserved_connections - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + tracing::warn!(%peer, "prune_in_transit: missing pending entry while releasing"); } Some(loc) diff --git a/crates/core/src/ring/mod.rs b/crates/core/src/ring/mod.rs index 16ce71be8..0d5e0de9b 100644 --- a/crates/core/src/ring/mod.rs +++ b/crates/core/src/ring/mod.rs @@ -228,14 +228,28 @@ impl Ring { .record_request(recipient, target, request_type); } - pub async fn add_connection(&self, loc: Location, peer: PeerId, was_reserved: bool) { - tracing::info!(%peer, this = ?self.connection_manager.get_peer_key(), %was_reserved, "Adding connection to peer"); - self.connection_manager - .add_connection(loc, peer.clone(), was_reserved); + pub async fn add_connection( + &self, + loc: Location, + peer: PeerId, + was_reserved: bool, + courtesy: bool, + ) -> Option { + tracing::info!( + %peer, + this = ?self.connection_manager.get_peer_key(), + %was_reserved, + courtesy, + "Adding connection to peer" + ); + let eviction_candidate = + self.connection_manager + .add_connection(loc, peer.clone(), was_reserved, courtesy); self.event_register .register_events(Either::Left(NetEventLog::connected(self, peer, loc))) .await; - self.refresh_density_request_cache() + self.refresh_density_request_cache(); + eviction_candidate } pub fn update_connection_identity(&self, old_peer: &PeerId, new_peer: PeerId) { diff --git a/crates/core/src/router/mod.rs b/crates/core/src/router/mod.rs index f5749154b..1687cc8a4 100644 --- a/crates/core/src/router/mod.rs +++ b/crates/core/src/router/mod.rs @@ -20,6 +20,16 @@ pub(crate) struct Router { } impl Router { + /// Some code paths (bootstrap, tests) hand Router peer entries before the + /// remote has published a location. Treat them as midway around the ring so + /// we still consider them instead of dropping the candidate set entirely. + #[inline] + fn peer_distance_or_default(peer: &PeerKeyLocation, target_location: &Location) -> Distance { + peer.location + .map(|loc| target_location.distance(loc)) + .unwrap_or_else(|| Distance::new(0.5)) + } + pub fn new(history: &[RouteEvent]) -> Self { let failure_outcomes: Vec = history .iter() @@ -163,10 +173,7 @@ impl Router { let mut peer_distances: Vec<_> = peers .into_iter() .map(|peer| { - let distance = peer - .location - .map(|loc| target_location.distance(loc)) - .unwrap_or_else(|| Distance::new(0.5)); + let distance = Self::peer_distance_or_default(peer, target_location); (peer, distance) }) .collect(); @@ -205,11 +212,9 @@ impl Router { let mut peer_distances: Vec<_> = peers .into_iter() - .filter_map(|peer| { - peer.location.map(|loc| { - let distance = target_location.distance(loc); - (peer, distance) - }) + .map(|peer| { + let distance = Self::peer_distance_or_default(peer, &target_location); + (peer, distance) }) .collect();