From 63f74ab12f31533af54b9d48c50594ecd613850a Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Wed, 19 Oct 2022 14:07:48 +0530 Subject: [PATCH 01/25] GH-631: get the skeleton ready --- node/src/neighborhood/gossip.rs | 23 ++++++++++- node/src/neighborhood/gossip_acceptor.rs | 50 ++++++++++++++++++++++++ node/src/neighborhood/mod.rs | 12 +++++- 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/node/src/neighborhood/gossip.rs b/node/src/neighborhood/gossip.rs index 0b540b37a..e3589be60 100644 --- a/node/src/neighborhood/gossip.rs +++ b/node/src/neighborhood/gossip.rs @@ -13,7 +13,7 @@ use crate::sub_lib::versioned_data::StepError; use pretty_hex::PrettyHex; use serde_cbor::Value; use serde_derive::{Deserialize, Serialize}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::convert::{TryFrom, TryInto}; use std::fmt::Debug; use std::fmt::Error; @@ -180,6 +180,27 @@ impl TryInto> for Gossip_0v1 { } } +// impl TryInto> for Gossip_0v1 { +// type Error = String; +// +// fn try_into(self) -> Result, Self::Error> { +// let mut hashmap: HashMap = HashMap::new(); +// for node_record in self.node_records { +// let node_addr = match &node_record.node_addr_opt { +// Some(node_addr) => node_addr.clone(), +// None => return Err(String::from("node addr not found")), +// }; +// let agr = match AccessibleGossipRecord::try_from(node_record) { +// Ok(agr) => agr, +// Err(e) => return Err(e), +// }; +// hashmap.insert(node_addr, &agr); +// } +// +// Ok(hashmap) +// } +// } + #[derive(Clone)] pub struct DotGossipEndpoint { pub public_key: PublicKey, diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 94ce0a43c..6e151a626 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -924,8 +924,23 @@ impl GossipHandler for StandardGossipHandler { gossip_source: SocketAddr, cpm_recipient: &Recipient, ) -> GossipAcceptanceResult { + // todo!("breaking the flow"); let initial_neighborship_status = StandardGossipHandler::check_full_neighbor(database, gossip_source.ip()); + + let hashmap = agrs + .iter() + .map(|agr| { + return (agr.inner.public_key.clone(), agr); + }) + .collect::>(); + + // let hashmap: HashMap = agrs.into(); + + eprintln!("HashMap: {:?}", hashmap); + + self.compute_patch(hashmap, 3, database); + let mut db_changed = self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); db_changed = self.identify_and_update_obsolete_nodes(database, agrs) || db_changed; @@ -962,6 +977,41 @@ impl StandardGossipHandler { StandardGossipHandler { logger } } + fn compute_patch( + &self, + agrs: HashMap, + minimum_hop_count: usize, + database: &NeighborhoodDatabase, + ) -> HashSet { + let mut patch: HashSet = HashSet::new(); + fn recurse( + patch: &mut HashSet, + hops_remaining: usize, + agr: &AccessibleGossipRecord, + ) { + if (hops_remaining > 0) && !patch.contains(agr.public_key()) { + patch.insert(agr.public_key()); + let neighbors = agr + .inner + .neighbors + .flat_map(|pk| agrs.get(pk)) + .collect::>(); + neighbors.for_each(|neighbor| recurse(patch, hops_remaining - 1, neighbor)); + } + }; + let node_record = database.root_node(); + patch.insert(node_record.public_key()); + let neighbors = database.get_full_neighbors(node_record.public_key()); + let neighbor_agrs = neighbors + .into_iter() + .map(|neighbor| agrs.get(neighbor.public_key())) + .collect::>(); + neighbor_agrs + .into_iter() + .for_each(|agr| recurse(&mut patch, minimum_hop_count - 1, agr)); + patch + } + fn identify_and_add_non_introductory_new_nodes( &self, database: &mut NeighborhoodDatabase, diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index ecd6e98a5..087eb67e9 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -8,7 +8,7 @@ pub mod neighborhood_database; pub mod node_record; pub mod overall_connection_status; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; @@ -414,6 +414,16 @@ impl TryFrom for AccessibleGossipRecord { } } +// impl<'agr> Into> for AccessibleGossipRecord { +// fn into(self) -> HashMap { +// self.iter() +// .map(|agr| { +// return (agr.inner.public_key.clone(), agr); +// }) +// .collect::>() +// } +// } + #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum RouteDirection { Over, From 4d38ea0d7465f8f3abca2c2d8e017c3f43bd3b98 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 1 Nov 2022 16:18:00 +0530 Subject: [PATCH 02/25] GH-631: add a working recursive method for computing patch --- node/src/neighborhood/gossip_acceptor.rs | 51 ++++++++++-------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 6e151a626..ad6f52e23 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -14,6 +14,7 @@ use masq_lib::logger::Logger; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr}; +use std::ops::Deref; use std::time::{Duration, SystemTime}; /// Note: if you decide to change this, make sure you test thoroughly. Values less than 5 may lead @@ -939,7 +940,9 @@ impl GossipHandler for StandardGossipHandler { eprintln!("HashMap: {:?}", hashmap); - self.compute_patch(hashmap, 3, database); + let root_node = database.root(); + let mut patch: HashSet = HashSet::new(); + self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3, database); let mut db_changed = self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); @@ -979,37 +982,23 @@ impl StandardGossipHandler { fn compute_patch( &self, - agrs: HashMap, - minimum_hop_count: usize, + patch: &mut HashSet, + node: &PublicKey, + agrs: &HashMap, + hops_remaining: usize, database: &NeighborhoodDatabase, - ) -> HashSet { - let mut patch: HashSet = HashSet::new(); - fn recurse( - patch: &mut HashSet, - hops_remaining: usize, - agr: &AccessibleGossipRecord, - ) { - if (hops_remaining > 0) && !patch.contains(agr.public_key()) { - patch.insert(agr.public_key()); - let neighbors = agr - .inner - .neighbors - .flat_map(|pk| agrs.get(pk)) - .collect::>(); - neighbors.for_each(|neighbor| recurse(patch, hops_remaining - 1, neighbor)); - } - }; - let node_record = database.root_node(); - patch.insert(node_record.public_key()); - let neighbors = database.get_full_neighbors(node_record.public_key()); - let neighbor_agrs = neighbors - .into_iter() - .map(|neighbor| agrs.get(neighbor.public_key())) - .collect::>(); - neighbor_agrs - .into_iter() - .for_each(|agr| recurse(&mut patch, minimum_hop_count - 1, agr)); - patch + ) { + if (hops_remaining > 0) && !patch.contains(node) { + patch.insert(node.clone()); + let agr = agrs + .get(node) + .expect(&format!("No agr found for public key {:?}", node)) + .deref() + .clone(); + agr.inner.neighbors.iter().for_each(|public_key| { + self.compute_patch(patch, public_key, agrs, hops_remaining - 1, database) + }); + } } fn identify_and_add_non_introductory_new_nodes( From 5d95637a4bb295fbe8ce20f309e3eb44edb2953d Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Wed, 2 Nov 2022 15:59:46 +0530 Subject: [PATCH 03/25] GH-631: add test for computing patch; not working yet --- node/src/neighborhood/gossip_acceptor.rs | 79 +++++++++++++++++++++--- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index ad6f52e23..05d01f6a6 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -942,7 +942,7 @@ impl GossipHandler for StandardGossipHandler { let root_node = database.root(); let mut patch: HashSet = HashSet::new(); - self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3, database); + self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3); let mut db_changed = self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); @@ -986,17 +986,23 @@ impl StandardGossipHandler { node: &PublicKey, agrs: &HashMap, hops_remaining: usize, - database: &NeighborhoodDatabase, + // database: &NeighborhoodDatabase, ) { if (hops_remaining > 0) && !patch.contains(node) { patch.insert(node.clone()); - let agr = agrs - .get(node) - .expect(&format!("No agr found for public key {:?}", node)) - .deref() - .clone(); + let agr = match agrs.get(node) { + Some(agr) => { + eprintln!( + "AGR found for Node with public key {:?}", + agr.inner.public_key + ); + agr.deref().clone() + } + None => panic!("No AGR found for public key {:?}", node), + }; + agr.inner.neighbors.iter().for_each(|public_key| { - self.compute_patch(patch, public_key, agrs, hops_remaining - 1, database) + self.compute_patch(patch, public_key, agrs, hops_remaining - 1) }); } } @@ -2262,6 +2268,63 @@ mod tests { assert_eq!(recording.len(), 0); } + #[test] + fn standard_gossip_handler_can_compute_patch() { + /* + For 3 hops, a patch originating from Node A will consist of only Node A, B, C and D + A---B---C---D---E + */ + + let subject = StandardGossipHandler::new(Logger::new("test")); + let mut patch: HashSet = HashSet::new(); + let node_a = make_node_record(1111, true); + let node_b = make_node_record(2222, true); + let node_c = make_node_record(3333, false); + let node_d = make_node_record(4444, false); + let node_e = make_node_record(5555, false); + let mut node_a_db = db_from_node(&node_a); + + let nodes = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + node_e.public_key().clone(), + ]; + eprintln!("Nodes: {:?}", nodes); + + node_a_db.add_node(node_b.clone()).unwrap(); + node_a_db.add_node(node_c.clone()).unwrap(); + node_a_db.add_node(node_d.clone()).unwrap(); + node_a_db.add_node(node_e.clone()).unwrap(); + node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_b.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_b.public_key(), node_c.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); + let gossip = GossipBuilder::new(&node_a_db) + .node(node_a.public_key(), true) + .build(); + let agrs: Vec = gossip.try_into().unwrap(); + let hashmap = agrs + .iter() + .map(|agr| { + return (agr.inner.public_key.clone(), agr); + }) + .collect::>(); + + subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3); + + let expected_hashset = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + ] + .into_iter() + .collect::>(); + assert_eq!(patch, expected_hashset); + } + #[test] fn no_cpm_is_sent_in_case_full_neighborship_doesn_t_exist_and_cannot_be_created() { // Received gossip from a node we couldn't make a neighbor {Degree too high or malefactor banned node} (false, false) From 507f0af7e35294b68d062306b89e191ee2b260d8 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Thu, 3 Nov 2022 12:31:22 +0530 Subject: [PATCH 04/25] GH-631: improve the test for the compute_patch function and make it working --- node/src/neighborhood/gossip_acceptor.rs | 44 ++++++++++++++++-------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 05d01f6a6..0e0746d97 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -988,22 +988,25 @@ impl StandardGossipHandler { hops_remaining: usize, // database: &NeighborhoodDatabase, ) { - if (hops_remaining > 0) && !patch.contains(node) { + if (hops_remaining >= 0) && !patch.contains(node) { patch.insert(node.clone()); - let agr = match agrs.get(node) { - Some(agr) => { - eprintln!( - "AGR found for Node with public key {:?}", - agr.inner.public_key - ); - agr.deref().clone() - } - None => panic!("No AGR found for public key {:?}", node), - }; + if hops_remaining > 0 { + let agr = match agrs.get(node) { + Some(agr) => { + eprintln!( + "AGR found for Node with public key {:?}", + agr.inner.public_key + ); + agr.deref().clone() + } + None => panic!("No AGR found for public key {:?}", node), + }; - agr.inner.neighbors.iter().for_each(|public_key| { - self.compute_patch(patch, public_key, agrs, hops_remaining - 1) - }); + agr.inner.neighbors.iter().for_each(|public_key| { + eprintln!("Making a recursive call to neighbor: {:?}", public_key); + self.compute_patch(patch, public_key, agrs, hops_remaining - 1) + }); + } } } @@ -2302,9 +2305,20 @@ mod tests { node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); let gossip = GossipBuilder::new(&node_a_db) - .node(node_a.public_key(), true) + .node(node_a.public_key(), false) + .node(node_b.public_key(), false) + .node(node_c.public_key(), false) + .node(node_d.public_key(), false) + .node(node_e.public_key(), false) .build(); let agrs: Vec = gossip.try_into().unwrap(); + for agr in &agrs { + eprintln!( + "AGR found with public key: {:?}, which has neighbors: {:?}", + agr.inner.public_key, agr.inner.neighbors + ) + } + // eprintln!("AGRS: {:?}", agrs); let hashmap = agrs .iter() .map(|agr| { From 36c466a518e5b06c978de34cf3bec6e7f6457781 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Thu, 3 Nov 2022 13:33:08 +0530 Subject: [PATCH 05/25] GH-631: refactor the recursive function to compute patch --- node/src/neighborhood/gossip_acceptor.rs | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 0e0746d97..c62af3bdb 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -988,26 +988,26 @@ impl StandardGossipHandler { hops_remaining: usize, // database: &NeighborhoodDatabase, ) { - if (hops_remaining >= 0) && !patch.contains(node) { - patch.insert(node.clone()); - if hops_remaining > 0 { - let agr = match agrs.get(node) { - Some(agr) => { - eprintln!( - "AGR found for Node with public key {:?}", - agr.inner.public_key - ); - agr.deref().clone() - } - None => panic!("No AGR found for public key {:?}", node), - }; + patch.insert(node.clone()); + let agr = match agrs.get(node) { + Some(agr) => { + eprintln!( + "AGR found for Node with public key {:?}", + agr.inner.public_key + ); + agr.deref().clone() + } + None => panic!( + "AGR records are insufficient. No AGR found for public key {:?}", + node + ), + }; - agr.inner.neighbors.iter().for_each(|public_key| { - eprintln!("Making a recursive call to neighbor: {:?}", public_key); - self.compute_patch(patch, public_key, agrs, hops_remaining - 1) - }); + agr.inner.neighbors.iter().for_each(|neighbor| { + if hops_remaining > 0 && !patch.contains(neighbor) { + self.compute_patch(patch, neighbor, agrs, hops_remaining - 1) } - } + }); } fn identify_and_add_non_introductory_new_nodes( From 280b3b25309e1b629071980f3e6b59b380356d4f Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Thu, 3 Nov 2022 13:51:43 +0530 Subject: [PATCH 06/25] GH-631: refactor the conditionals inside the recursion --- node/src/neighborhood/gossip_acceptor.rs | 32 +++++++++++------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index c62af3bdb..0371f4dee 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -989,25 +989,21 @@ impl StandardGossipHandler { // database: &NeighborhoodDatabase, ) { patch.insert(node.clone()); - let agr = match agrs.get(node) { - Some(agr) => { - eprintln!( - "AGR found for Node with public key {:?}", - agr.inner.public_key - ); - agr.deref().clone() - } - None => panic!( - "AGR records are insufficient. No AGR found for public key {:?}", - node - ), - }; - - agr.inner.neighbors.iter().for_each(|neighbor| { - if hops_remaining > 0 && !patch.contains(neighbor) { - self.compute_patch(patch, neighbor, agrs, hops_remaining - 1) + if hops_remaining > 0 { + if let Some(agr) = agrs.get(node) { + let agr = agr.deref().clone(); + agr.inner.neighbors.iter().for_each(|neighbor| { + if !patch.contains(neighbor) { + self.compute_patch(patch, neighbor, agrs, hops_remaining - 1) + } + }); + } else { + panic!( + "AGR records are insufficient. No AGR found for public key {:?}", + node + ) } - }); + } } fn identify_and_add_non_introductory_new_nodes( From 8f8b28fa903ed46a317da95b24548cf4098d3fe9 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Thu, 3 Nov 2022 14:02:46 +0530 Subject: [PATCH 07/25] GH-631: refactor the compute_patch() to use match --- node/src/neighborhood/gossip_acceptor.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 0371f4dee..a21e1bc12 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -990,18 +990,20 @@ impl StandardGossipHandler { ) { patch.insert(node.clone()); if hops_remaining > 0 { - if let Some(agr) = agrs.get(node) { - let agr = agr.deref().clone(); - agr.inner.neighbors.iter().for_each(|neighbor| { - if !patch.contains(neighbor) { - self.compute_patch(patch, neighbor, agrs, hops_remaining - 1) + match agrs.get(node) { + Some(agr) => { + let neighbors = agr.deref().clone().inner.neighbors; + for neighbor in &neighbors { + if !patch.contains(neighbor) { + self.compute_patch(patch, neighbor, agrs, hops_remaining - 1) + } } - }); - } else { - panic!( + } + None => warning!( + self.logger, "AGR records are insufficient. No AGR found for public key {:?}", node - ) + ), } } } From 8277905193d49bdf83108858906fc06295f87ff3 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Thu, 3 Nov 2022 15:48:24 +0530 Subject: [PATCH 08/25] GH-631: compute_patch() should search for neighbors in it's database for the root node --- node/src/neighborhood/gossip_acceptor.rs | 45 ++++++++++++------- .../src/neighborhood/neighborhood_database.rs | 44 ++++++++++++++++++ 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index a21e1bc12..a9e50982a 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -942,7 +942,7 @@ impl GossipHandler for StandardGossipHandler { let root_node = database.root(); let mut patch: HashSet = HashSet::new(); - self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3); + self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3, &database); let mut db_changed = self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); @@ -986,24 +986,39 @@ impl StandardGossipHandler { node: &PublicKey, agrs: &HashMap, hops_remaining: usize, - // database: &NeighborhoodDatabase, + database: &NeighborhoodDatabase, ) { patch.insert(node.clone()); if hops_remaining > 0 { - match agrs.get(node) { - Some(agr) => { - let neighbors = agr.deref().clone().inner.neighbors; - for neighbor in &neighbors { - if !patch.contains(neighbor) { - self.compute_patch(patch, neighbor, agrs, hops_remaining - 1) + if patch.len() == 1 { + let neighbors = database.get_all_neighbors(); + for neighbor in neighbors { + if !patch.contains(neighbor) { + self.compute_patch(patch, neighbor, agrs, hops_remaining - 1, database) + } + } + } else { + match agrs.get(node) { + Some(agr) => { + let neighbors = agr.deref().clone().inner.neighbors; + for neighbor in &neighbors { + if !patch.contains(neighbor) { + self.compute_patch( + patch, + neighbor, + agrs, + hops_remaining - 1, + database, + ) + } } } + None => warning!( + self.logger, + "AGR records are insufficient. No AGR found for public key {:?}", + node + ), } - None => warning!( - self.logger, - "AGR records are insufficient. No AGR found for public key {:?}", - node - ), } } } @@ -2303,7 +2318,7 @@ mod tests { node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); let gossip = GossipBuilder::new(&node_a_db) - .node(node_a.public_key(), false) + // .node(node_a.public_key(), false) .node(node_b.public_key(), false) .node(node_c.public_key(), false) .node(node_d.public_key(), false) @@ -2324,7 +2339,7 @@ mod tests { }) .collect::>(); - subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3); + subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3, &node_a_db); let expected_hashset = vec![ node_a.public_key().clone(), diff --git a/node/src/neighborhood/neighborhood_database.rs b/node/src/neighborhood/neighborhood_database.rs index 79a4d1d1f..640173533 100644 --- a/node/src/neighborhood/neighborhood_database.rs +++ b/node/src/neighborhood/neighborhood_database.rs @@ -19,6 +19,7 @@ use std::fmt::Debug; use std::fmt::Error; use std::fmt::Formatter; use std::net::IpAddr; +use std::ops::Deref; pub const ISOLATED_NODE_GRACE_PERIOD_SECS: u32 = 30; @@ -108,6 +109,13 @@ impl NeighborhoodDatabase { self.has_half_neighbor(from, to) && self.has_half_neighbor(to, from) } + pub fn get_all_neighbors(&self) -> HashSet<&PublicKey> { + self.by_public_key + .keys() + .filter(|public_key| self.has_full_neighbor(&self.this_node, public_key)) + .collect::>() + } + pub fn gossip_target_degree(&self, target: &PublicKey) -> usize { let target_node = match self.node_by_key(target) { None => return 0, @@ -681,6 +689,42 @@ mod tests { assert_eq!(Ok(false), result, "add_arbitrary_neighbor done goofed"); } + #[test] + fn can_retrieve_all_full_neighbors() { + let root_node = make_node_record(1111, true); + let full_neighbor_1 = make_node_record(2222, true); + let full_neighbor_2 = make_node_record(3333, true); + let full_neighbor_3 = make_node_record(4444, true); + let half_neighbor_1 = make_node_record(5555, false); + let half_neighbor_2 = make_node_record(6666, false); + let some_node = make_node_record(7777, false); + let mut subject = db_from_node(&root_node); + subject.add_node(full_neighbor_1.clone()); + subject.add_node(full_neighbor_2.clone()); + subject.add_node(full_neighbor_3.clone()); + subject.add_node(half_neighbor_1.clone()); + subject.add_node(half_neighbor_2.clone()); + subject.add_node(some_node.clone()); + subject.add_arbitrary_full_neighbor(root_node.public_key(), full_neighbor_1.public_key()); + subject.add_arbitrary_full_neighbor(root_node.public_key(), full_neighbor_2.public_key()); + subject.add_arbitrary_full_neighbor(root_node.public_key(), full_neighbor_3.public_key()); + subject.add_arbitrary_half_neighbor(root_node.public_key(), half_neighbor_1.public_key()); + subject.add_arbitrary_half_neighbor(root_node.public_key(), half_neighbor_2.public_key()); + + let result = subject.get_all_neighbors(); + + assert_eq!( + result, + vec![ + full_neighbor_1.public_key().clone(), + full_neighbor_2.public_key().clone(), + full_neighbor_3.public_key().clone() + ] + .iter() + .collect::>() + ); + } + #[test] fn gossip_target_degree() { let root = make_node_record(1000, true); From 3f8fd66c471b7e5d03de66af42d48443818a86b6 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Thu, 3 Nov 2022 18:36:41 +0530 Subject: [PATCH 09/25] GH-631: improve the solution for the compute_patch() --- node/src/neighborhood/gossip_acceptor.rs | 42 +++++++++--------------- node/src/neighborhood/mod.rs | 4 +++ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index a9e50982a..868cad4db 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -990,34 +990,22 @@ impl StandardGossipHandler { ) { patch.insert(node.clone()); if hops_remaining > 0 { - if patch.len() == 1 { - let neighbors = database.get_all_neighbors(); - for neighbor in neighbors { - if !patch.contains(neighbor) { - self.compute_patch(patch, neighbor, agrs, hops_remaining - 1, database) - } - } + let neighbors = if patch.len() == 1 { + database.get_all_neighbors() } else { - match agrs.get(node) { - Some(agr) => { - let neighbors = agr.deref().clone().inner.neighbors; - for neighbor in &neighbors { - if !patch.contains(neighbor) { - self.compute_patch( - patch, - neighbor, - agrs, - hops_remaining - 1, - database, - ) - } - } - } - None => warning!( - self.logger, - "AGR records are insufficient. No AGR found for public key {:?}", - node - ), + agrs.get(node) + .unwrap_or_else(|| { + panic!( + "AGR records are insufficient. No AGR found for public key {:?}", + node + ) + }) + .get_all_neighbors() + }; + + for neighbor in neighbors { + if !patch.contains(neighbor) { + self.compute_patch(patch, neighbor, agrs, hops_remaining - 1, database) } } } diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index 087eb67e9..8efe12cb5 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -396,6 +396,10 @@ impl AccessibleGossipRecord { self.signed_gossip = signed_gossip; self.signature = signature; } + + pub fn get_all_neighbors(&self) -> HashSet<&PublicKey> { + self.inner.neighbors.iter().collect::>() + } } impl TryFrom for AccessibleGossipRecord { From e2f99228e60b65272054412840843e62c163b4b3 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Fri, 4 Nov 2022 12:11:32 +0530 Subject: [PATCH 10/25] GH-631: minor code cleanup --- node/src/neighborhood/gossip_acceptor.rs | 27 ++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 868cad4db..ea6504d16 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -925,7 +925,6 @@ impl GossipHandler for StandardGossipHandler { gossip_source: SocketAddr, cpm_recipient: &Recipient, ) -> GossipAcceptanceResult { - // todo!("breaking the flow"); let initial_neighborship_status = StandardGossipHandler::check_full_neighbor(database, gossip_source.ip()); @@ -991,6 +990,7 @@ impl StandardGossipHandler { patch.insert(node.clone()); if hops_remaining > 0 { let neighbors = if patch.len() == 1 { + // Root Node database.get_all_neighbors() } else { agrs.get(node) @@ -2287,16 +2287,6 @@ mod tests { let node_d = make_node_record(4444, false); let node_e = make_node_record(5555, false); let mut node_a_db = db_from_node(&node_a); - - let nodes = vec![ - node_a.public_key().clone(), - node_b.public_key().clone(), - node_c.public_key().clone(), - node_d.public_key().clone(), - node_e.public_key().clone(), - ]; - eprintln!("Nodes: {:?}", nodes); - node_a_db.add_node(node_b.clone()).unwrap(); node_a_db.add_node(node_c.clone()).unwrap(); node_a_db.add_node(node_d.clone()).unwrap(); @@ -2306,20 +2296,31 @@ mod tests { node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); let gossip = GossipBuilder::new(&node_a_db) - // .node(node_a.public_key(), false) .node(node_b.public_key(), false) .node(node_c.public_key(), false) .node(node_d.public_key(), false) .node(node_e.public_key(), false) .build(); let agrs: Vec = gossip.try_into().unwrap(); + + //----------------------- Debugging ----------------------------- + let nodes = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + node_e.public_key().clone(), + ]; + eprintln!("Nodes: {:?}", nodes); + for agr in &agrs { eprintln!( "AGR found with public key: {:?}, which has neighbors: {:?}", agr.inner.public_key, agr.inner.neighbors ) } - // eprintln!("AGRS: {:?}", agrs); + //---------------------------------------------------------------- + let hashmap = agrs .iter() .map(|agr| { From 00270a3efc2ed12d029e2e105784978a44d9016c Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Fri, 4 Nov 2022 12:47:41 +0530 Subject: [PATCH 11/25] GH-631: add one more test to make compute_patch() robust --- node/src/neighborhood/gossip_acceptor.rs | 103 +++++++++++++++++++++-- node/src/neighborhood/mod.rs | 4 +- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index ea6504d16..07102d38f 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -993,14 +993,16 @@ impl StandardGossipHandler { // Root Node database.get_all_neighbors() } else { - agrs.get(node) - .unwrap_or_else(|| { - panic!( - "AGR records are insufficient. No AGR found for public key {:?}", - node - ) - }) - .get_all_neighbors() + match agrs.get(node) { + Some(agr) => agr.get_all_neighbors(), + None => { + error!( + self.logger, + "AGR records are insufficient. No AGR found for public key {:?}", node + ); + HashSet::new() + } + } }; for neighbor in neighbors { @@ -1289,6 +1291,7 @@ mod tests { use crate::test_utils::unshared_test_utils::make_cpm_recipient; use crate::test_utils::{assert_contains, main_cryptde, vec_to_set}; use actix::{Actor, System}; + use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; use std::convert::TryInto; use std::ops::{Add, Sub}; @@ -2341,6 +2344,90 @@ mod tests { assert_eq!(patch, expected_hashset); } + #[test] + fn standard_gossip_handler_can_handle_node_for_which_agr_is_not_found_while_computing_patch() { + /* + For 3 hops, a patch originating from Node A will consist of only Node A, X, B, C and D + A---B---C---D---E + | + X + In this test, we won't provide AGR for Node X and the compute_patch() is still able to + add Node X inside the patch (since it's a neighbor) and is also able to log that it + failed to find the AGR for it. + */ + + init_test_logging(); + let subject = StandardGossipHandler::new(Logger::new("test")); + let mut patch: HashSet = HashSet::new(); + let node_a = make_node_record(1111, true); + let node_b = make_node_record(2222, true); + let node_c = make_node_record(3333, false); + let node_d = make_node_record(4444, false); + let node_e = make_node_record(5555, false); + let node_x = make_node_record(6666, false); + let mut node_a_db = db_from_node(&node_a); + node_a_db.add_node(node_b.clone()).unwrap(); + node_a_db.add_node(node_c.clone()).unwrap(); + node_a_db.add_node(node_d.clone()).unwrap(); + node_a_db.add_node(node_e.clone()).unwrap(); + node_a_db.add_node(node_x.clone()).unwrap(); + node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_b.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_x.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_b.public_key(), node_c.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); + let gossip = GossipBuilder::new(&node_a_db) + .node(node_b.public_key(), false) + .node(node_c.public_key(), false) + .node(node_d.public_key(), false) + .node(node_e.public_key(), false) + .build(); + let agrs: Vec = gossip.try_into().unwrap(); + + //----------------------- Debugging ----------------------------- + let nodes = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + node_e.public_key().clone(), + node_x.public_key().clone(), + ]; + eprintln!("Nodes: {:?}", nodes); + + for agr in &agrs { + eprintln!( + "AGR found with public key: {:?}, which has neighbors: {:?}", + agr.inner.public_key, agr.inner.neighbors + ) + } + //---------------------------------------------------------------- + + let hashmap = agrs + .iter() + .map(|agr| { + return (agr.inner.public_key.clone(), agr); + }) + .collect::>(); + + subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3, &node_a_db); + + let expected_hashset = vec![ + node_a.public_key().clone(), + node_x.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + ] + .into_iter() + .collect::>(); + assert_eq!(patch, expected_hashset); + TestLogHandler::new().exists_log_matching(&format!( + "AGR records are insufficient. No AGR found for public key {:?}", + node_x.public_key() + )); + } + #[test] fn no_cpm_is_sent_in_case_full_neighborship_doesn_t_exist_and_cannot_be_created() { // Received gossip from a node we couldn't make a neighbor {Degree too high or malefactor banned node} (false, false) diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index 8efe12cb5..100df919e 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -46,7 +46,6 @@ use crate::sub_lib::cryptde::{CryptDE, CryptData, PlainData}; use crate::sub_lib::dispatcher::{Component, StreamShutdownMsg}; use crate::sub_lib::hopper::{ExpiredCoresPackage, NoLookupIncipientCoresPackage}; use crate::sub_lib::hopper::{IncipientCoresPackage, MessageType}; -use crate::sub_lib::neighborhood::NodeQueryMessage; use crate::sub_lib::neighborhood::NodeQueryResponseMetadata; use crate::sub_lib::neighborhood::NodeRecordMetadataMessage; use crate::sub_lib::neighborhood::RemoveNeighborMessage; @@ -57,6 +56,7 @@ use crate::sub_lib::neighborhood::{ConnectionProgressEvent, ExpectedServices}; use crate::sub_lib::neighborhood::{ConnectionProgressMessage, ExpectedService}; use crate::sub_lib::neighborhood::{DispatcherNodeQueryMessage, GossipFailure_0v1}; use crate::sub_lib::neighborhood::{NeighborhoodSubs, NeighborhoodTools}; +use crate::sub_lib::neighborhood::{NodeQueryMessage, DEFAULT_RATE_PACK}; use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::peer_actors::{BindMessage, NewPublicIp, StartMessage}; use crate::sub_lib::proxy_server::DEFAULT_MINIMUM_HOP_COUNT; @@ -67,6 +67,7 @@ use crate::sub_lib::stream_handler_pool::DispatcherNodeQueryResponse; use crate::sub_lib::utils::{handle_ui_crash_request, NODE_MAILBOX_CAPACITY}; use crate::sub_lib::versioned_data::VersionedData; use crate::sub_lib::wallet::Wallet; +use crate::test_utils::make_wallet; use gossip_acceptor::GossipAcceptor; use gossip_acceptor::GossipAcceptorReal; use gossip_producer::GossipProducer; @@ -74,6 +75,7 @@ use gossip_producer::GossipProducerReal; use masq_lib::blockchains::chains::Chain; use masq_lib::crash_point::CrashPoint; use masq_lib::logger::Logger; +use masq_lib::messages::UiSetupResponseValueStatus::Default; use neighborhood_database::NeighborhoodDatabase; use node_record::NodeRecord; From 83ab0f3913d1213470017750c16fdae57de7c0c7 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 8 Nov 2022 12:28:04 +0530 Subject: [PATCH 12/25] GH-631: add one more test for compute_patch() --- node/src/neighborhood/gossip_acceptor.rs | 82 ++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 07102d38f..f15d80df9 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -2280,6 +2280,8 @@ mod tests { /* For 3 hops, a patch originating from Node A will consist of only Node A, B, C and D A---B---C---D---E + What does this test proves: + The distance of A and E is more than 3 hops, hence E is being excluded. */ let subject = StandardGossipHandler::new(Logger::new("test")); @@ -2344,6 +2346,86 @@ mod tests { assert_eq!(patch, expected_hashset); } + #[test] + fn standard_gossip_handler_computes_neighbors_from_database_and_neighbors_of_neighbors_from_agrs( + ) { + /* + For 3 hops, a patch originating from Node A will consist of only Node A, B, C and D + A---B---C---D + | + Y + What does this test proves: + 1) To find neighbors, we'll look into the root node's database. (For Example, A---B). + 2) To find neighbors of neighbors, we'll look into the AGRs. (For Example, B---Y, B---C, and C---D). + */ + + let subject = StandardGossipHandler::new(Logger::new("test")); + let mut patch: HashSet = HashSet::new(); + let node_a = make_node_record(1111, true); + let node_b = make_node_record(2222, true); + let node_c = make_node_record(3333, false); + let node_d = make_node_record(4444, false); + let node_y = make_node_record(5555, false); + let mut node_a_db = db_from_node(&node_a); + node_a_db.add_node(node_b.clone()).unwrap(); + node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_b.public_key()); + let mut node_b_db = db_from_node(&node_b); + node_b_db.add_node(node_a.clone()).unwrap(); + node_b_db.add_node(node_y.clone()).unwrap(); + node_b_db.add_node(node_c.clone()).unwrap(); + node_b_db.add_node(node_d.clone()).unwrap(); + node_b_db.add_arbitrary_full_neighbor(node_b.public_key(), node_a.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_b.public_key(), node_y.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_b.public_key(), node_c.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); + + let gossip = GossipBuilder::new(&node_b_db) + .node(node_b.public_key(), false) + .node(node_c.public_key(), false) + .node(node_d.public_key(), false) + .node(node_y.public_key(), false) + .build(); + let agrs: Vec = gossip.try_into().unwrap(); + + //----------------------- Debugging ----------------------------- + let nodes = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + node_y.public_key().clone(), + ]; + eprintln!("Nodes: {:?}", nodes); + + for agr in &agrs { + eprintln!( + "AGR found with public key: {:?}, which has neighbors: {:?}", + agr.inner.public_key, agr.inner.neighbors + ) + } + //---------------------------------------------------------------- + + let hashmap = agrs + .iter() + .map(|agr| { + return (agr.inner.public_key.clone(), agr); + }) + .collect::>(); + + subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3, &node_a_db); + + let expected_hashset = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + node_y.public_key().clone(), + ] + .into_iter() + .collect::>(); + assert_eq!(patch, expected_hashset); + } + #[test] fn standard_gossip_handler_can_handle_node_for_which_agr_is_not_found_while_computing_patch() { /* From cad669534505c7f37b51d3023e42e2110adf61af Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 8 Nov 2022 13:24:30 +0530 Subject: [PATCH 13/25] GH-631: reduce the number of recursion calls and minize hashset size --- node/src/neighborhood/gossip_acceptor.rs | 33 +++++++++++------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index f15d80df9..e58e4b0df 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -996,11 +996,13 @@ impl StandardGossipHandler { match agrs.get(node) { Some(agr) => agr.get_all_neighbors(), None => { - error!( + patch.remove(&node); + trace!( self.logger, - "AGR records are insufficient. No AGR found for public key {:?}", node + "While computing patch no AGR record found for public key {:?}", + node ); - HashSet::new() + return; } } }; @@ -2278,7 +2280,7 @@ mod tests { #[test] fn standard_gossip_handler_can_compute_patch() { /* - For 3 hops, a patch originating from Node A will consist of only Node A, B, C and D + Over here, root node is A and patch contains [A, B, C, D]. A---B---C---D---E What does this test proves: The distance of A and E is more than 3 hops, hence E is being excluded. @@ -2350,7 +2352,7 @@ mod tests { fn standard_gossip_handler_computes_neighbors_from_database_and_neighbors_of_neighbors_from_agrs( ) { /* - For 3 hops, a patch originating from Node A will consist of only Node A, B, C and D + Over here, root node is A and patch contains [A, B, Y, C, D]. A---B---C---D | Y @@ -2429,40 +2431,36 @@ mod tests { #[test] fn standard_gossip_handler_can_handle_node_for_which_agr_is_not_found_while_computing_patch() { /* - For 3 hops, a patch originating from Node A will consist of only Node A, X, B, C and D - A---B---C---D---E + Over here, root node is A and patch contains [A, B, C, D]. + A---B---C---D | X - In this test, we won't provide AGR for Node X and the compute_patch() is still able to - add Node X inside the patch (since it's a neighbor) and is also able to log that it - failed to find the AGR for it. + In this test, we won't provide AGR for Node X and thereby compute_patch() will not add + Node X inside the patch. Also, logger will log it as it happens. */ init_test_logging(); - let subject = StandardGossipHandler::new(Logger::new("test")); + let test_name = "standard_gossip_handler_can_handle_node_for_which_agr_is_not_found_while_computing_patch"; + let subject = StandardGossipHandler::new(Logger::new(test_name)); let mut patch: HashSet = HashSet::new(); let node_a = make_node_record(1111, true); let node_b = make_node_record(2222, true); let node_c = make_node_record(3333, false); let node_d = make_node_record(4444, false); - let node_e = make_node_record(5555, false); let node_x = make_node_record(6666, false); let mut node_a_db = db_from_node(&node_a); node_a_db.add_node(node_b.clone()).unwrap(); node_a_db.add_node(node_c.clone()).unwrap(); node_a_db.add_node(node_d.clone()).unwrap(); - node_a_db.add_node(node_e.clone()).unwrap(); node_a_db.add_node(node_x.clone()).unwrap(); node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_b.public_key()); node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_x.public_key()); node_a_db.add_arbitrary_full_neighbor(node_b.public_key(), node_c.public_key()); node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); - node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); let gossip = GossipBuilder::new(&node_a_db) .node(node_b.public_key(), false) .node(node_c.public_key(), false) .node(node_d.public_key(), false) - .node(node_e.public_key(), false) .build(); let agrs: Vec = gossip.try_into().unwrap(); @@ -2472,7 +2470,6 @@ mod tests { node_b.public_key().clone(), node_c.public_key().clone(), node_d.public_key().clone(), - node_e.public_key().clone(), node_x.public_key().clone(), ]; eprintln!("Nodes: {:?}", nodes); @@ -2496,7 +2493,6 @@ mod tests { let expected_hashset = vec![ node_a.public_key().clone(), - node_x.public_key().clone(), node_b.public_key().clone(), node_c.public_key().clone(), node_d.public_key().clone(), @@ -2505,7 +2501,8 @@ mod tests { .collect::>(); assert_eq!(patch, expected_hashset); TestLogHandler::new().exists_log_matching(&format!( - "AGR records are insufficient. No AGR found for public key {:?}", + "TRACE: {}: While computing patch no AGR record found for public key {:?}", + test_name, node_x.public_key() )); } From c4143b97d88aa14ab797d796771065311d31060d Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Wed, 16 Nov 2022 18:11:17 +0530 Subject: [PATCH 14/25] GH-631: add test to check if the standard gossip handler ignores gossips that are outside the patch --- node/src/neighborhood/gossip_acceptor.rs | 90 +++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index e58e4b0df..f143e7239 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -937,12 +937,14 @@ impl GossipHandler for StandardGossipHandler { // let hashmap: HashMap = agrs.into(); - eprintln!("HashMap: {:?}", hashmap); + eprintln!("HashMap: {:?}", hashmap.keys()); let root_node = database.root(); let mut patch: HashSet = HashSet::new(); self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3, &database); + eprintln!("Patch: {:?}", patch); + let mut db_changed = self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); db_changed = self.identify_and_update_obsolete_nodes(database, agrs) || db_changed; @@ -2507,6 +2509,92 @@ mod tests { )); } + #[test] + fn standard_gossip_handler_will_ignore_gossips_from_outside_the_patch() { + /* + Over here, root node is A and patch contains [A, B, C, D]. + A---B---C---D---E---F + |___________||______| + + The node B sends a gossip to A. + A <---- B + Node B tells Node A about the full neighborship of E and F. + Since, the relation between E and F is outside the patch, standard gossip handler + will not perform any database changes and will mark the gossip as ignored. + + */ + + let cryptde = main_cryptde(); + let (cpm_recipient, _) = make_cpm_recipient(); + let subject = StandardGossipHandler::new(Logger::new("test")); + let mut patch: HashSet = HashSet::new(); + let node_a = make_node_record(1111, true); + let node_b = make_node_record(2222, true); + let node_c = make_node_record(3333, false); + let node_d = make_node_record(4444, false); + let node_e = make_node_record(5555, false); + let node_f = make_node_record(6666, false); + let mut node_a_db = db_from_node(&node_a); + node_a_db.add_node(node_b.clone()).unwrap(); + node_a_db.add_node(node_c.clone()).unwrap(); + node_a_db.add_node(node_d.clone()).unwrap(); + node_a_db.add_node(node_e.clone()).unwrap(); + node_a_db.add_arbitrary_full_neighbor(node_a.public_key(), node_b.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_b.public_key(), node_c.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); + node_a_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); + let gossip_source: SocketAddr = node_b.node_addr_opt().unwrap().into(); + let mut node_b_db = db_from_node(&node_b); + node_b_db.add_node(node_a.clone()).unwrap(); + node_b_db.add_node(node_c.clone()).unwrap(); + node_b_db.add_node(node_d.clone()).unwrap(); + node_b_db.add_node(node_e.clone()).unwrap(); + node_b_db.add_node(node_f.clone()).unwrap(); + node_b_db.add_arbitrary_full_neighbor(node_a.public_key(), node_b.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_b.public_key(), node_c.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_c.public_key(), node_d.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_d.public_key(), node_e.public_key()); + node_b_db.add_arbitrary_full_neighbor(node_e.public_key(), node_f.public_key()); + let gossip = GossipBuilder::new(&node_b_db) + .node(node_b.public_key(), true) + .node(node_c.public_key(), false) + .node(node_d.public_key(), false) + .node(node_e.public_key(), false) + .node(node_f.public_key(), false) + .build(); + let agrs: Vec = gossip.try_into().unwrap(); + + //----------------------- Debugging ----------------------------- + let nodes = vec![ + node_a.public_key().clone(), + node_b.public_key().clone(), + node_c.public_key().clone(), + node_d.public_key().clone(), + node_e.public_key().clone(), + node_f.public_key().clone(), + ]; + eprintln!("Nodes: {:?}", nodes); + + for agr in &agrs { + eprintln!( + "AGR found with public key: {:?}, which has neighbors: {:?}", + agr.inner.public_key, agr.inner.neighbors + ) + } + //---------------------------------------------------------------- + + let hashmap = agrs + .iter() + .map(|agr| { + return (agr.inner.public_key.clone(), agr); + }) + .collect::>(); + + let result = subject.handle(cryptde, &mut node_a_db, agrs, gossip_source, &cpm_recipient); + + assert_eq!(result, GossipAcceptanceResult::Ignored); + } + #[test] fn no_cpm_is_sent_in_case_full_neighborship_doesn_t_exist_and_cannot_be_created() { // Received gossip from a node we couldn't make a neighbor {Degree too high or malefactor banned node} (false, false) From 541260c7e4aba6b38b4535bf2cedc37628d68abb Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Wed, 16 Nov 2022 18:18:36 +0530 Subject: [PATCH 15/25] GH-631: make the test work --- node/src/neighborhood/gossip_acceptor.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index f143e7239..274dc04cb 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -945,8 +945,12 @@ impl GossipHandler for StandardGossipHandler { eprintln!("Patch: {:?}", patch); - let mut db_changed = - self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); + let mut db_changed = self.identify_and_add_non_introductory_new_nodes( + database, + &agrs, + &patch, + gossip_source, + ); db_changed = self.identify_and_update_obsolete_nodes(database, agrs) || db_changed; db_changed = self.handle_root_node(cryptde, database, gossip_source) || db_changed; let final_neighborship_status = @@ -1021,6 +1025,7 @@ impl StandardGossipHandler { &self, database: &mut NeighborhoodDatabase, agrs: &[AccessibleGossipRecord], + patch: &HashSet, gossip_source: SocketAddr, ) -> bool { let all_keys = database @@ -1029,6 +1034,7 @@ impl StandardGossipHandler { .cloned() .collect::>(); agrs.iter() + .filter(|agr| patch.contains(&agr.inner.public_key)) .filter(|agr| !all_keys.contains(&agr.inner.public_key)) // TODO: A node that tells us the IP Address of the node that isn't in our database should be malefactor banned .filter(|agr| match &agr.node_addr_opt { From 325f1a7b95f6e398aae57783cd88f832f55e9f3b Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Fri, 18 Nov 2022 19:07:53 +0530 Subject: [PATCH 16/25] GH-631: fix the test proper_standard_gossip_is_matched_and_handled --- node/src/neighborhood/gossip_acceptor.rs | 41 +++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 274dc04cb..88b5b60f6 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -930,9 +930,7 @@ impl GossipHandler for StandardGossipHandler { let hashmap = agrs .iter() - .map(|agr| { - return (agr.inner.public_key.clone(), agr); - }) + .map(|agr| (agr.inner.public_key.clone(), agr)) .collect::>(); // let hashmap: HashMap = agrs.into(); @@ -2233,20 +2231,41 @@ mod tests { #[test] fn proper_standard_gossip_is_matched_and_handled() { + /* + Destination Node ==> + S---D + + Source Node ==> + A---S---D + | + B + + The source node(S) will gossip about Node A and B + to the destination node(D). + */ let src_root = make_node_record(1234, true); let dest_root = make_node_record(2345, true); let mut src_db = db_from_node(&src_root); - let node_a_key = &src_db.add_node(make_node_record(3456, true)).unwrap(); - let node_b_key = &src_db.add_node(make_node_record(4567, true)).unwrap(); + let node_a = make_node_record(3456, true); + let node_b = make_node_record(4567, true); let mut dest_db = db_from_node(&dest_root); dest_db.add_node(src_root.clone()).unwrap(); dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), src_root.public_key()); src_db.add_node(dest_db.root().clone()).unwrap(); + src_db.add_node(node_a.clone()).unwrap(); + src_db.add_node(node_b.clone()).unwrap(); src_db.add_arbitrary_full_neighbor(src_root.public_key(), dest_root.public_key()); + src_db.add_arbitrary_half_neighbor(src_root.public_key(), &node_a.public_key()); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), &node_b.public_key()); + src_db + .node_by_key_mut(src_root.public_key()) + .unwrap() + .increment_version(); + src_db.resign_node(src_root.public_key()); let gossip = GossipBuilder::new(&src_db) .node(src_root.public_key(), true) - .node(node_a_key, false) - .node(node_b_key, false) + .node(node_a.public_key(), false) + .node(node_b.public_key(), false) .build(); let subject = StandardGossipHandler::new(Logger::new("test")); let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); @@ -2272,12 +2291,12 @@ mod tests { ); assert!(dest_db.has_full_neighbor(dest_db.root().public_key(), src_db.root().public_key())); assert_eq!( - &src_db.node_by_key(node_a_key).unwrap().inner, - &dest_db.node_by_key(node_a_key).unwrap().inner + &src_db.node_by_key(node_a.public_key()).unwrap().inner, + &dest_db.node_by_key(node_a.public_key()).unwrap().inner ); assert_eq!( - &src_db.node_by_key(node_b_key).unwrap().inner, - &dest_db.node_by_key(node_b_key).unwrap().inner + &src_db.node_by_key(node_b.public_key()).unwrap().inner, + &dest_db.node_by_key(node_b.public_key()).unwrap().inner ); System::current().stop(); assert_eq!(system.run(), 0); From 92a8eb927755f5969f08041b54c32b60f004a6ad Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 22 Nov 2022 13:12:39 +0530 Subject: [PATCH 17/25] GH-631: fix the tests inside gossip_acceptor.rs --- node/src/neighborhood/gossip_acceptor.rs | 44 +++++++++++++++--------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 88b5b60f6..db63fec9c 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -2649,7 +2649,7 @@ mod tests { assert_eq!(system.run(), 0); let recording = recording_arc.lock().unwrap(); assert_eq!(recording.len(), 0); - assert_eq!(result, GossipAcceptanceResult::Accepted); + assert_eq!(result, GossipAcceptanceResult::Ignored); } #[test] @@ -3646,6 +3646,26 @@ mod tests { #[test] fn initial_standard_gossip_does_not_produce_neighborship_if_destination_degree_is_already_full() { + /* + Destination Database ==> + N2 N3 + \ / + N1---Dest---Third + / \ + N4 N5 + + Source Database ==> + Src---Third---Dest + + Disc + + Expected Destination Database ==> + N2 N3 + \ / Third + N1---Dest---| + / \ Src + N4 N5 + */ let dest_node = make_node_record(1234, true); let dest_node_cryptde = CryptDENull::from(&dest_node.public_key(), TEST_DEFAULT_CHAIN); let mut dest_db = db_from_node(&dest_node); @@ -3661,10 +3681,10 @@ mod tests { dest_db.add_arbitrary_half_neighbor(dest_node.public_key(), failed_node_key); } dest_db.add_node(third_node.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor(dest_node.public_key(), third_node.public_key()); src_db.add_node(dest_node.clone()).unwrap(); src_db.add_node(third_node.clone()).unwrap(); src_db.add_node(disconnected_node.clone()).unwrap(); - dest_db.add_arbitrary_full_neighbor(dest_node.public_key(), third_node.public_key()); src_db.add_arbitrary_full_neighbor(dest_node.public_key(), third_node.public_key()); src_db.add_arbitrary_full_neighbor(src_node.public_key(), third_node.public_key()); src_db @@ -3693,11 +3713,7 @@ mod tests { let after = time_t_timestamp(); let mut expected_dest_db = src_db.clone(); expected_dest_db.add_arbitrary_half_neighbor(dest_node.public_key(), src_node.public_key()); - expected_dest_db - .node_by_key_mut(disconnected_node.public_key()) - .unwrap() - .metadata - .node_addr_opt = None; + expected_dest_db.remove_neighbor(disconnected_node.public_key()); for idx in 0..MAX_DEGREE { let failed_node_key = &expected_dest_db .add_node(make_node_record(4000 + idx as u16, true)) @@ -3732,16 +3748,10 @@ mod tests { before, after, ); - assert_node_records_eq( - dest_db - .node_by_key_mut(disconnected_node.public_key()) - .unwrap(), - expected_dest_db - .node_by_key(disconnected_node.public_key()) - .unwrap(), - before, - after, - ); + // If you're here because you're working on GH-650, and this assert is failing, + // everything's fine: GH-650 is supposed to make it fail. Modify the assert so + // that it ensures that the disconnected Node is preserved. + assert_eq!(dest_db.node_by_key(disconnected_node.public_key()), None); } #[test] From 6a2f92f0f662a20bb51f8c7640225224e04cd1b4 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 22 Nov 2022 14:05:50 +0530 Subject: [PATCH 18/25] GH-631: refactor the compute_patch() --- node/src/neighborhood/gossip_acceptor.rs | 4 ++-- node/src/neighborhood/mod.rs | 2 +- node/src/neighborhood/neighborhood_database.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index db63fec9c..5d7568d1f 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -995,10 +995,10 @@ impl StandardGossipHandler { if hops_remaining > 0 { let neighbors = if patch.len() == 1 { // Root Node - database.get_all_neighbors() + database.get_all_half_neighbors() } else { match agrs.get(node) { - Some(agr) => agr.get_all_neighbors(), + Some(agr) => agr.get_all_inner_neighbors(), None => { patch.remove(&node); trace!( diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index 100df919e..e8bf3475d 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -399,7 +399,7 @@ impl AccessibleGossipRecord { self.signature = signature; } - pub fn get_all_neighbors(&self) -> HashSet<&PublicKey> { + pub fn get_all_inner_neighbors(&self) -> HashSet<&PublicKey> { self.inner.neighbors.iter().collect::>() } } diff --git a/node/src/neighborhood/neighborhood_database.rs b/node/src/neighborhood/neighborhood_database.rs index 640173533..2f982c8e5 100644 --- a/node/src/neighborhood/neighborhood_database.rs +++ b/node/src/neighborhood/neighborhood_database.rs @@ -109,10 +109,10 @@ impl NeighborhoodDatabase { self.has_half_neighbor(from, to) && self.has_half_neighbor(to, from) } - pub fn get_all_neighbors(&self) -> HashSet<&PublicKey> { + pub fn get_all_half_neighbors(&self) -> HashSet<&PublicKey> { self.by_public_key .keys() - .filter(|public_key| self.has_full_neighbor(&self.this_node, public_key)) + .filter(|public_key| self.has_half_neighbor(&self.this_node, public_key)) .collect::>() } @@ -711,7 +711,7 @@ mod tests { subject.add_arbitrary_half_neighbor(root_node.public_key(), half_neighbor_1.public_key()); subject.add_arbitrary_half_neighbor(root_node.public_key(), half_neighbor_2.public_key()); - let result = subject.get_all_neighbors(); + let result = subject.get_all_half_neighbors(); assert_eq!( result, From 18191a0b4a5c8bc77e0a206d3d8d6b08411a041e Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Fri, 25 Nov 2022 14:11:36 +0530 Subject: [PATCH 19/25] GH-631: remove compiler warnings --- node/src/neighborhood/gossip.rs | 2 +- node/src/neighborhood/gossip_acceptor.rs | 13 +++---------- node/src/neighborhood/mod.rs | 6 ++---- node/src/neighborhood/neighborhood_database.rs | 17 +++++++++-------- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/node/src/neighborhood/gossip.rs b/node/src/neighborhood/gossip.rs index e3589be60..bab823d8a 100644 --- a/node/src/neighborhood/gossip.rs +++ b/node/src/neighborhood/gossip.rs @@ -13,7 +13,7 @@ use crate::sub_lib::versioned_data::StepError; use pretty_hex::PrettyHex; use serde_cbor::Value; use serde_derive::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::convert::{TryFrom, TryInto}; use std::fmt::Debug; use std::fmt::Error; diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 5d7568d1f..44601c94e 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -14,7 +14,6 @@ use masq_lib::logger::Logger; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr}; -use std::ops::Deref; use std::time::{Duration, SystemTime}; /// Note: if you decide to change this, make sure you test thoroughly. Values less than 5 may lead @@ -2552,7 +2551,6 @@ mod tests { let cryptde = main_cryptde(); let (cpm_recipient, _) = make_cpm_recipient(); let subject = StandardGossipHandler::new(Logger::new("test")); - let mut patch: HashSet = HashSet::new(); let node_a = make_node_record(1111, true); let node_b = make_node_record(2222, true); let node_c = make_node_record(3333, false); @@ -2608,13 +2606,6 @@ mod tests { } //---------------------------------------------------------------- - let hashmap = agrs - .iter() - .map(|agr| { - return (agr.inner.public_key.clone(), agr); - }) - .collect::>(); - let result = subject.handle(cryptde, &mut node_a_db, agrs, gossip_source, &cpm_recipient); assert_eq!(result, GossipAcceptanceResult::Ignored); @@ -3713,7 +3704,9 @@ mod tests { let after = time_t_timestamp(); let mut expected_dest_db = src_db.clone(); expected_dest_db.add_arbitrary_half_neighbor(dest_node.public_key(), src_node.public_key()); - expected_dest_db.remove_neighbor(disconnected_node.public_key()); + expected_dest_db + .remove_neighbor(disconnected_node.public_key()) + .unwrap(); for idx in 0..MAX_DEGREE { let failed_node_key = &expected_dest_db .add_node(make_node_record(4000 + idx as u16, true)) diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index e8bf3475d..db2c50be0 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -8,7 +8,7 @@ pub mod neighborhood_database; pub mod node_record; pub mod overall_connection_status; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::convert::TryFrom; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; @@ -46,6 +46,7 @@ use crate::sub_lib::cryptde::{CryptDE, CryptData, PlainData}; use crate::sub_lib::dispatcher::{Component, StreamShutdownMsg}; use crate::sub_lib::hopper::{ExpiredCoresPackage, NoLookupIncipientCoresPackage}; use crate::sub_lib::hopper::{IncipientCoresPackage, MessageType}; +use crate::sub_lib::neighborhood::NodeQueryMessage; use crate::sub_lib::neighborhood::NodeQueryResponseMetadata; use crate::sub_lib::neighborhood::NodeRecordMetadataMessage; use crate::sub_lib::neighborhood::RemoveNeighborMessage; @@ -56,7 +57,6 @@ use crate::sub_lib::neighborhood::{ConnectionProgressEvent, ExpectedServices}; use crate::sub_lib::neighborhood::{ConnectionProgressMessage, ExpectedService}; use crate::sub_lib::neighborhood::{DispatcherNodeQueryMessage, GossipFailure_0v1}; use crate::sub_lib::neighborhood::{NeighborhoodSubs, NeighborhoodTools}; -use crate::sub_lib::neighborhood::{NodeQueryMessage, DEFAULT_RATE_PACK}; use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::peer_actors::{BindMessage, NewPublicIp, StartMessage}; use crate::sub_lib::proxy_server::DEFAULT_MINIMUM_HOP_COUNT; @@ -67,7 +67,6 @@ use crate::sub_lib::stream_handler_pool::DispatcherNodeQueryResponse; use crate::sub_lib::utils::{handle_ui_crash_request, NODE_MAILBOX_CAPACITY}; use crate::sub_lib::versioned_data::VersionedData; use crate::sub_lib::wallet::Wallet; -use crate::test_utils::make_wallet; use gossip_acceptor::GossipAcceptor; use gossip_acceptor::GossipAcceptorReal; use gossip_producer::GossipProducer; @@ -75,7 +74,6 @@ use gossip_producer::GossipProducerReal; use masq_lib::blockchains::chains::Chain; use masq_lib::crash_point::CrashPoint; use masq_lib::logger::Logger; -use masq_lib::messages::UiSetupResponseValueStatus::Default; use neighborhood_database::NeighborhoodDatabase; use node_record::NodeRecord; diff --git a/node/src/neighborhood/neighborhood_database.rs b/node/src/neighborhood/neighborhood_database.rs index 2f982c8e5..2ecb134cb 100644 --- a/node/src/neighborhood/neighborhood_database.rs +++ b/node/src/neighborhood/neighborhood_database.rs @@ -19,7 +19,6 @@ use std::fmt::Debug; use std::fmt::Error; use std::fmt::Formatter; use std::net::IpAddr; -use std::ops::Deref; pub const ISOLATED_NODE_GRACE_PERIOD_SECS: u32 = 30; @@ -699,12 +698,12 @@ mod tests { let half_neighbor_2 = make_node_record(6666, false); let some_node = make_node_record(7777, false); let mut subject = db_from_node(&root_node); - subject.add_node(full_neighbor_1.clone()); - subject.add_node(full_neighbor_2.clone()); - subject.add_node(full_neighbor_3.clone()); - subject.add_node(half_neighbor_1.clone()); - subject.add_node(half_neighbor_2.clone()); - subject.add_node(some_node.clone()); + subject.add_node(full_neighbor_1.clone()).unwrap(); + subject.add_node(full_neighbor_2.clone()).unwrap(); + subject.add_node(full_neighbor_3.clone()).unwrap(); + subject.add_node(half_neighbor_1.clone()).unwrap(); + subject.add_node(half_neighbor_2.clone()).unwrap(); + subject.add_node(some_node.clone()).unwrap(); subject.add_arbitrary_full_neighbor(root_node.public_key(), full_neighbor_1.public_key()); subject.add_arbitrary_full_neighbor(root_node.public_key(), full_neighbor_2.public_key()); subject.add_arbitrary_full_neighbor(root_node.public_key(), full_neighbor_3.public_key()); @@ -718,7 +717,9 @@ mod tests { vec![ full_neighbor_1.public_key().clone(), full_neighbor_2.public_key().clone(), - full_neighbor_3.public_key().clone() + full_neighbor_3.public_key().clone(), + half_neighbor_1.public_key().clone(), + half_neighbor_2.public_key().clone() ] .iter() .collect::>() From d1281af570e0070724d464ac27ac2e53825a5b8c Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Fri, 25 Nov 2022 14:20:40 +0530 Subject: [PATCH 20/25] GH-631: filter agrs before calling the function that'll add introductory node --- node/src/neighborhood/gossip_acceptor.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 44601c94e..0bd27dc17 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -934,20 +934,17 @@ impl GossipHandler for StandardGossipHandler { // let hashmap: HashMap = agrs.into(); - eprintln!("HashMap: {:?}", hashmap.keys()); - let root_node = database.root(); let mut patch: HashSet = HashSet::new(); self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3, &database); - eprintln!("Patch: {:?}", patch); + let agrs = agrs + .into_iter() + .filter(|agr| patch.contains(&agr.inner.public_key)) + .collect::>(); - let mut db_changed = self.identify_and_add_non_introductory_new_nodes( - database, - &agrs, - &patch, - gossip_source, - ); + let mut db_changed = + self.identify_and_add_non_introductory_new_nodes(database, &agrs, gossip_source); db_changed = self.identify_and_update_obsolete_nodes(database, agrs) || db_changed; db_changed = self.handle_root_node(cryptde, database, gossip_source) || db_changed; let final_neighborship_status = @@ -1022,7 +1019,6 @@ impl StandardGossipHandler { &self, database: &mut NeighborhoodDatabase, agrs: &[AccessibleGossipRecord], - patch: &HashSet, gossip_source: SocketAddr, ) -> bool { let all_keys = database @@ -1031,7 +1027,6 @@ impl StandardGossipHandler { .cloned() .collect::>(); agrs.iter() - .filter(|agr| patch.contains(&agr.inner.public_key)) .filter(|agr| !all_keys.contains(&agr.inner.public_key)) // TODO: A node that tells us the IP Address of the node that isn't in our database should be malefactor banned .filter(|agr| match &agr.node_addr_opt { From 663ba06f64050d6ef6dc2fc9e765b7fad6ede941 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Fri, 25 Nov 2022 14:49:42 +0530 Subject: [PATCH 21/25] GH-631: only use references to build hashmap from agr --- node/src/neighborhood/gossip_acceptor.rs | 35 +++++++++++------------- node/src/neighborhood/mod.rs | 10 ------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 0bd27dc17..74d1a096d 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -929,14 +929,17 @@ impl GossipHandler for StandardGossipHandler { let hashmap = agrs .iter() - .map(|agr| (agr.inner.public_key.clone(), agr)) - .collect::>(); + .map(|agr| (&agr.inner.public_key, agr)) + .collect::>(); - // let hashmap: HashMap = agrs.into(); - - let root_node = database.root(); let mut patch: HashSet = HashSet::new(); - self.compute_patch(&mut patch, root_node.public_key(), &hashmap, 3, &database); + self.compute_patch( + &mut patch, + database.root().public_key(), + &hashmap, + 3, + &database, + ); let agrs = agrs .into_iter() @@ -983,7 +986,7 @@ impl StandardGossipHandler { &self, patch: &mut HashSet, node: &PublicKey, - agrs: &HashMap, + agrs: &HashMap<&PublicKey, &AccessibleGossipRecord>, hops_remaining: usize, database: &NeighborhoodDatabase, ) { @@ -2351,10 +2354,8 @@ mod tests { let hashmap = agrs .iter() - .map(|agr| { - return (agr.inner.public_key.clone(), agr); - }) - .collect::>(); + .map(|agr| (&agr.inner.public_key, agr)) + .collect::>(); subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3, &node_a_db); @@ -2430,10 +2431,8 @@ mod tests { let hashmap = agrs .iter() - .map(|agr| { - return (agr.inner.public_key.clone(), agr); - }) - .collect::>(); + .map(|agr| (&agr.inner.public_key, agr)) + .collect::>(); subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3, &node_a_db); @@ -2505,10 +2504,8 @@ mod tests { let hashmap = agrs .iter() - .map(|agr| { - return (agr.inner.public_key.clone(), agr); - }) - .collect::>(); + .map(|agr| (&agr.inner.public_key, agr)) + .collect::>(); subject.compute_patch(&mut patch, node_a.public_key(), &hashmap, 3, &node_a_db); diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index db2c50be0..23d38a97a 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -418,16 +418,6 @@ impl TryFrom for AccessibleGossipRecord { } } -// impl<'agr> Into> for AccessibleGossipRecord { -// fn into(self) -> HashMap { -// self.iter() -// .map(|agr| { -// return (agr.inner.public_key.clone(), agr); -// }) -// .collect::>() -// } -// } - #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum RouteDirection { Over, From ad33fadd36a0552676b188e7383cee2157da75e2 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Mon, 28 Nov 2022 17:52:20 +0530 Subject: [PATCH 22/25] GH-631: use constant for computing patch --- node/src/neighborhood/gossip_acceptor.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 74d1a096d..f0d72c46f 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -9,6 +9,7 @@ use crate::sub_lib::neighborhood::{ ConnectionProgressEvent, ConnectionProgressMessage, GossipFailure_0v1, }; use crate::sub_lib::node_addr::NodeAddr; +use crate::sub_lib::proxy_server::DEFAULT_MINIMUM_HOP_COUNT; use actix::Recipient; use masq_lib::logger::Logger; use std::cell::RefCell; @@ -927,7 +928,7 @@ impl GossipHandler for StandardGossipHandler { let initial_neighborship_status = StandardGossipHandler::check_full_neighbor(database, gossip_source.ip()); - let hashmap = agrs + let agrs_by_key = agrs .iter() .map(|agr| (&agr.inner.public_key, agr)) .collect::>(); @@ -936,8 +937,8 @@ impl GossipHandler for StandardGossipHandler { self.compute_patch( &mut patch, database.root().public_key(), - &hashmap, - 3, + &agrs_by_key, + DEFAULT_MINIMUM_HOP_COUNT, &database, ); From fff58c8c63a62a13da91c3abcede9d3477722238 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 29 Nov 2022 13:29:15 +0530 Subject: [PATCH 23/25] GH-631: remove redundant code --- node/src/neighborhood/gossip.rs | 21 ------- node/src/neighborhood/gossip_acceptor.rs | 73 ------------------------ 2 files changed, 94 deletions(-) diff --git a/node/src/neighborhood/gossip.rs b/node/src/neighborhood/gossip.rs index bab823d8a..0b540b37a 100644 --- a/node/src/neighborhood/gossip.rs +++ b/node/src/neighborhood/gossip.rs @@ -180,27 +180,6 @@ impl TryInto> for Gossip_0v1 { } } -// impl TryInto> for Gossip_0v1 { -// type Error = String; -// -// fn try_into(self) -> Result, Self::Error> { -// let mut hashmap: HashMap = HashMap::new(); -// for node_record in self.node_records { -// let node_addr = match &node_record.node_addr_opt { -// Some(node_addr) => node_addr.clone(), -// None => return Err(String::from("node addr not found")), -// }; -// let agr = match AccessibleGossipRecord::try_from(node_record) { -// Ok(agr) => agr, -// Err(e) => return Err(e), -// }; -// hashmap.insert(node_addr, &agr); -// } -// -// Ok(hashmap) -// } -// } - #[derive(Clone)] pub struct DotGossipEndpoint { pub public_key: PublicKey, diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index f0d72c46f..663348f9a 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -2335,24 +2335,6 @@ mod tests { .build(); let agrs: Vec = gossip.try_into().unwrap(); - //----------------------- Debugging ----------------------------- - let nodes = vec![ - node_a.public_key().clone(), - node_b.public_key().clone(), - node_c.public_key().clone(), - node_d.public_key().clone(), - node_e.public_key().clone(), - ]; - eprintln!("Nodes: {:?}", nodes); - - for agr in &agrs { - eprintln!( - "AGR found with public key: {:?}, which has neighbors: {:?}", - agr.inner.public_key, agr.inner.neighbors - ) - } - //---------------------------------------------------------------- - let hashmap = agrs .iter() .map(|agr| (&agr.inner.public_key, agr)) @@ -2412,24 +2394,6 @@ mod tests { .build(); let agrs: Vec = gossip.try_into().unwrap(); - //----------------------- Debugging ----------------------------- - let nodes = vec![ - node_a.public_key().clone(), - node_b.public_key().clone(), - node_c.public_key().clone(), - node_d.public_key().clone(), - node_y.public_key().clone(), - ]; - eprintln!("Nodes: {:?}", nodes); - - for agr in &agrs { - eprintln!( - "AGR found with public key: {:?}, which has neighbors: {:?}", - agr.inner.public_key, agr.inner.neighbors - ) - } - //---------------------------------------------------------------- - let hashmap = agrs .iter() .map(|agr| (&agr.inner.public_key, agr)) @@ -2485,24 +2449,6 @@ mod tests { .build(); let agrs: Vec = gossip.try_into().unwrap(); - //----------------------- Debugging ----------------------------- - let nodes = vec![ - node_a.public_key().clone(), - node_b.public_key().clone(), - node_c.public_key().clone(), - node_d.public_key().clone(), - node_x.public_key().clone(), - ]; - eprintln!("Nodes: {:?}", nodes); - - for agr in &agrs { - eprintln!( - "AGR found with public key: {:?}, which has neighbors: {:?}", - agr.inner.public_key, agr.inner.neighbors - ) - } - //---------------------------------------------------------------- - let hashmap = agrs .iter() .map(|agr| (&agr.inner.public_key, agr)) @@ -2580,25 +2526,6 @@ mod tests { .build(); let agrs: Vec = gossip.try_into().unwrap(); - //----------------------- Debugging ----------------------------- - let nodes = vec![ - node_a.public_key().clone(), - node_b.public_key().clone(), - node_c.public_key().clone(), - node_d.public_key().clone(), - node_e.public_key().clone(), - node_f.public_key().clone(), - ]; - eprintln!("Nodes: {:?}", nodes); - - for agr in &agrs { - eprintln!( - "AGR found with public key: {:?}, which has neighbors: {:?}", - agr.inner.public_key, agr.inner.neighbors - ) - } - //---------------------------------------------------------------- - let result = subject.handle(cryptde, &mut node_a_db, agrs, gossip_source, &cpm_recipient); assert_eq!(result, GossipAcceptanceResult::Ignored); From 53bd30a05ac6c55e201322dddf2e14b36f3fbb23 Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 29 Nov 2022 13:40:07 +0530 Subject: [PATCH 24/25] GH-631: remove clippy warnings --- node/src/neighborhood/gossip_acceptor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 663348f9a..bca39c7e5 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -939,7 +939,7 @@ impl GossipHandler for StandardGossipHandler { database.root().public_key(), &agrs_by_key, DEFAULT_MINIMUM_HOP_COUNT, - &database, + database, ); let agrs = agrs @@ -1000,7 +1000,7 @@ impl StandardGossipHandler { match agrs.get(node) { Some(agr) => agr.get_all_inner_neighbors(), None => { - patch.remove(&node); + patch.remove(node); trace!( self.logger, "While computing patch no AGR record found for public key {:?}", From 0588c33ccaf5b88f9fcc4a30a5d7fdf18cd8c9dc Mon Sep 17 00:00:00 2001 From: utkarshg6 Date: Tue, 29 Nov 2022 13:45:23 +0530 Subject: [PATCH 25/25] GH-631: rename test to can_retrieve_all_full_and_half_neighbors() --- node/src/neighborhood/neighborhood_database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/neighborhood/neighborhood_database.rs b/node/src/neighborhood/neighborhood_database.rs index 2ecb134cb..5f16375dc 100644 --- a/node/src/neighborhood/neighborhood_database.rs +++ b/node/src/neighborhood/neighborhood_database.rs @@ -689,7 +689,7 @@ mod tests { } #[test] - fn can_retrieve_all_full_neighbors() { + fn can_retrieve_all_full_and_half_neighbors() { let root_node = make_node_record(1111, true); let full_neighbor_1 = make_node_record(2222, true); let full_neighbor_2 = make_node_record(3333, true);