Test ChannelManager automatic retries

TODO: more tests

Author: valentinewallace

lightning/src/ln/payment_tests.rs

@@ -17,8 +17,10 @@ use crate::chain::transaction::OutPoint;
 use crate::chain::keysinterface::{EntropySource, KeysInterface};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
 use crate::ln::channelmanager::{self, BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS};
+use crate::ln::features::InvoiceFeatures;
 use crate::ln::msgs;
 use crate::ln::msgs::ChannelMessageHandler;
+use crate::ln::outbound_payment::Retry;
 use crate::routing::gossip::RoutingFees;
 use crate::routing::router::{get_route, PaymentParameters, RouteHint, RouteHintHop, RouteParameters};
 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
@@ -34,6 +36,11 @@ use crate::prelude::*;
 
 use crate::ln::functional_test_utils::*;
 use crate::routing::gossip::NodeId;
+#[cfg(feature = "std")]
+use {
+	crate::util::time::tests::SinceEpoch,
+	std::time::{SystemTime, Duration}
+};
 
 #[test]
 fn retry_single_path_payment() {
@@ -1564,3 +1571,187 @@ fn do_test_intercepted_payment(test: InterceptTest) {
 		assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
 	}
 }
+
+#[derive(PartialEq)]
+enum AutoRetry {
+	Success,
+	FailAttempts,
+	FailTimeout,
+	FailOnRestart,
+}
+
+#[test]
+fn automatic_retries() {
+	do_automatic_retries(AutoRetry::Success);
+	do_automatic_retries(AutoRetry::FailAttempts);
+	do_automatic_retries(AutoRetry::FailTimeout);
+	do_automatic_retries(AutoRetry::FailOnRestart);
+}
+fn do_automatic_retries(test: AutoRetry) {
+	// Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
+	// below.
+	let chanmon_cfgs = create_chanmon_cfgs(3);
+	let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+	let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+
+	let persister;
+	let new_chain_monitor;
+	let node_0_deserialized;
+
+	let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+	let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+	let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+	// Marshall data to send the payment
+	#[cfg(feature = "std")]
+	let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
+	#[cfg(not(feature = "std"))]
+	let payment_expiry_secs = 60 * 60;
+	let amt_msat = 1000;
+	let mut invoice_features = InvoiceFeatures::empty();
+	invoice_features.set_variable_length_onion_required();
+	invoice_features.set_payment_secret_required();
+	invoice_features.set_basic_mpp_optional();
+	let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id())
+		.with_expiry_time(payment_expiry_secs as u64)
+		.with_features(invoice_features);
+	let route_params = RouteParameters {
+		payment_params,
+		final_value_msat: amt_msat,
+		final_cltv_expiry_delta: TEST_FINAL_CLTV,
+	};
+	let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
+
+	macro_rules! pass_failed_attempt_with_retry_along_path {
+		($failing_channel_id: expr) => {
+			// Send a payment attempt that fails due to lack of liquidity on the second hop
+			check_added_monitors!(nodes[0], 1);
+			let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
+			let mut update_add = update_0.update_add_htlcs[0].clone();
+			nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
+			commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
+			expect_pending_htlcs_forwardable_ignore!(nodes[1]);
+			nodes[1].node.process_pending_htlc_forwards();
+			expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
+				vec![HTLCDestination::NextHopChannel {
+					node_id: Some(nodes[2].node.get_our_node_id()),
+					channel_id: $failing_channel_id,
+				}]);
+			nodes[1].node.process_pending_htlc_forwards();
+			let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+			check_added_monitors!(&nodes[1], 1);
+			assert!(update_1.update_fail_htlcs.len() == 1);
+			let fail_msg = update_1.update_fail_htlcs[0].clone();
+
+			nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
+			commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
+
+			// Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
+			let mut events = nodes[0].node.get_and_clear_pending_events();
+			assert_eq!(events.len(), 2);
+			match events[0] {
+				Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
+					assert_eq!(payment_hash, ev_payment_hash);
+					assert_eq!(payment_failed_permanently, false);
+				},
+				_ => panic!("Unexpected event"),
+			}
+			match events[1] {
+				Event::PendingHTLCsForwardable { .. } => {},
+				_ => panic!("Unexpected event"),
+			}
+		}
+	}
+
+	if test == AutoRetry::Success {
+		// Test that we can succeed on the first retry.
+		nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+		pass_failed_attempt_with_retry_along_path!(channel_id_2);
+
+		// Open a new channel with liquidity on the second hop so we can find a route for the retry
+		// attempt, since the initial second hop channel will be excluded from pathfinding
+		create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+
+		// We retry payments in `process_pending_htlc_forwards`
+		nodes[0].node.process_pending_htlc_forwards();
+		check_added_monitors!(nodes[0], 1);
+		let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
+		assert_eq!(msg_events.len(), 1);
+		pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
+		claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
+	} else if test == AutoRetry::FailAttempts {
+		// Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
+		nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+		pass_failed_attempt_with_retry_along_path!(channel_id_2);
+
+		// Open a new channel with no liquidity on the second hop so we can find a (bad) route for
+		// the retry attempt, since the initial second hop channel will be excluded from pathfinding
+		let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+		// We retry payments in `process_pending_htlc_forwards`
+		nodes[0].node.process_pending_htlc_forwards();
+		pass_failed_attempt_with_retry_along_path!(channel_id_3);
+
+		// Ensure we won't retry a second time.
+		nodes[0].node.process_pending_htlc_forwards();
+		nodes[0].logger.assert_log_regex("lightning::ln::channelmanager".to_string(), regex::Regex::new(r"Payment [0-9a-f]* is not retryable").unwrap(), 1);
+		let mut events = nodes[0].node.get_and_clear_pending_events();
+		assert_eq!(events.len(), 1);
+		match events[0] {
+			Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+				assert_eq!(payment_hash, *ev_payment_hash);
+				assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+			},
+			_ => panic!("Unexpected event"),
+		}
+	} else if test == AutoRetry::FailTimeout {
+		#[cfg(not(feature = "no-std"))] {
+			// Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
+			nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
+			pass_failed_attempt_with_retry_along_path!(channel_id_2);
+
+			// Advance the time so the second attempt fails due to timeout.
+			SinceEpoch::advance(Duration::from_secs(61));
+			nodes[0].node.process_pending_htlc_forwards();
+			nodes[0].logger.assert_log_regex("lightning::ln::channelmanager".to_string(), regex::Regex::new(r"Payment [0-9a-f]* is not retryable").unwrap(), 1);
+			let mut events = nodes[0].node.get_and_clear_pending_events();
+			assert_eq!(events.len(), 1);
+			match events[0] {
+				Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+					assert_eq!(payment_hash, *ev_payment_hash);
+					assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+				},
+				_ => panic!("Unexpected event"),
+			}
+		}
+	} else if test == AutoRetry::FailOnRestart {
+		// Ensure ChannelManager will not retry a payment after restart, even if there were retry
+		// attempts remaining prior to restart.
+		nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
+		pass_failed_attempt_with_retry_along_path!(channel_id_2);
+
+		// Open a new channel with no liquidity on the second hop so we can find a (bad) route for
+		// the retry attempt, since the initial second hop channel will be excluded from pathfinding
+		let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+		// Ensure the first retry attempt fails, with 1 retry attempt remaining
+		nodes[0].node.process_pending_htlc_forwards();
+		pass_failed_attempt_with_retry_along_path!(channel_id_3);
+
+		// Restart the node and ensure that ChannelManager does not use its remaining retry attempt
+		let node_encoded = nodes[0].node.encode();
+		let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
+		reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
+		nodes[0].node.process_pending_htlc_forwards();
+		nodes[0].logger.assert_log_regex("lightning::ln::channelmanager".to_string(), regex::Regex::new(r"Payment [0-9a-f]* is not retryable").unwrap(), 1);
+		let mut events = nodes[0].node.get_and_clear_pending_events();
+		assert_eq!(events.len(), 1);
+		match events[0] {
+			Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+				assert_eq!(payment_hash, *ev_payment_hash);
+				assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+			},
+			_ => panic!("Unexpected event"),
+		}
+	}
+}