Add fee and mempool information to txs fetched in the mempool (#508)

Author: NicolasDorier

NBXplorer.Client/Models/GetTransactionsResponse.cs

@@ -106,5 +106,6 @@ public IMoney BalanceChange
 		public uint256 ReplacedBy { get; set; }
 		public uint256 Replacing { get; set; }
 		public bool Replaceable { get; set; }
+		public TransactionMetadata Metadata { get; set; }
 	}
 }

NBXplorer.Client/Models/TransactionMetadata.cs

@@ -0,0 +1,28 @@
+using NBitcoin;
+using NBitcoin.JsonConverters;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace NBXplorer.Models
+{
+	public class TransactionMetadata
+	{
+		[JsonProperty("vsize", DefaultValueHandling = DefaultValueHandling.Ignore)]
+		public int? VirtualSize { get; set; }
+		[JsonProperty("fees", DefaultValueHandling = DefaultValueHandling.Ignore)]
+		[JsonConverter(typeof(NBXplorer.JsonConverters.MoneyJsonConverter))]
+		public Money Fees { get; set; }
+		[JsonProperty("feeRate", DefaultValueHandling = DefaultValueHandling.Ignore)]
+		[JsonConverter(typeof(NBitcoin.JsonConverters.FeeRateJsonConverter))]
+		public FeeRate FeeRate { get; set; }
+		public static TransactionMetadata Parse(string json) => JsonConvert.DeserializeObject<TransactionMetadata>(json);
+		public string ToString(bool indented) => JsonConvert.SerializeObject(this, indented ? Formatting.Indented : Formatting.None);
+		public override string ToString() => ToString(true);
+
+		[JsonExtensionData]
+		public IDictionary<string, JToken> AdditionalData { get; set; } = new Dictionary<string, JToken>();
+	}
+}

NBXplorer.Client/Models/TransactionResult.cs

@@ -55,5 +55,7 @@ public DateTimeOffset Timestamp
 			set;
 		}
 		public uint256 ReplacedBy { get; set; }
+
+		public TransactionMetadata Metadata { get; set; }
 	}
 }

NBXplorer.Tests/ServerTester.cs

@@ -206,7 +206,6 @@ public static string GetTestPostgres(string dbName, string applicationName)
 		public HttpClient HttpClient { get; internal set; }
 
 		string datadir;
-
 		public void ResetExplorer(bool deleteAll = true)
 		{
 			Host.Dispose();

NBXplorer.Tests/UnitTest1.cs

@@ -27,7 +27,6 @@
 using System.Globalization;
 using System.Net;
 using NBXplorer.HostedServices;
-using NBitcoin.Altcoins;
 using static NBXplorer.Backend.DbConnectionHelper;
 
 namespace NBXplorer.Tests
@@ -1179,8 +1178,12 @@ public async Task ShowRBFedTransaction4()
 		[InlineData(false)]
 		public async Task ShowRBFedTransaction3(bool cancelB)
 		{
-			// Let's do a chain of two transactions implicating Bob A and B.
-			// Then B get replaced by B'.
+			// Let's do a chain of two transactions
+			// A: Cashcow sends money to Bob (100K sats)
+			// B: Cashcow spends the change to another address of Bob (200K sats)
+			// Cashcow then create B' which will double spend B.
+			// If `cancelB==true`: B' cancel the 200K output of B and send it back to himself
+			// Else, B' just bump the fees.
 			// We should make sure that B' is still saved in the database, and B properly marked as replaced.
 			// If cancelB is true, then B' output shouldn't be related to Bob.
 			using var tester = ServerTester.Create();
@@ -1191,6 +1194,7 @@ public async Task ShowRBFedTransaction3(bool cancelB)
 			var bobAddr = await tester.Client.GetUnusedAsync(bob, DerivationFeature.Deposit, 0);
 			var bobAddr1 = await tester.Client.GetUnusedAsync(bob, DerivationFeature.Deposit, 1);
 
+			// A: Cashcow sends money to Bob (100K sats)
 			var aId = tester.RPC.SendToAddress(bobAddr.ScriptPubKey, Money.Satoshis(100_000), new SendToAddressParameters() { Replaceable = true });
 			var a = tester.Notifications.WaitForTransaction(bob, aId).TransactionData.Transaction;
 			Logs.Tester.LogInformation("a: " + aId);
@@ -1199,13 +1203,16 @@ public async Task ShowRBFedTransaction3(bool cancelB)
 			var changeAddr = a.Outputs.Where(o => o.ScriptPubKey != bobAddr.ScriptPubKey).First().ScriptPubKey;
 			LockTestCoins(tester.RPC, new HashSet<Script>() { changeAddr });
 
+			// B: Cashcow spends the change to another address of Bob (200K sats)
 			var bId = tester.RPC.SendToAddress(bobAddr1.ScriptPubKey, Money.Satoshis(200_000), new SendToAddressParameters() { Replaceable = true });
 			var b = tester.Notifications.WaitForTransaction(bob, bId).TransactionData.Transaction;
 			Logs.Tester.LogInformation("b: " + bId);
 
 			// b' shouldn't have any output belonging to our wallets.
 			var bp = b.Clone();
 			var o = bp.Outputs.First(o => o.ScriptPubKey == bobAddr1.ScriptPubKey);
+
+			// If `cancelB==true`: B' cancel the 200K output of B and send it back to himself
 			if (cancelB)
 				o.ScriptPubKey = changeAddr;
 			o.Value -= Money.Satoshis(5000); // Add some fee to bump the tx
@@ -1218,17 +1225,28 @@ public async Task ShowRBFedTransaction3(bool cancelB)
 			await tester.RPC.SendRawTransactionAsync(bp);
 			Logs.Tester.LogInformation("bp: " + bp.GetHash());
 
-			// If not a cancellation, B' should send an event, and replacing B
+			// If not a cancellation, B' should send an event to bob wallet, and replacing B
 			if (!cancelB)
 			{
 				var evt = tester.Notifications.WaitForTransaction(bob, bp.GetHash());
 				Assert.Equal(bId, Assert.Single(evt.Replacing));
+				Assert.NotNull(evt.TransactionData.Metadata.VirtualSize);
+				Assert.NotNull(evt.TransactionData.Metadata.FeeRate);
+				Assert.NotNull(evt.TransactionData.Metadata.Fees);
 			}
 
 			tester.Notifications.WaitForBlocks(tester.RPC.EnsureGenerate(1));
+
 			var bpr = await tester.Client.GetTransactionAsync(bp.GetHash());
 			Assert.NotNull(bpr?.Transaction);
 			Assert.Equal(1, bpr.Confirmations);
+
+			// We are sure that bpr passed by the mempool before being mined
+			if (!cancelB)
+			{
+				Assert.NotNull(bpr.Metadata);
+			}
+
 			var br = await tester.Client.GetTransactionAsync(b.GetHash());
 			Assert.NotNull(br?.Transaction);
 			Assert.Equal(bp.GetHash(), br.ReplacedBy);
@@ -2857,6 +2875,11 @@ public async Task CanGetTransactionsOfDerivation()
 				Assert.Equal(Money.Coins(-0.8m), result.UnconfirmedTransactions.Transactions[0].BalanceChange);
 				var tx3 = await tester.Client.GetTransactionAsync(pubkey, txId3);
 				Assert.Equal(Money.Coins(-0.8m), tx3.BalanceChange);
+
+				var metadata = result.UnconfirmedTransactions.Transactions[0].Metadata;
+				Assert.NotNull(metadata.Fees);
+				Assert.NotNull(metadata.FeeRate);
+				Assert.NotNull(metadata.VirtualSize);
 			}
 		}
 
@@ -3082,6 +3105,9 @@ public async Task CanTrack()
 
 				Logs.Tester.LogInformation("Let's check that we can query the UTXO with 2 confirmations");
 				tx = tester.Client.GetTransaction(tx.Transaction.GetHash());
+				Assert.Equal(tx.Transaction.GetVirtualSize(), tx.Metadata.VirtualSize);
+				Assert.NotNull(tx.Metadata.Fees);
+				Assert.NotNull(tx.Metadata.FeeRate);
 				Assert.Equal(2, tx.Confirmations);
 				Assert.NotNull(tx.BlockId);
 

NBXplorer/Backend/DbConnectionHelper.cs

@@ -1,6 +1,7 @@
 #nullable enable
 using Dapper;
 using NBitcoin;
+using NBitcoin.RPC;
 using NBXplorer.DerivationStrategy;
 using Npgsql;
 using System;
@@ -115,7 +116,7 @@ public record SaveTransactionRecord(Transaction? Transaction, uint256 Id, uint25
 					);
 		}
 
-		public async Task SaveTransactions(IEnumerable<SaveTransactionRecord> transactions)
+		public async Task SaveTransactions(IEnumerable<SaveTransactionRecord> transactions, Dictionary<uint256, MempoolEntry> mempoolEntries)
 		{
 			var parameters = transactions
 				.DistinctBy(o => o.Id)
@@ -124,19 +125,26 @@ public async Task SaveTransactions(IEnumerable<SaveTransactionRecord> transactio
 			{
 				code = Network.CryptoCode,
 				blk_id = tx.BlockId?.ToString(),
-				id = tx.Id?.ToString() ?? tx.Transaction?.GetHash()?.ToString(),
+				id = tx.Id.ToString(),
 				raw = tx.Transaction?.ToBytes(),
 				mempool = tx.BlockId is null,
 				seen_at = tx.SeenAt,
 				blk_idx = tx.BlockIndex is int i ? i : 0,
 				blk_height = tx.BlockHeight,
-				immature = tx.Immature
+				immature = tx.Immature,
+				metadata = mempoolEntries.TryGetValue(tx.Id, out var meta) ? meta.ToTransactionMetadata().ToString(false) : null
 			})
 			.Where(o => o.id is not null)
 			.ToArray();
-			await Connection.ExecuteAsync("INSERT INTO txs(code, tx_id, raw, immature, seen_at) VALUES (@code, @id, @raw, @immature, COALESCE(@seen_at, CURRENT_TIMESTAMP)) " +
-										  " ON CONFLICT (code, tx_id) " +
-										  " DO UPDATE SET seen_at=LEAST(COALESCE(@seen_at, CURRENT_TIMESTAMP), txs.seen_at), raw = COALESCE(@raw, txs.raw), immature=EXCLUDED.immature", parameters);
+			await Connection.ExecuteAsync("""
+				INSERT INTO txs(code, tx_id, raw, immature, seen_at, metadata) VALUES (@code, @id, @raw, @immature, COALESCE(@seen_at, CURRENT_TIMESTAMP), @metadata::JSONB)
+				ON CONFLICT (code, tx_id)
+				DO UPDATE SET
+					seen_at=LEAST(COALESCE(@seen_at, CURRENT_TIMESTAMP), txs.seen_at),
+					raw = COALESCE(@raw, txs.raw),
+					immature=EXCLUDED.immature,
+					metadata=COALESCE(@metadata::JSONB, txs.metadata);
+				""", parameters);
 			await Connection.ExecuteAsync("INSERT INTO blks_txs VALUES (@code, @blk_id, @id, @blk_idx) ON CONFLICT DO NOTHING", parameters.Where(p => p.blk_id is not null).AsList());
 		}
 

NBXplorer/Backend/Indexer.cs

@@ -503,7 +503,8 @@ private async Task SaveMatches(DbConnectionHelper conn, List<Transaction> transa
 							Confirmations = confirmations,
 							Timestamp = now,
 							Transaction = matches[i].Transaction,
-							TransactionHash = matches[i].TransactionHash
+							TransactionHash = matches[i].TransactionHash,
+							Metadata = matches[i].Metadata
 						},
 						Inputs = matches[i].MatchedInputs,
 						Outputs = matches[i].MatchedOutputs,

NBXplorer/Backend/Repository.cs

@@ -18,7 +18,6 @@
 using System.Text.RegularExpressions;
 using Npgsql;
 using static NBXplorer.Backend.DbConnectionHelper;
-using Microsoft.AspNetCore.DataProtection.KeyManagement;
 
 
 namespace NBXplorer.Backend
@@ -72,7 +71,7 @@ JOIN unnest(@keypaths) AS k(keypath) ON nbxv1_get_keypath_index(d.metadata, k.ke
 				code = Network.CryptoCode,
 				wid = w.wid,
 				keypaths = keyPaths.Select(k => k.ToString()).ToArray()
-			}) ;
+			});
 		}
 
 		public record DescriptorKey(string code, string descriptor);
@@ -629,19 +628,25 @@ public async Task<TrackedTransaction[]> GetMatches(DbConnectionHelper connection
 		}
 		async Task<(TrackedTransaction[] TrackedTransactions, SaveTransactionRecord[] Saved)> SaveMatches(DbConnectionHelper connection, MatchQuery matchQuery, IList<SaveTransactionRecord> records, CancellationToken cancellationToken = default)
 		{
-			HashSet<uint256> unconfTxs = await connection.GetUnconfirmedTxs();
+			HashSet<uint256> unconfTxs = null;;
 			Dictionary<uint256, SaveTransactionRecord> txs = new();
 			HashSet<uint256> savedTxs = new();
 			List<dynamic> matchedConflicts = new List<dynamic>();
 			var scripts = new List<Script>();
 			foreach (var record in records)
 			{
 				txs.TryAdd(record.Id, record);
-				if (record.BlockId is not null && unconfTxs.Contains(record.Id))
-					// If a block has been found, and we have some unconf transactions
-					// then we want to add an entry in blks_txs, even if the unconf tx isn't matching
-					// any wallet. So we add record.
-					savedTxs.Add(record.Id);
+				if (record.BlockId is not null)
+				{
+					unconfTxs ??= await connection.GetUnconfirmedTxs();
+					if (unconfTxs.Contains(record.Id))
+					{
+						// If a block has been found, and we have some unconf transactions
+						// then we want to add an entry in blks_txs, even if the unconf tx isn't matching
+						// any wallet. So we add record.
+						savedTxs.Add(record.Id);
+					}
+				}
 			}
 			SaveTransactionRecord[] GetSavedTxs() => txs.Values.Where(r => savedTxs.Contains(r.Id)).ToArray();
 
@@ -681,7 +686,11 @@ public async Task<TrackedTransaction[]> GetMatches(DbConnectionHelper connection
 			end:
 			if (savedTxs.Count is 0)
 				return (Array.Empty<TrackedTransaction>(), GetSavedTxs());
-			await connection.SaveTransactions(savedTxs.Select(h => txs[h]).ToArray());
+			
+			var metadata = await rpc.FetchMempoolInfo(GetSavedTxs().Where(tx => tx.BlockId is null).Select(tx => tx.Id), cancellationToken);
+			await connection.SaveTransactions(GetSavedTxs(), metadata);
+			
+
 			if (scripts.Count is 0)
 				return (Array.Empty<TrackedTransaction>(), GetSavedTxs());
 			var allKeyInfos = await GetKeyInformations(connection.Connection, scripts);
@@ -727,21 +736,22 @@ private void AddReplacementInfo(List<dynamic> matchedConflicts, TrackedTransacti
 		public Task CommitMatches(DbConnection connection)
 		=> connection.ExecuteAsync("CALL save_matches(@code)", new { code = Network.CryptoCode });
 
-		record SavedTransactionRow(byte[] raw, string blk_id, long? blk_height, string replaced_by, DateTime seen_at);
-		public async Task<SavedTransaction[]> GetSavedTransactions(uint256 txid)
+		record SavedTransactionRow(byte[] raw, string metadata, string blk_id, long? blk_height, string replaced_by, DateTime seen_at);
+		public async Task<SavedTransaction> GetSavedTransaction(uint256 txid)
 		{
 			await using var connection = await connectionFactory.CreateConnectionHelper(Network);
-			var tx = await connection.Connection.QueryFirstOrDefaultAsync<SavedTransactionRow>("SELECT raw, blk_id, blk_height, replaced_by, seen_at FROM txs WHERE code=@code AND tx_id=@tx_id", new { code = Network.CryptoCode, tx_id = txid.ToString() });
+			var tx = await connection.Connection.QueryFirstOrDefaultAsync<SavedTransactionRow>("SELECT raw, metadata, blk_id, blk_height, replaced_by, seen_at FROM txs WHERE code=@code AND tx_id=@tx_id", new { code = Network.CryptoCode, tx_id = txid.ToString() });
 			if (tx?.raw is null)
-				return Array.Empty<SavedTransaction>();
-			return new[] { new SavedTransaction()
+				return null;
+			return new SavedTransaction()
 			{
 				BlockHash = tx.blk_id is null ? null : uint256.Parse(tx.blk_id),
 				BlockHeight = tx.blk_height,
 				Timestamp = new DateTimeOffset(tx.seen_at),
 				Transaction = Transaction.Load(tx.raw, Network.NBitcoinNetwork),
-				ReplacedBy = tx.replaced_by is null ? null : uint256.Parse(tx.replaced_by)
-			}};
+				ReplacedBy = tx.replaced_by is null ? null : uint256.Parse(tx.replaced_by),
+				Metadata = tx.metadata is null ? null : TransactionMetadata.Parse(tx.metadata)
+			};
 		}
 		public async Task<TrackedTransaction[]> GetTransactions(GetTransactionQuery query, bool includeTransactions = true, CancellationToken cancellation = default)
 		{
@@ -794,26 +804,32 @@ async Task<TrackedTransaction[]> GetTransactions(DbConnectionHelper connection,
 				}
 			}
 
-			var txsToFetch = (includeTransactions ? trackedById.Keys.Select(t => t.Item2).AsList() :
-												  // For double spend detection, we need the full transactions from unconfs
-												  trackedById.Where(t => t.Value.BlockHash is null).Select(t => t.Key.Item2).AsList()).ToHashSet();
-			var txRaws = txsToFetch.Count > 0
-				? await connection.Connection.QueryAsync<(string tx_id, byte[] raw)>(
-					"SELECT	t.tx_id, t.raw FROM unnest(@txId) i " +
-					"JOIN txs t ON t.code=@code AND t.tx_id=i " +
-					"WHERE t.raw IS NOT NULL;", new { code = Network.CryptoCode, txId = txsToFetch.ToArray() })
-				: Array.Empty<(string tx_id, byte[] raw)>();
-
-			var txRawsById = txRaws.ToDictionary(t => t.tx_id);
+			var txsToFetch = trackedById.Keys.Select(t => t.Item2).ToHashSet();
+
+			var txMetadata = await connection.Connection.QueryAsync<(string tx_id, byte[] raw, string metadata)>(
+					$"SELECT t.tx_id, t.raw, t.metadata FROM unnest(@txId) i " +
+					"JOIN txs t ON t.code=@code AND t.tx_id=i;", new { code = Network.CryptoCode, txId = txsToFetch.ToArray() });
+
+			var txMetadataByIds = txMetadata.ToDictionary(t => t.tx_id);
 			foreach (var tracked in trackedById.Values)
 			{
 				tracked.Sort();
-				if (!txRawsById.TryGetValue(tracked.Key.TxId.ToString(), out var row))
+				if (!txMetadataByIds.TryGetValue(tracked.Key.TxId.ToString(), out var row))
 					continue;
-				tracked.Transaction = Transaction.Load(row.raw, Network.NBitcoinNetwork);
-				tracked.Key = tracked.Key with { IsPruned = false };
-				if (tracked.BlockHash is null) // Only need the spend outpoint for double spend detection on unconf txs
-					tracked.SpentOutpoints.AddInputs(tracked.Transaction);
+				Transaction tx = (includeTransactions || tracked.BlockHash is null) && row.raw is not null ? Transaction.Load(row.raw, Network.NBitcoinNetwork) : null;
+				if (tx is not null)
+				{
+					tracked.Transaction = tx;
+					tracked.Key = tracked.Key with { IsPruned = false };
+				}
+				if (row.metadata is not null)
+				{
+					tracked.Metadata = TransactionMetadata.Parse(row.metadata);
+				}
+				if (tracked.BlockHash is null && tx is not null) // Only need the spend outpoint for double spend detection on unconf txs
+				{
+					tracked.SpentOutpoints.AddInputs(tx);
+				}
 			}
 
 			if (Network.IsElement)
@@ -824,7 +840,7 @@ async Task<TrackedTransaction[]> GetTransactions(DbConnectionHelper connection,
 
 		private async Task UnblindTrackedTransactions(DbConnectionHelper connection, IEnumerable<TrackedTransaction> trackedTransactions, GetTransactionQuery query)
 		{
-			var keyInfos = ((query as GetTransactionQuery.ScriptsTxIds)?.KeyInfos)?.ToMultiValueDictionary(k => k.ScriptPubKey); 
+			var keyInfos = ((query as GetTransactionQuery.ScriptsTxIds)?.KeyInfos)?.ToMultiValueDictionary(k => k.ScriptPubKey);
 			if (keyInfos is null)
 				keyInfos = (await this.GetKeyInformations(connection.Connection, trackedTransactions.SelectMany(t => t.InOuts).Select(s => s.ScriptPubKey).ToList()));
 			foreach (var tracked in trackedTransactions)
@@ -1065,12 +1081,6 @@ public async Task<TMetadata> GetMetadata<TMetadata>(TrackedSource source, string
 			return await helper.GetMetadata<TMetadata>(walletKey.wid, key);
 		}
 
-		public async Task SaveTransactions(SaveTransactionRecord[] records)
-		{
-			await using var helper = await connectionFactory.CreateConnectionHelper(Network);
-			await helper.SaveTransactions(records);
-		}
-
 		public async Task SetIndexProgress(BlockLocator locator)
 		{
 			await using var conn = await connectionFactory.CreateConnection();

NBXplorer/Backend/SavedTransaction.cs

@@ -1,4 +1,5 @@
 using NBitcoin;
+using NBXplorer.Models;
 using System;
 
 namespace NBXplorer.Backend
@@ -23,5 +24,7 @@ public DateTimeOffset Timestamp
 			set;
 		}
 		public uint256 ReplacedBy { get; set; }
+
+		public TransactionMetadata Metadata { get; set; }
 	}
 }

NBXplorer/Controllers/MainController.PSBT.cs

@@ -584,8 +584,8 @@ await Task.WhenAll(update.PSBT.Inputs
 								// If this is not segwit, or we are unsure of it, let's try to grab from our saved transactions
 								if (input.NonWitnessUtxo == null)
 								{
-									var prev = await repo.GetSavedTransactions(input.PrevOut.Hash);
-									if (prev.FirstOrDefault() is SavedTransaction saved)
+									var prev = await repo.GetSavedTransaction(input.PrevOut.Hash);
+									if (prev is SavedTransaction saved)
 									{
 										input.NonWitnessUtxo = saved.Transaction;
 									}