From cc0d0541cce1f3eb194032be6196d6093314adac Mon Sep 17 00:00:00 2001 From: Jamie M Date: Mon, 27 Feb 2023 16:59:36 +0000 Subject: [PATCH 1/3] started to work on processing logs --- main/externalData/balances/logs/index.ts | 107 +++++++++++++++++++++++ main/externalData/balances/scan.ts | 89 ++++++++++++++++--- 2 files changed, 183 insertions(+), 13 deletions(-) create mode 100644 main/externalData/balances/logs/index.ts diff --git a/main/externalData/balances/logs/index.ts b/main/externalData/balances/logs/index.ts new file mode 100644 index 000000000..6546b9dae --- /dev/null +++ b/main/externalData/balances/logs/index.ts @@ -0,0 +1,107 @@ +import { toTokenId } from '../../../../resources/domain/balance' +import { hexZeroPad } from '@ethersproject/bytes' +import { BigNumber } from '@ethersproject/bignumber' +import { utils } from 'ethers' +import log from 'electron-log' +import { TokenDefinition } from '../scan' + +//TODO: plumb changes of token list / custom tokens into metadata so that we know which tokens to handle... +export enum LogTopic { + TRANSFER = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + WITHDRAWAL = '0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65', + DEPOSIT = '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c' +} + +type TokenId = string +type Address = string +type AccountBalances = Record + +interface Log { + address: Address + blockHash: string + blockNumber: string + data: string + logIndex: string + removed: boolean + topics: string[] + transactionHash: string + transactionIndex: string +} + +const metadata: Record = {} + +export const setMetadata = (tokens: TokenDefinition[]) => { + tokens.forEach((token) => (metadata[toTokenId(token)] = token)) +} + +export class LogProcessor { + private balances: AccountBalances = {} + private ownerPadded + + private handlers: Record void> = { + [LogTopic.TRANSFER]: this.handleTransfer, + [LogTopic.WITHDRAWAL]: this.handleWithdrawal, + [LogTopic.DEPOSIT]: this.handleDeposit + } + + private async processDelta(tokenId: TokenId, delta: BigNumber) { + const existing = this.balances[tokenId] + if (!existing) { + if (!metadata[tokenId]) { + log.warn('Unsupported token', { tokenId }) + return + } + log.info('Token with known metadata but no seeded balance...', { tokenId }) + //TODO: + //Fetch balance from chain, set the balance to this... + return + } + + const { balance: currentBalance, decimals } = existing + + const newBalance = delta.add(currentBalance) + this.balances[tokenId].balance = newBalance.toString() + this.balances[tokenId].displayBalance = utils.formatUnits(newBalance, decimals) + } + + private async handleTransfer(chainId: number, log: Log) { + if (parseInt(log.blockNumber, 16) <= this.lastProcessedBlock) return + const [fromPadded, toPadded, valueHex] = log.topics + const tokenId = toTokenId({ address: log.address, chainId }) + const value = BigNumber.from(valueHex) + + let delta = BigNumber.from(0) + if (fromPadded === this.ownerPadded) delta = delta.add(value.mul(-1)) + if (toPadded === this.ownerPadded) delta = delta.add(value) + + await this.processDelta(tokenId, value) + } + + private async handleWithdrawal(chainId: number, log: Log) { + const [addressPadded, valueHex] = log.topics + if (addressPadded !== this.ownerPadded) return + + const tokenId = toTokenId({ address: log.address, chainId }) + await this.processDelta(tokenId, BigNumber.from(valueHex).mul(-1)) + } + + private async handleDeposit(chainId: number, log: Log) { + const [addressPadded, valueHex] = log.topics + if (addressPadded !== this.ownerPadded) return + + const tokenId = toTokenId({ address: log.address, chainId }) + await this.processDelta(tokenId, BigNumber.from(valueHex)) + } + + public async process(chainId: number, logs: Log[], latestBlock: number) { + log.info('Processing logs', { latestBlock, chainId, owner: this.owner }) + await Promise.all(logs.map((log) => this.handlers[log.topics[0] as LogTopic](chainId, log))) + this.lastProcessedBlock = latestBlock + return Object.values(this.balances) + } + + constructor(private owner: Address, balances: Balance[], public lastProcessedBlock: number) { + balances.forEach((balance) => (this.balances[toTokenId(balance)] = balance)) + this.ownerPadded = hexZeroPad(owner, 32) + } +} diff --git a/main/externalData/balances/scan.ts b/main/externalData/balances/scan.ts index c2e2b1ecc..c3b19c9b5 100644 --- a/main/externalData/balances/scan.ts +++ b/main/externalData/balances/scan.ts @@ -3,13 +3,27 @@ import { BigNumber as EthersBigNumber } from '@ethersproject/bignumber' import { Interface } from '@ethersproject/abi' import { addHexPrefix } from '@ethereumjs/util' import log from 'electron-log' +import { hexZeroPad, BytesLike } from '@ethersproject/bytes' import multicall, { Call, supportsChain as multicallSupportsChain } from '../../multicall' import erc20TokenAbi from './erc-20-abi' import { groupByChain, TokensByChain } from './reducers' - -import type { BytesLike } from '@ethersproject/bytes' import type EthereumProvider from 'ethereum-provider' +import { LogProcessor, LogTopic } from './logs' + +interface Log { + address: Address + blockHash: string + blockNumber: string + data: string + logIndex: string + removed: boolean + topics: string[] + transactionHash: string + transactionIndex: string +} + +const logProcessors: Record = {} const erc20Interface = new Interface(erc20TokenAbi) @@ -41,6 +55,30 @@ function createBalance(rawBalance: string, decimals: number): ExternalBalance { } export default function (eth: EthereumProvider) { + async function getLatestBlock(chainId: number) { + const blockNumber: string = await eth.request({ + method: 'eth_blockNumber', + params: [], + chainId: addHexPrefix(chainId.toString(16)) + }) + return parseInt(blockNumber) + } + + async function getTransferLogs(address: string, chainId: number, fromBlock: number): Promise { + //TODO: fix this filter: need to also get logs where account is recipient... + const topics = [[LogTopic.TRANSFER, LogTopic.DEPOSIT, LogTopic.WITHDRAWAL], [hexZeroPad(address, 32)]] + const filter = { + fromBlock: '0x' + fromBlock.toString(16), + toBlock: 'latest', + topics + } + return eth.request({ + method: 'eth_getLogs', + params: [filter], + chainId: addHexPrefix(chainId.toString(16)) + }) + } + function balanceCalls(owner: string, tokens: TokenDefinition[]): Call[] { return tokens.map((token) => ({ target: token.address, @@ -84,15 +122,20 @@ export default function (eth: EthereumProvider) { return result.balance.toHexString() } - async function getTokenBalancesFromContracts(owner: string, tokens: TokenDefinition[]) { + async function getTokenBalancesFromContracts( + owner: string, + tokens: TokenDefinition[], + latestBlock: number + ) { const balances = tokens.map(async (token) => { try { const rawBalance = await getTokenBalance(token, owner) - return { + const balance = { ...token, ...createBalance(rawBalance, token.decimals) } + return balance } catch (e) { log.warn(`could not load balance for token with address ${token.address}`, e) return undefined @@ -104,21 +147,28 @@ export default function (eth: EthereumProvider) { return loadedBalances.filter((bal) => bal !== undefined) as Balance[] } - async function getTokenBalancesFromMulticall(owner: string, tokens: TokenDefinition[], chainId: number) { + async function getTokenBalancesFromMulticall( + owner: string, + tokens: TokenDefinition[], + chainId: number, + latestBlock: number + ) { const calls = balanceCalls(owner, tokens) const results = await multicall(chainId, eth).batchCall(calls) - return results.reduce((acc, result, i) => { + const balances = results.reduce((acc, result, i) => { if (result.success) { - acc.push({ + const balance = { ...tokens[i], ...result.returnValues[0] - }) + } + acc.push(balance) } return acc }, [] as Balance[]) + return balances } return { @@ -131,12 +181,25 @@ export default function (eth: EthereumProvider) { const tokensByChain = tokens.reduce(groupByChain, {} as TokensByChain) const tokenBalances = await Promise.all( - Object.entries(tokensByChain).map(([chain, tokens]) => { + Object.entries(tokensByChain).map(async ([chain, tokens]) => { const chainId = parseInt(chain) - - return multicallSupportsChain(chainId) - ? getTokenBalancesFromMulticall(owner, tokens, chainId) - : getTokenBalancesFromContracts(owner, tokens) + const latestBlock = await getLatestBlock(chainId) + const logProcessor = logProcessors[owner] + if (logProcessor) { + const logs = await getTransferLogs(owner, chainId, logProcessor.lastProcessedBlock) + return logProcessor.process(chainId, logs, latestBlock) + } else { + log.info('not seeded... will be soon though!') + } + + const balances = multicallSupportsChain(chainId) + ? await getTokenBalancesFromMulticall(owner, tokens, chainId, latestBlock) + : await getTokenBalancesFromContracts(owner, tokens, latestBlock) + + if (!logProcessor) { + logProcessors[owner] = new LogProcessor(owner, balances, latestBlock) + } + return balances }) ) From 21c07699830e7a19f48fbc96449d0a78964b0a77 Mon Sep 17 00:00:00 2001 From: Jamie M Date: Wed, 1 Mar 2023 15:43:49 +0000 Subject: [PATCH 2/3] token balance updating via logs hooked into scanning flow --- main/externalData/balances/logs/index.ts | 124 +++++++++++++++-------- main/externalData/balances/scan.ts | 89 ++++++++-------- 2 files changed, 125 insertions(+), 88 deletions(-) diff --git a/main/externalData/balances/logs/index.ts b/main/externalData/balances/logs/index.ts index 6546b9dae..62ccae71f 100644 --- a/main/externalData/balances/logs/index.ts +++ b/main/externalData/balances/logs/index.ts @@ -1,11 +1,12 @@ import { toTokenId } from '../../../../resources/domain/balance' import { hexZeroPad } from '@ethersproject/bytes' import { BigNumber } from '@ethersproject/bignumber' -import { utils } from 'ethers' import log from 'electron-log' -import { TokenDefinition } from '../scan' +import { TokenDefinition } from 'nebula/dist/ipfs/manifest/tokens' +import { BytesLike, formatUnits } from 'ethers/lib/utils' +import type EthereumProvider from 'ethereum-provider' +import { erc20Interface } from '../../../../resources/contracts' -//TODO: plumb changes of token list / custom tokens into metadata so that we know which tokens to handle... export enum LogTopic { TRANSFER = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', WITHDRAWAL = '0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65', @@ -16,7 +17,7 @@ type TokenId = string type Address = string type AccountBalances = Record -interface Log { +export interface Log { address: Address blockHash: string blockNumber: string @@ -28,79 +29,118 @@ interface Log { transactionIndex: string } -const metadata: Record = {} +type TokensDict = Record -export const setMetadata = (tokens: TokenDefinition[]) => { - tokens.forEach((token) => (metadata[toTokenId(token)] = token)) -} +const toTokenDict = (definitions: TokenDefinition[]) => + definitions.reduce((tokens: TokensDict, token) => { + const { address, chainId } = token + tokens[toTokenId({ address, chainId: parseInt(chainId) })] = token + return tokens + }, {}) export class LogProcessor { private balances: AccountBalances = {} private ownerPadded - private handlers: Record void> = { - [LogTopic.TRANSFER]: this.handleTransfer, - [LogTopic.WITHDRAWAL]: this.handleWithdrawal, - [LogTopic.DEPOSIT]: this.handleDeposit + // Map of chainId => last processed + public lastProcessed: Record = {} + + private handlers: Record void> = { + [LogTopic.TRANSFER]: this.handleTransfer.bind(this), + [LogTopic.WITHDRAWAL]: this.handleWithdrawal.bind(this), + [LogTopic.DEPOSIT]: this.handleDeposit.bind(this) + } + + private async getTokenBalance(token: TokenDefinition) { + const functionData = erc20Interface.encodeFunctionData('balanceOf', [this.owner]) + + const response: BytesLike = await this.provider.request({ + method: 'eth_call', + chainId: '0x' + Number(token.chainId).toString(16), + params: [{ to: token.address, value: '0x0', data: functionData }, 'latest'] + }) + + return BigNumber.from(response)._hex } - private async processDelta(tokenId: TokenId, delta: BigNumber) { + private async processDelta(tokenId: TokenId, delta: BigNumber, tokens: TokensDict) { const existing = this.balances[tokenId] - if (!existing) { - if (!metadata[tokenId]) { - log.warn('Unsupported token', { tokenId }) - return - } - log.info('Token with known metadata but no seeded balance...', { tokenId }) - //TODO: - //Fetch balance from chain, set the balance to this... + const tokenDefinition = tokens[tokenId] + if (!existing && !tokenDefinition) { + log.warn('Unsupported Token', { tokenId, chainId: this.chainId }) return } - const { balance: currentBalance, decimals } = existing + const balance = existing ? delta.add(existing.balance)._hex : await this.getTokenBalance(tokenDefinition) + + const { decimals } = tokenDefinition || balance - const newBalance = delta.add(currentBalance) - this.balances[tokenId].balance = newBalance.toString() - this.balances[tokenId].displayBalance = utils.formatUnits(newBalance, decimals) + this.balances[tokenId] = { + ...existing, + ...tokenDefinition, + chainId: this.chainId, + balance, + displayBalance: formatUnits(balance, decimals) + } } - private async handleTransfer(chainId: number, log: Log) { + private async handleTransfer(log: Log, tokens: TokensDict) { if (parseInt(log.blockNumber, 16) <= this.lastProcessedBlock) return - const [fromPadded, toPadded, valueHex] = log.topics - const tokenId = toTokenId({ address: log.address, chainId }) - const value = BigNumber.from(valueHex) + const [, fromPadded, toPadded] = log.topics + const tokenId = toTokenId({ address: log.address, chainId: this.chainId }) + const value = BigNumber.from(log.data) let delta = BigNumber.from(0) if (fromPadded === this.ownerPadded) delta = delta.add(value.mul(-1)) if (toPadded === this.ownerPadded) delta = delta.add(value) - await this.processDelta(tokenId, value) + await this.processDelta(tokenId, value, tokens) } - private async handleWithdrawal(chainId: number, log: Log) { - const [addressPadded, valueHex] = log.topics + private async handleWithdrawal(log: Log, tokens: TokensDict) { + const [, addressPadded] = log.topics if (addressPadded !== this.ownerPadded) return - const tokenId = toTokenId({ address: log.address, chainId }) - await this.processDelta(tokenId, BigNumber.from(valueHex).mul(-1)) + const tokenId = toTokenId({ address: log.address, chainId: this.chainId }) + await this.processDelta(tokenId, BigNumber.from(log.data).mul(-1), tokens) } - private async handleDeposit(chainId: number, log: Log) { - const [addressPadded, valueHex] = log.topics + private async handleDeposit(log: Log, tokens: TokensDict) { + const [, addressPadded] = log.topics if (addressPadded !== this.ownerPadded) return - const tokenId = toTokenId({ address: log.address, chainId }) - await this.processDelta(tokenId, BigNumber.from(valueHex)) + const tokenId = toTokenId({ address: log.address, chainId: this.chainId }) + await this.processDelta(tokenId, BigNumber.from(log.data), tokens) + } + + private async handle(eventLog: Log, tokensDict: TokensDict) { + const logBlock = parseInt(eventLog.blockNumber, 16) + log.info('Processing logs', { + lastProcessed: this.lastProcessedBlock, + logBlock, + process: logBlock > this.lastProcessedBlock + }) + + return logBlock > this.lastProcessedBlock + ? this.handlers[eventLog.topics[0] as LogTopic](eventLog, tokensDict) + : new Promise((r) => r(null)) } - public async process(chainId: number, logs: Log[], latestBlock: number) { - log.info('Processing logs', { latestBlock, chainId, owner: this.owner }) - await Promise.all(logs.map((log) => this.handlers[log.topics[0] as LogTopic](chainId, log))) + public async process(logs: Log[], latestBlock: number, tokens: TokenDefinition[]) { + log.info('Processing logs', { latestBlock, owner: this.owner }) + const tokensDict = toTokenDict(tokens) + await Promise.all(logs.map((log) => this.handle(log, tokensDict))) this.lastProcessedBlock = latestBlock return Object.values(this.balances) } - constructor(private owner: Address, balances: Balance[], public lastProcessedBlock: number) { + constructor( + private owner: Address, + balances: Balance[], + public lastProcessedBlock: number, + private chainId: number, + private provider: EthereumProvider + ) { balances.forEach((balance) => (this.balances[toTokenId(balance)] = balance)) this.ownerPadded = hexZeroPad(owner, 32) } diff --git a/main/externalData/balances/scan.ts b/main/externalData/balances/scan.ts index c3b19c9b5..bd42ef607 100644 --- a/main/externalData/balances/scan.ts +++ b/main/externalData/balances/scan.ts @@ -9,19 +9,10 @@ import multicall, { Call, supportsChain as multicallSupportsChain } from '../../ import erc20TokenAbi from './erc-20-abi' import { groupByChain, TokensByChain } from './reducers' import type EthereumProvider from 'ethereum-provider' -import { LogProcessor, LogTopic } from './logs' - -interface Log { - address: Address - blockHash: string - blockNumber: string - data: string - logIndex: string - removed: boolean - topics: string[] - transactionHash: string - transactionIndex: string -} +import { Log, LogProcessor, LogTopic } from './logs' + +//TODO: move the log processing outside of the scanning system - on startup seed the balances and then get logs for each block // at a polling interval +const toLogProcessorKey = (owner: Address, chainId: number) => `${chainId}:${owner}` const logProcessors: Record = {} @@ -65,18 +56,31 @@ export default function (eth: EthereumProvider) { } async function getTransferLogs(address: string, chainId: number, fromBlock: number): Promise { - //TODO: fix this filter: need to also get logs where account is recipient... - const topics = [[LogTopic.TRANSFER, LogTopic.DEPOSIT, LogTopic.WITHDRAWAL], [hexZeroPad(address, 32)]] - const filter = { - fromBlock: '0x' + fromBlock.toString(16), - toBlock: 'latest', - topics - } - return eth.request({ - method: 'eth_getLogs', - params: [filter], - chainId: addHexPrefix(chainId.toString(16)) - }) + const logs = (await Promise.all([ + eth.request({ + method: 'eth_getLogs', + params: [ + { + fromBlock: '0x' + fromBlock.toString(16), + toBlock: 'latest', + topics: [[LogTopic.TRANSFER, LogTopic.DEPOSIT, LogTopic.WITHDRAWAL], [hexZeroPad(address, 32)]] + } + ], + chainId: addHexPrefix(chainId.toString(16)) + }), + eth.request({ + method: 'eth_getLogs', + params: [ + { + fromBlock: '0x' + fromBlock.toString(16), + toBlock: 'latest', + topics: [[LogTopic.TRANSFER], [], [hexZeroPad(address, 32)]] + } + ], + chainId: addHexPrefix(chainId.toString(16)) + }) + ])) as [Log[], Log[]] + return logs.flat() } function balanceCalls(owner: string, tokens: TokenDefinition[]): Call[] { @@ -122,11 +126,7 @@ export default function (eth: EthereumProvider) { return result.balance.toHexString() } - async function getTokenBalancesFromContracts( - owner: string, - tokens: TokenDefinition[], - latestBlock: number - ) { + async function getTokenBalancesFromContracts(owner: string, tokens: TokenDefinition[]) { const balances = tokens.map(async (token) => { try { const rawBalance = await getTokenBalance(token, owner) @@ -147,12 +147,7 @@ export default function (eth: EthereumProvider) { return loadedBalances.filter((bal) => bal !== undefined) as Balance[] } - async function getTokenBalancesFromMulticall( - owner: string, - tokens: TokenDefinition[], - chainId: number, - latestBlock: number - ) { + async function getTokenBalancesFromMulticall(owner: string, tokens: TokenDefinition[], chainId: number) { const calls = balanceCalls(owner, tokens) const results = await multicall(chainId, eth).batchCall(calls) @@ -184,21 +179,23 @@ export default function (eth: EthereumProvider) { Object.entries(tokensByChain).map(async ([chain, tokens]) => { const chainId = parseInt(chain) const latestBlock = await getLatestBlock(chainId) - const logProcessor = logProcessors[owner] + const logProcessorKey = toLogProcessorKey(owner, chainId) + const logProcessor = logProcessors[logProcessorKey] if (logProcessor) { - const logs = await getTransferLogs(owner, chainId, logProcessor.lastProcessedBlock) - return logProcessor.process(chainId, logs, latestBlock) - } else { - log.info('not seeded... will be soon though!') + try { + const logs = await getTransferLogs(owner, chainId, logProcessor.lastProcessedBlock) + return logProcessor.process(logs, latestBlock, tokens) + } catch (error) { + log.warn('Unable to update balances using eth_getLogs', { chainId }) + } } const balances = multicallSupportsChain(chainId) - ? await getTokenBalancesFromMulticall(owner, tokens, chainId, latestBlock) - : await getTokenBalancesFromContracts(owner, tokens, latestBlock) + ? await getTokenBalancesFromMulticall(owner, tokens, chainId) + : await getTokenBalancesFromContracts(owner, tokens) + + logProcessors[logProcessorKey] = new LogProcessor(owner, balances, latestBlock, chainId, eth) - if (!logProcessor) { - logProcessors[owner] = new LogProcessor(owner, balances, latestBlock) - } return balances }) ) From 675ba052dbde11461815114e3bde98b219ef93ac Mon Sep 17 00:00:00 2001 From: Jamie M Date: Wed, 1 Mar 2023 15:45:32 +0000 Subject: [PATCH 3/3] remove lastProcessed dict --- main/externalData/balances/logs/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/main/externalData/balances/logs/index.ts b/main/externalData/balances/logs/index.ts index 62ccae71f..37e18d935 100644 --- a/main/externalData/balances/logs/index.ts +++ b/main/externalData/balances/logs/index.ts @@ -42,9 +42,6 @@ export class LogProcessor { private balances: AccountBalances = {} private ownerPadded - // Map of chainId => last processed - public lastProcessed: Record = {} - private handlers: Record void> = { [LogTopic.TRANSFER]: this.handleTransfer.bind(this), [LogTopic.WITHDRAWAL]: this.handleWithdrawal.bind(this),