From 8a0da45d09c805499adbd1aebd4afc374537b542 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 27 Mar 2026 00:53:05 +0000 Subject: [PATCH 1/3] fix(explorer): fix token holder count for high-volume tokens like PathUSD Replace GROUP BY (from, to) with two separate queries grouped by individual holder (GROUP BY to for received, GROUP BY from for sent). This reduces row count from O(unique_transfer_pairs) to O(unique_holders), avoiding truncation by tidx's 10k row hard limit. Co-authored-by: o-az <23618431+o-az@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d2c9a-d957-714c-add1-f823c928ddfe --- apps/explorer/src/lib/server/tempo-queries.ts | 155 +++++++++++------- apps/explorer/test/tempo-queries.test.ts | 35 ++-- 2 files changed, 108 insertions(+), 82 deletions(-) diff --git a/apps/explorer/src/lib/server/tempo-queries.ts b/apps/explorer/src/lib/server/tempo-queries.ts index 3c4e034c9..194d8a5bf 100644 --- a/apps/explorer/src/lib/server/tempo-queries.ts +++ b/apps/explorer/src/lib/server/tempo-queries.ts @@ -19,9 +19,8 @@ type QueryWithWhere = TQuery & { export type TokenHolderBalance = { address: string; balance: bigint } -type TokenHolderAggregationRow = { - from: string - to: string +type HolderAggregationRow = { + holder: string tokens: string | number | bigint } @@ -40,41 +39,53 @@ function sortTokenHolderBalances( .sort((a, b) => (b.balance > a.balance ? 1 : -1)) } -function aggregateTokenHolderBalances( - rows: TokenHolderAggregationRow[], -): TokenHolderBalance[] { - const balances = new Map() - - for (const row of rows) { - const tokens = BigInt(row.tokens) - if (row.to !== zeroAddress) { - balances.set(row.to, (balances.get(row.to) ?? 0n) + tokens) - } - if (row.from !== zeroAddress) { - balances.set(row.from, (balances.get(row.from) ?? 0n) - tokens) - } - } - - return sortTokenHolderBalances(balances) -} - +/** + * Fetches holder balances using two separate queries (received / sent) grouped + * by individual holder address instead of the (from, to) pair. This keeps the + * row count proportional to unique holders rather than unique transfer pairs, + * avoiding truncation by tidx's 10 000-row hard limit on high-volume tokens + * like PathUSD. + */ export async function fetchTokenHolderBalances( address: Address.Address, chainId: number, ): Promise { const qb = QB(chainId).withSignatures([TRANSFER_SIGNATURE]) - const transfers = (await qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('from').as('from'), - eb.ref('to').as('to'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', '=', address) - .groupBy(['from', 'to']) - .execute()) as TokenHolderAggregationRow[] - return aggregateTokenHolderBalances(transfers) + const [receivedRows, sentRows] = await Promise.all([ + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('to').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', '=', address) + .where('to', '!=', zeroAddress) + .groupBy(['to']) + .execute() as Promise, + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('from').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', '=', address) + .where('from', '!=', zeroAddress) + .groupBy(['from']) + .execute() as Promise, + ]) + + const balances = new Map() + for (const row of receivedRows) { + const holder = row.holder.toLowerCase() + balances.set(holder, (balances.get(holder) ?? 0n) + BigInt(row.tokens)) + } + for (const row of sentRows) { + const holder = row.holder.toLowerCase() + balances.set(holder, (balances.get(holder) ?? 0n) - BigInt(row.tokens)) + } + + return sortTokenHolderBalances(balances) } export async function fetchTokenHoldersCountRows( @@ -85,42 +96,72 @@ export async function fetchTokenHoldersCountRows( if (addresses.length === 0) return [] const qb = QB(chainId).withSignatures([TRANSFER_SIGNATURE]) - const transfers = (await qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('address').as('address'), - eb.ref('from').as('from'), - eb.ref('to').as('to'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', 'in', addresses) - .groupBy(['address', 'from', 'to']) - .execute()) as Array<{ - address: string - from: string - to: string - tokens: string | number | bigint - }> + + const [receivedRows, sentRows] = await Promise.all([ + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('address').as('address'), + eb.ref('to').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', 'in', addresses) + .where('to', '!=', zeroAddress) + .groupBy(['address', 'to']) + .execute() as Promise< + Array<{ + address: string + holder: string + tokens: string | number | bigint + }> + >, + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('address').as('address'), + eb.ref('from').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', 'in', addresses) + .where('from', '!=', zeroAddress) + .groupBy(['address', 'from']) + .execute() as Promise< + Array<{ + address: string + holder: string + tokens: string | number | bigint + }> + >, + ]) const balancesByToken = new Map>() - for (const row of transfers) { + for (const row of receivedRows) { const token = row.address.toLowerCase() let tokenBalances = balancesByToken.get(token) if (!tokenBalances) { tokenBalances = new Map() balancesByToken.set(token, tokenBalances) } + const holder = row.holder.toLowerCase() + tokenBalances.set( + holder, + (tokenBalances.get(holder) ?? 0n) + BigInt(row.tokens), + ) + } - const tokens = BigInt(row.tokens) - if (row.to !== zeroAddress) { - const to = row.to.toLowerCase() - tokenBalances.set(to, (tokenBalances.get(to) ?? 0n) + tokens) - } - if (row.from !== zeroAddress) { - const from = row.from.toLowerCase() - tokenBalances.set(from, (tokenBalances.get(from) ?? 0n) - tokens) + for (const row of sentRows) { + const token = row.address.toLowerCase() + let tokenBalances = balancesByToken.get(token) + if (!tokenBalances) { + tokenBalances = new Map() + balancesByToken.set(token, tokenBalances) } + const holder = row.holder.toLowerCase() + tokenBalances.set( + holder, + (tokenBalances.get(holder) ?? 0n) - BigInt(row.tokens), + ) } return addresses.map((address) => { diff --git a/apps/explorer/test/tempo-queries.test.ts b/apps/explorer/test/tempo-queries.test.ts index 27af034fc..dee615bb8 100644 --- a/apps/explorer/test/tempo-queries.test.ts +++ b/apps/explorer/test/tempo-queries.test.ts @@ -330,18 +330,13 @@ describe('tempo-queries', () => { it('fetchTokenHolderBalances aggregates holders from raw transfer rows', async () => { mockQueryBuilder.setResponses([ + // received query (GROUP BY to) [ - { - from: '0x0000000000000000000000000000000000000000', - to: '0xaaaa', - tokens: '10', - }, - { - from: '0xaaaa', - to: '0xbbbb', - tokens: '4', - }, + { holder: '0xaaaa', tokens: '10' }, + { holder: '0xbbbb', tokens: '4' }, ], + // sent query (GROUP BY from) + [{ holder: '0xaaaa', tokens: '4' }], ]) await expect( @@ -354,23 +349,13 @@ describe('tempo-queries', () => { it('fetchTokenHolderBalances aggregates incoming and outgoing balances', async () => { mockQueryBuilder.setResponses([ + // received query (GROUP BY to) [ - { - from: '0x1111', - to: '0x0000000000000000000000000000000000000000', - tokens: '5', - }, - { - from: '0x0000000000000000000000000000000000000000', - to: '0x1111', - tokens: '10', - }, - { - from: '0x0000000000000000000000000000000000000000', - to: '0x2222', - tokens: '3', - }, + { holder: '0x1111', tokens: '10' }, + { holder: '0x2222', tokens: '3' }, ], + // sent query (GROUP BY from) + [{ holder: '0x1111', tokens: '5' }], ]) const balances = await fetchTokenHolderBalances( From f568d4d48a3f3e57141b87337b69a31bd3d0952f Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 27 Mar 2026 01:02:04 +0000 Subject: [PATCH 2/3] fix(explorer): add pagination to handle tokens with >10k unique holders PathUSD has more unique senders than tidx's 10k row limit, so even with the GROUP BY holder fix, results were still truncated. Add a fetchAllPages helper that paginates through the limit using LIMIT/OFFSET. Co-authored-by: o-az <23618431+o-az@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d2c9a-d957-714c-add1-f823c928ddfe --- apps/explorer/src/lib/server/tempo-queries.ts | 151 +++++++++++------- 1 file changed, 94 insertions(+), 57 deletions(-) diff --git a/apps/explorer/src/lib/server/tempo-queries.ts b/apps/explorer/src/lib/server/tempo-queries.ts index 194d8a5bf..219801c2e 100644 --- a/apps/explorer/src/lib/server/tempo-queries.ts +++ b/apps/explorer/src/lib/server/tempo-queries.ts @@ -39,12 +39,35 @@ function sortTokenHolderBalances( .sort((a, b) => (b.balance > a.balance ? 1 : -1)) } +/** tidx hard-caps query results at 10 000 rows. */ +const TIDX_PAGE_SIZE = 10_000 + +/** + * Paginate a tidx query that may exceed the 10 000-row hard limit. + * Fetches pages of `TIDX_PAGE_SIZE` until a page returns fewer rows. + */ +async function fetchAllPages( + buildQuery: (limit: number, offset: number) => Promise, +): Promise { + const all: T[] = [] + let offset = 0 + // biome-ignore lint/correctness/noConstantCondition: pagination loop + while (true) { + const page = await buildQuery(TIDX_PAGE_SIZE, offset) + all.push(...page) + if (page.length < TIDX_PAGE_SIZE) break + offset += TIDX_PAGE_SIZE + } + return all +} + /** * Fetches holder balances using two separate queries (received / sent) grouped * by individual holder address instead of the (from, to) pair. This keeps the - * row count proportional to unique holders rather than unique transfer pairs, - * avoiding truncation by tidx's 10 000-row hard limit on high-volume tokens - * like PathUSD. + * row count proportional to unique holders rather than unique transfer pairs. + * + * Paginates through tidx's 10 000-row hard limit so high-volume tokens like + * PathUSD (which has more unique senders than the limit) are counted correctly. */ export async function fetchTokenHolderBalances( address: Address.Address, @@ -53,26 +76,36 @@ export async function fetchTokenHolderBalances( const qb = QB(chainId).withSignatures([TRANSFER_SIGNATURE]) const [receivedRows, sentRows] = await Promise.all([ - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('to').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', '=', address) - .where('to', '!=', zeroAddress) - .groupBy(['to']) - .execute() as Promise, - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('from').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', '=', address) - .where('from', '!=', zeroAddress) - .groupBy(['from']) - .execute() as Promise, + fetchAllPages( + (limit, offset) => + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('to').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', '=', address) + .where('to', '!=', zeroAddress) + .groupBy(['to']) + .limit(limit) + .offset(offset) + .execute() as Promise, + ), + fetchAllPages( + (limit, offset) => + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('from').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', '=', address) + .where('from', '!=', zeroAddress) + .groupBy(['from']) + .limit(limit) + .offset(offset) + .execute() as Promise, + ), ]) const balances = new Map() @@ -97,41 +130,45 @@ export async function fetchTokenHoldersCountRows( const qb = QB(chainId).withSignatures([TRANSFER_SIGNATURE]) + type BatchRow = { + address: string + holder: string + tokens: string | number | bigint + } + const [receivedRows, sentRows] = await Promise.all([ - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('address').as('address'), - eb.ref('to').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', 'in', addresses) - .where('to', '!=', zeroAddress) - .groupBy(['address', 'to']) - .execute() as Promise< - Array<{ - address: string - holder: string - tokens: string | number | bigint - }> - >, - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('address').as('address'), - eb.ref('from').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', 'in', addresses) - .where('from', '!=', zeroAddress) - .groupBy(['address', 'from']) - .execute() as Promise< - Array<{ - address: string - holder: string - tokens: string | number | bigint - }> - >, + fetchAllPages( + (limit, offset) => + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('address').as('address'), + eb.ref('to').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', 'in', addresses) + .where('to', '!=', zeroAddress) + .groupBy(['address', 'to']) + .limit(limit) + .offset(offset) + .execute() as Promise, + ), + fetchAllPages( + (limit, offset) => + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('address').as('address'), + eb.ref('from').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', 'in', addresses) + .where('from', '!=', zeroAddress) + .groupBy(['address', 'from']) + .limit(limit) + .offset(offset) + .execute() as Promise, + ), ]) const balancesByToken = new Map>() From eaf2d02af9773fed5db7fca769542abaa5820693 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 27 Mar 2026 01:14:34 +0000 Subject: [PATCH 3/3] fix(explorer): revert pagination - LIMIT/OFFSET breaks tidx CTE queries The LIMIT/OFFSET approach causes tidx to return empty results for all tokens, likely due to interaction with the CTE-based query rewriting. The GROUP BY holder fix alone already fixes most tokens (USDC.e went from 3,168 to 7,943 holders). PathUSD still needs a tidx-side fix for tokens with >10k unique senders. Co-authored-by: o-az <23618431+o-az@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019d2c9a-d957-714c-add1-f823c928ddfe --- apps/explorer/src/lib/server/tempo-queries.ts | 150 +++++++----------- 1 file changed, 56 insertions(+), 94 deletions(-) diff --git a/apps/explorer/src/lib/server/tempo-queries.ts b/apps/explorer/src/lib/server/tempo-queries.ts index 219801c2e..8c74d086e 100644 --- a/apps/explorer/src/lib/server/tempo-queries.ts +++ b/apps/explorer/src/lib/server/tempo-queries.ts @@ -39,35 +39,11 @@ function sortTokenHolderBalances( .sort((a, b) => (b.balance > a.balance ? 1 : -1)) } -/** tidx hard-caps query results at 10 000 rows. */ -const TIDX_PAGE_SIZE = 10_000 - -/** - * Paginate a tidx query that may exceed the 10 000-row hard limit. - * Fetches pages of `TIDX_PAGE_SIZE` until a page returns fewer rows. - */ -async function fetchAllPages( - buildQuery: (limit: number, offset: number) => Promise, -): Promise { - const all: T[] = [] - let offset = 0 - // biome-ignore lint/correctness/noConstantCondition: pagination loop - while (true) { - const page = await buildQuery(TIDX_PAGE_SIZE, offset) - all.push(...page) - if (page.length < TIDX_PAGE_SIZE) break - offset += TIDX_PAGE_SIZE - } - return all -} - /** * Fetches holder balances using two separate queries (received / sent) grouped * by individual holder address instead of the (from, to) pair. This keeps the - * row count proportional to unique holders rather than unique transfer pairs. - * - * Paginates through tidx's 10 000-row hard limit so high-volume tokens like - * PathUSD (which has more unique senders than the limit) are counted correctly. + * row count proportional to unique holders rather than unique transfer pairs, + * avoiding truncation by tidx's 10 000-row hard limit on high-volume tokens. */ export async function fetchTokenHolderBalances( address: Address.Address, @@ -76,36 +52,26 @@ export async function fetchTokenHolderBalances( const qb = QB(chainId).withSignatures([TRANSFER_SIGNATURE]) const [receivedRows, sentRows] = await Promise.all([ - fetchAllPages( - (limit, offset) => - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('to').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', '=', address) - .where('to', '!=', zeroAddress) - .groupBy(['to']) - .limit(limit) - .offset(offset) - .execute() as Promise, - ), - fetchAllPages( - (limit, offset) => - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('from').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', '=', address) - .where('from', '!=', zeroAddress) - .groupBy(['from']) - .limit(limit) - .offset(offset) - .execute() as Promise, - ), + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('to').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', '=', address) + .where('to', '!=', zeroAddress) + .groupBy(['to']) + .execute() as Promise, + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('from').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', '=', address) + .where('from', '!=', zeroAddress) + .groupBy(['from']) + .execute() as Promise, ]) const balances = new Map() @@ -130,45 +96,41 @@ export async function fetchTokenHoldersCountRows( const qb = QB(chainId).withSignatures([TRANSFER_SIGNATURE]) - type BatchRow = { - address: string - holder: string - tokens: string | number | bigint - } - const [receivedRows, sentRows] = await Promise.all([ - fetchAllPages( - (limit, offset) => - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('address').as('address'), - eb.ref('to').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', 'in', addresses) - .where('to', '!=', zeroAddress) - .groupBy(['address', 'to']) - .limit(limit) - .offset(offset) - .execute() as Promise, - ), - fetchAllPages( - (limit, offset) => - qb - .selectFrom('transfer') - .select((eb) => [ - eb.ref('address').as('address'), - eb.ref('from').as('holder'), - eb.fn.sum('tokens').as('tokens'), - ]) - .where('address', 'in', addresses) - .where('from', '!=', zeroAddress) - .groupBy(['address', 'from']) - .limit(limit) - .offset(offset) - .execute() as Promise, - ), + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('address').as('address'), + eb.ref('to').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', 'in', addresses) + .where('to', '!=', zeroAddress) + .groupBy(['address', 'to']) + .execute() as Promise< + Array<{ + address: string + holder: string + tokens: string | number | bigint + }> + >, + qb + .selectFrom('transfer') + .select((eb) => [ + eb.ref('address').as('address'), + eb.ref('from').as('holder'), + eb.fn.sum('tokens').as('tokens'), + ]) + .where('address', 'in', addresses) + .where('from', '!=', zeroAddress) + .groupBy(['address', 'from']) + .execute() as Promise< + Array<{ + address: string + holder: string + tokens: string | number | bigint + }> + >, ]) const balancesByToken = new Map>()