From 31e62e4578e00488cd2624697afe61d83d7d6c06 Mon Sep 17 00:00:00 2001 From: Joe Fitter Date: Wed, 19 Nov 2025 01:53:17 +0100 Subject: [PATCH 1/3] Feature - add helpers to optimise fetching and parsing multiple metadata accounts I am working on an app which parses many transactions using codama and as part of this I need to fetch IDLs for all programs in all instructions. As the only published method for fetching and parsing program-metadata content only supports single calls this results in many getAccountInfo calls which can exceed RPC rate limits for some suppliers. The addition of fetchAndParseAllMetadataContent which uses getMultipleAccounts to prefetch all PDA accounts, and additionally fetches all external accounts in a total of 2 RPC calls instead of potentially hundreds. --- clients/js/src/fetchMetadataContent.ts | 71 ++++++++++++- clients/js/src/packData.ts | 105 ++++++++++++++++--- clients/js/test/fetchMetadataContent.test.ts | 87 ++++++++++++++- 3 files changed, 242 insertions(+), 21 deletions(-) diff --git a/clients/js/src/fetchMetadataContent.ts b/clients/js/src/fetchMetadataContent.ts index fbc7e21..69e0b39 100644 --- a/clients/js/src/fetchMetadataContent.ts +++ b/clients/js/src/fetchMetadataContent.ts @@ -1,8 +1,20 @@ import { parse as parseToml } from '@iarna/toml'; -import { Address, GetAccountInfoApi, Rpc } from '@solana/kit'; +import { + Address, + assertAccountsExist, + GetAccountInfoApi, + GetMultipleAccountsApi, + Rpc, +} from '@solana/kit'; import { parse as parseYaml } from 'yaml'; -import { fetchMetadataFromSeeds, Format, SeedArgs } from './generated'; -import { unpackAndFetchData } from './packData'; +import { + fetchAllMaybeMetadata, + fetchMetadataFromSeeds, + findMetadataPda, + Format, + SeedArgs, +} from './generated'; +import { unpackAndFetchAllData, unpackAndFetchData } from './packData'; export async function fetchMetadataContent( rpc: Rpc, @@ -18,6 +30,28 @@ export async function fetchMetadataContent( return await unpackAndFetchData({ rpc, ...account.data }); } +type FetchAllMetadataContentInput = { + program: Address; + seed: SeedArgs; + authority: Address | null; +}[]; + +export async function fetchAllMetadataContent( + rpc: Rpc, + input: FetchAllMetadataContentInput +): Promise { + const addresses = await Promise.all( + input.map(async ({ program, authority, seed }) => { + const [address] = await findMetadataPda({ program, authority, seed }); + + return address; + }) + ); + const accounts = await fetchAllMaybeMetadata(rpc, addresses); + assertAccountsExist(accounts); + return await unpackAndFetchAllData({ rpc, accounts }); +} + export async function fetchAndParseMetadataContent( rpc: Rpc, program: Address, @@ -42,3 +76,34 @@ export async function fetchAndParseMetadataContent( return content; } } + +type FetchAndParseAllMetadataContentInput = FetchAllMetadataContentInput; + +export async function fetchAndParseAllMetadataContent( + rpc: Rpc, + input: FetchAndParseAllMetadataContentInput +): Promise { + const addresses = await Promise.all( + input.map(async ({ program, authority, seed }) => { + const [address] = await findMetadataPda({ program, authority, seed }); + + return address; + }) + ); + const accounts = await fetchAllMaybeMetadata(rpc, addresses); + assertAccountsExist(accounts); + const unpacked = await unpackAndFetchAllData({ rpc, accounts }); + return unpacked.map((content, index) => { + switch (accounts[index].data.format) { + case Format.Json: + return JSON.parse(content); + case Format.Yaml: + return parseYaml(content); + case Format.Toml: + return parseToml(content); + case Format.None: + default: + return content; + } + }); +} diff --git a/clients/js/src/packData.ts b/clients/js/src/packData.ts index fd1bb9b..05d2e9d 100644 --- a/clients/js/src/packData.ts +++ b/clients/js/src/packData.ts @@ -1,7 +1,10 @@ import { + Account, Address, assertAccountExists, + assertAccountsExist, fetchEncodedAccount, + fetchEncodedAccounts, GetAccountInfoApi, getBase16Decoder, getBase16Encoder, @@ -9,6 +12,7 @@ import { getBase58Encoder, getBase64Decoder, getBase64Encoder, + GetMultipleAccountsApi, getUtf8Decoder, getUtf8Encoder, pipe, @@ -23,6 +27,7 @@ import { Encoding, getExternalDataDecoder, getExternalDataEncoder, + Metadata, } from './generated'; export type PackedData = { @@ -32,6 +37,12 @@ export type PackedData = { data: ReadonlyUint8Array; }; +export type UnpackedData = { + address: Address; + offset?: number; + length?: number; +}; + export function packDirectData(input: { content: string; /** Defaults to `Compression.Zlib`. */ @@ -120,11 +131,7 @@ export async function unpackAndFetchUrlData( return await response.text(); } -export function unpackExternalData(data: ReadonlyUint8Array): { - address: Address; - offset?: number; - length?: number; -} { +export function unpackExternalData(data: ReadonlyUint8Array): UnpackedData { const externalData = getExternalDataDecoder().decode(data); return { address: externalData.address, @@ -133,27 +140,47 @@ export function unpackExternalData(data: ReadonlyUint8Array): { }; } -export async function unpackAndFetchExternalData( - input: Omit & { rpc: Rpc } -): Promise { - const externalData = unpackExternalData(input.data); - const account = await fetchEncodedAccount(input.rpc, externalData.address); - assertAccountExists(account); +export function uncompressAndDecodeExternalData({ + compression, + encoding, + account, + unpackedExternalData, +}: Pick & { + account: Account; + unpackedExternalData: UnpackedData; +}): string { let data = account.data; - if (externalData.offset !== undefined) { - data = data.slice(externalData.offset); + if (unpackedExternalData.offset !== undefined) { + data = data.slice(unpackedExternalData.offset); } - if (externalData.length !== undefined) { - data = data.slice(0, externalData.length); + if (unpackedExternalData.length !== undefined) { + data = data.slice(0, unpackedExternalData.length); } if (data.length === 0) { return ''; } return pipe( data, - (d) => uncompressData(d, input.compression), - (d) => decodeData(d, input.encoding) + (d) => uncompressData(d, compression), + (d) => decodeData(d, encoding) + ); +} + +export async function unpackAndFetchExternalData( + input: Omit & { rpc: Rpc } +): Promise { + const unpackedExternalData = unpackExternalData(input.data); + const account = await fetchEncodedAccount( + input.rpc, + unpackedExternalData.address ); + assertAccountExists(account); + return uncompressAndDecodeExternalData({ + compression: input.compression, + encoding: input.encoding, + account, + unpackedExternalData, + }); } export async function unpackAndFetchData( @@ -171,6 +198,50 @@ export async function unpackAndFetchData( } } +export async function unpackAndFetchAllData({ + accounts, + rpc, +}: { + accounts: Account[]; + rpc: Rpc; +}) { + const unpackedExternalAccounts = accounts + .filter((account) => account.data.dataSource === DataSource.External) + .map((account) => ({ + address: account.address, + unpacked: unpackExternalData(account.data.data), + })); + + const fetchedExternalAccounts = await fetchEncodedAccounts( + rpc, + unpackedExternalAccounts.map((account) => account.unpacked.address) + ); + assertAccountsExist(fetchedExternalAccounts); + + return Promise.all( + accounts.map(async (account) => { + switch (account.data.dataSource) { + case DataSource.Direct: + return unpackDirectData(account.data); + case DataSource.Url: + return await unpackAndFetchUrlData(account.data); + case DataSource.External: { + const accountIndex = unpackedExternalAccounts.findIndex( + (acc) => acc.address === account.address + ); + return uncompressAndDecodeExternalData({ + compression: account.data.compression, + encoding: account.data.encoding, + account: fetchedExternalAccounts[accountIndex], + unpackedExternalData: + unpackedExternalAccounts[accountIndex].unpacked, + }); + } + } + }) + ); +} + export function compressData( data: ReadonlyUint8Array, compression: Compression diff --git a/clients/js/test/fetchMetadataContent.test.ts b/clients/js/test/fetchMetadataContent.test.ts index 3f36480..b954d62 100644 --- a/clients/js/test/fetchMetadataContent.test.ts +++ b/clients/js/test/fetchMetadataContent.test.ts @@ -1,12 +1,17 @@ -import { address } from '@solana/kit'; +import { address, generateKeyPairSigner, getUtf8Encoder } from '@solana/kit'; import test from 'ava'; import { + Compression, + Encoding, + fetchAndParseAllMetadataContent, fetchAndParseMetadataContent, Format, packDirectData, + packExternalData, writeMetadata, } from '../src'; import { + createBuffer, createDefaultSolanaClient, createDeployedProgram, generateKeyPairSignerWithSol, @@ -74,3 +79,83 @@ test('it fetches and parses direct IDLs from non-canonical metadata accounts', a version: '1.0.0', }); }); + +test('it fetches and parses multiple direct IDLs from metadata accounts', async (t) => { + t.timeout(30_000); + // Given the following authority and deployed program. + const client = createDefaultSolanaClient(); + const authority = await generateKeyPairSignerWithSol(client); + const program = address('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA'); + + const metadata1 = await generateKeyPairSigner(); + const metadata2 = await generateKeyPairSigner(); + + // And given the following IDLs exist for the programs. + const idl1 = '{"kind":"rootNode","standard":"codama","version":"1.0.0"}'; + const idl2 = '{"kind":"rootNode","standard":"codama","version":"1.0.1"}'; + const buffer = await generateKeyPairSigner(); + + // We create a buffer account to hold the IDL data + await createBuffer(client, { + buffer: buffer.address, + authority: buffer, + payer: authority, + data: getUtf8Encoder().encode(idl2), + }); + + // And we create metadata accounts for direct and external data + await Promise.all([ + writeMetadata({ + ...client, + ...packDirectData({ content: idl1 }), + payer: authority, + authority: metadata1, + program, + seed: 'idl', + format: Format.Json, + }), + writeMetadata({ + ...client, + ...packExternalData({ + address: buffer.address, + offset: 96, + length: idl2.length, + compression: Compression.None, + encoding: Encoding.Utf8, + }), + payer: authority, + authority: metadata2, + program, + seed: 'idl', + format: Format.Json, + }), + ]); + + // When we fetch the IDLs for the programs. + const result = await fetchAndParseAllMetadataContent(client.rpc, [ + { + program, + seed: 'idl', + authority: metadata1.address, + }, + { + program, + seed: 'idl', + authority: metadata2.address, + }, + ]); + + // Then we expect the following IDLs to be fetched and parsed. + t.deepEqual(result, [ + { + kind: 'rootNode', + standard: 'codama', + version: '1.0.0', + }, + { + kind: 'rootNode', + standard: 'codama', + version: '1.0.1', + }, + ]); +}); From 599c1b8f3dcc0f13fe152c14373336aebe9f737e Mon Sep 17 00:00:00 2001 From: Joe Fitter Date: Wed, 19 Nov 2025 12:14:23 +0100 Subject: [PATCH 2/3] dont assert all accounts exist, we may want to call for a batch of programs, not all will have metadata --- clients/js/src/fetchMetadataContent.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/js/src/fetchMetadataContent.ts b/clients/js/src/fetchMetadataContent.ts index 69e0b39..b63e71d 100644 --- a/clients/js/src/fetchMetadataContent.ts +++ b/clients/js/src/fetchMetadataContent.ts @@ -90,8 +90,8 @@ export async function fetchAndParseAllMetadataContent( return address; }) ); - const accounts = await fetchAllMaybeMetadata(rpc, addresses); - assertAccountsExist(accounts); + const maybeAccounts = await fetchAllMaybeMetadata(rpc, addresses); + const accounts = maybeAccounts.filter((acc) => acc.exists); const unpacked = await unpackAndFetchAllData({ rpc, accounts }); return unpacked.map((content, index) => { switch (accounts[index].data.format) { From dc9bd1d80ac84f6a357a70bcbbe9ef87bd5c57a1 Mon Sep 17 00:00:00 2001 From: Joe Fitter Date: Thu, 20 Nov 2025 15:51:35 +0100 Subject: [PATCH 3/3] Fixed lint and addressed points raised in PR --- clients/js/src/fetchMetadataContent.ts | 38 +++++++++++--------------- clients/js/src/packData.ts | 16 ++++++----- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/clients/js/src/fetchMetadataContent.ts b/clients/js/src/fetchMetadataContent.ts index b63e71d..0b72c71 100644 --- a/clients/js/src/fetchMetadataContent.ts +++ b/clients/js/src/fetchMetadataContent.ts @@ -52,6 +52,20 @@ export async function fetchAllMetadataContent( return await unpackAndFetchAllData({ rpc, accounts }); } +function parseContent(format: Format, content: string) { + switch (format) { + case Format.Json: + return JSON.parse(content); + case Format.Yaml: + return parseYaml(content); + case Format.Toml: + return parseToml(content); + case Format.None: + default: + return content; + } +} + export async function fetchAndParseMetadataContent( rpc: Rpc, program: Address, @@ -64,17 +78,7 @@ export async function fetchAndParseMetadataContent( seed, }); const content = await unpackAndFetchData({ rpc, ...account.data }); - switch (account.data.format) { - case Format.Json: - return JSON.parse(content); - case Format.Yaml: - return parseYaml(content); - case Format.Toml: - return parseToml(content); - case Format.None: - default: - return content; - } + return parseContent(account.data.format, content); } type FetchAndParseAllMetadataContentInput = FetchAllMetadataContentInput; @@ -94,16 +98,6 @@ export async function fetchAndParseAllMetadataContent( const accounts = maybeAccounts.filter((acc) => acc.exists); const unpacked = await unpackAndFetchAllData({ rpc, accounts }); return unpacked.map((content, index) => { - switch (accounts[index].data.format) { - case Format.Json: - return JSON.parse(content); - case Format.Yaml: - return parseYaml(content); - case Format.Toml: - return parseToml(content); - case Format.None: - default: - return content; - } + return parseContent(accounts[index].data.format, content); }); } diff --git a/clients/js/src/packData.ts b/clients/js/src/packData.ts index 05d2e9d..b101c08 100644 --- a/clients/js/src/packData.ts +++ b/clients/js/src/packData.ts @@ -37,7 +37,7 @@ export type PackedData = { data: ReadonlyUint8Array; }; -export type UnpackedData = { +type UnpackedExternalData = { address: Address; offset?: number; length?: number; @@ -131,7 +131,9 @@ export async function unpackAndFetchUrlData( return await response.text(); } -export function unpackExternalData(data: ReadonlyUint8Array): UnpackedData { +export function unpackExternalData( + data: ReadonlyUint8Array +): UnpackedExternalData { const externalData = getExternalDataDecoder().decode(data); return { address: externalData.address, @@ -140,14 +142,14 @@ export function unpackExternalData(data: ReadonlyUint8Array): UnpackedData { }; } -export function uncompressAndDecodeExternalData({ +export function unpackFetchedExternalData({ compression, encoding, account, unpackedExternalData, }: Pick & { account: Account; - unpackedExternalData: UnpackedData; + unpackedExternalData: UnpackedExternalData; }): string { let data = account.data; if (unpackedExternalData.offset !== undefined) { @@ -175,7 +177,7 @@ export async function unpackAndFetchExternalData( unpackedExternalData.address ); assertAccountExists(account); - return uncompressAndDecodeExternalData({ + return unpackFetchedExternalData({ compression: input.compression, encoding: input.encoding, account, @@ -218,7 +220,7 @@ export async function unpackAndFetchAllData({ ); assertAccountsExist(fetchedExternalAccounts); - return Promise.all( + return await Promise.all( accounts.map(async (account) => { switch (account.data.dataSource) { case DataSource.Direct: @@ -229,7 +231,7 @@ export async function unpackAndFetchAllData({ const accountIndex = unpackedExternalAccounts.findIndex( (acc) => acc.address === account.address ); - return uncompressAndDecodeExternalData({ + return unpackFetchedExternalData({ compression: account.data.compression, encoding: account.data.encoding, account: fetchedExternalAccounts[accountIndex],