diff --git a/bun.lock b/bun.lock index aa17d5b0..ad0c8e99 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "kong", @@ -136,9 +137,9 @@ "version": "0.1.0", "dependencies": { "@apollo/server": "^4.9.5", - "@apollo/utils.keyvadapter": "^3.1.0", + "@apollo/utils.keyvadapter": "4.0.1", "@as-integrations/next": "^3.0.0", - "@keyv/redis": "^4.4.1", + "@keyv/redis": "5.1.6", "autoprefixer": "10.4.15", "bullmq": "^5.21.2", "chart.js": "^4.4.0", @@ -203,7 +204,7 @@ "@apollo/utils.isnodelike": ["@apollo/utils.isnodelike@2.0.1", "", {}, "sha512-w41XyepR+jBEuVpoRM715N2ZD0xMD413UiJx8w5xnAZD2ZkSJnMJBoIzauK83kJpSgNuR6ywbV29jG9NmxjK0Q=="], - "@apollo/utils.keyvadapter": ["@apollo/utils.keyvadapter@3.1.0", "", { "dependencies": { "@apollo/utils.keyvaluecache": "^3.1.0", "dataloader": "^2.1.0", "keyv": "^4.4.0" } }, "sha512-q41MxH2gKwvXL28hUEfQHdSQ0F4u8KrPHUbvN/IkA7vh+KTRj0tG2eWLu42c438jjoV7hUhY1Ru/Dwq8zndEqg=="], + "@apollo/utils.keyvadapter": ["@apollo/utils.keyvadapter@4.0.1", "", { "dependencies": { "@apollo/utils.keyvaluecache": "^4.0.0", "dataloader": "^2.1.0", "keyv": "^5.1.0" } }, "sha512-nCsAM/uNi17sta6f98FOBiQc8qwx3As78LfUGmHWzaKnwpolzeV8Phz04W723CSaDGxeo/gxEkkQiK0oAdvGxw=="], "@apollo/utils.keyvaluecache": ["@apollo/utils.keyvaluecache@2.1.1", "", { "dependencies": { "@apollo/utils.logger": "^2.0.1", "lru-cache": "^7.14.1" } }, "sha512-qVo5PvUUMD8oB9oYvq4ViCjYAMWnZ5zZwEjNF37L2m1u528x5mueMlU+Cr1UinupCgdB78g+egA1G98rbJ03Vw=="], @@ -437,7 +438,7 @@ "@json-rpc-tools/utils": ["@json-rpc-tools/utils@1.7.6", "", { "dependencies": { "@json-rpc-tools/types": "^1.7.6", "@pedrouid/environment": "^1.0.1" } }, "sha512-HjA8x/U/Q78HRRe19yh8HVKoZ+Iaoo3YZjakJYxR+rw52NHo6jM+VE9b8+7ygkCFXl/EHID5wh/MkXaE/jGyYw=="], - "@keyv/redis": ["@keyv/redis@4.4.1", "", { "dependencies": { "@redis/client": "^1.6.0", "cluster-key-slot": "^1.1.2" }, "peerDependencies": { "keyv": "^5.3.4" } }, "sha512-ALRB/prv0ZQW+m20EaO9f9Jduzakd2edBhfq/Ro/T/AA6RQ4bn3FYNJKSORgPFgn19tWCYdivovSQgSsxkxufg=="], + "@keyv/redis": ["@keyv/redis@5.1.6", "", { "dependencies": { "@redis/client": "^5.10.0", "cluster-key-slot": "^1.1.2", "hookified": "^1.13.0" }, "peerDependencies": { "keyv": "^5.6.0" } }, "sha512-eKvW6pspvVaU5dxigaIDZr635/Uw6urTXL3gNbY9WTR8d3QigZQT+r8gxYSEOsw4+1cCBsC4s7T2ptR0WC9LfQ=="], "@keyv/serialize": ["@keyv/serialize@1.1.1", "", {}, "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA=="], @@ -1275,6 +1276,8 @@ "hoek": ["hoek@4.3.1", "", {}, "sha512-v7E+yIjcHECn973i0xHm4kJkEpv3C8sbYS4344WXbzYqRyiDD7rjnnKo4hsJkejQBAFdRMUGNHySeSPKSH9Rqw=="], + "hookified": ["hookified@1.15.1", "", {}, "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg=="], + "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="], "hyphenate-style-name": ["hyphenate-style-name@1.1.0", "", {}, "sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw=="], @@ -1417,7 +1420,7 @@ "keccak": ["keccak@3.0.4", "", { "dependencies": { "node-addon-api": "^2.0.0", "node-gyp-build": "^4.2.0", "readable-stream": "^3.6.0" } }, "sha512-3vKuW0jV8J3XNTzvfyicFR5qvxrSAGl7KIhvgOu5cmWwM7tZRj3fMbj/pfIf4be7aznbc+prBWGjywox/g2Y6Q=="], - "keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], + "keyv": ["keyv@5.5.5", "", { "dependencies": { "@keyv/serialize": "^1.1.1" } }, "sha512-FA5LmZVF1VziNc0bIdCSA1IoSVnDCqE8HJIZZv2/W8YmoAM50+tnUgJR/gQZwEeIMleuIOnRnHA/UaZRNeV4iQ=="], "keyvaluestorage-interface": ["keyvaluestorage-interface@1.0.0", "", {}, "sha512-8t6Q3TclQ4uZynJY9IGr2+SsIGwK9JHcO6ootkHCGA0CrQCRy+VkouYNO2xicET6b9al7QKzpebNow+gkpCL8g=="], @@ -2063,7 +2066,7 @@ "@apollo/server/lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], - "@apollo/utils.keyvadapter/@apollo/utils.keyvaluecache": ["@apollo/utils.keyvaluecache@3.1.0", "", { "dependencies": { "@apollo/utils.logger": "^3.0.0", "lru-cache": "^10.0.0" } }, "sha512-MM/DKIqpQQbuNG1gNPAlGc45THdWkroTmN8o/J09merFwf/LlZ7+lAfcHFDXIYIknwKmUjJrOMS3OxYbjrz2hA=="], + "@apollo/utils.keyvadapter/@apollo/utils.keyvaluecache": ["@apollo/utils.keyvaluecache@4.0.0", "", { "dependencies": { "@apollo/utils.logger": "^3.0.0", "lru-cache": "^11.0.0" } }, "sha512-mKw1myRUkQsGPNB+9bglAuhviodJ2L2MRYLTafCMw5BIo7nbvCPNCkLnIHjZ1NOzH7SnMAr5c9LmXiqsgYqLZw=="], "@apollo/utils.keyvaluecache/lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], @@ -2097,7 +2100,7 @@ "@json-rpc-tools/provider/ws": ["ws@7.5.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg=="], - "@keyv/redis/keyv": ["keyv@5.5.5", "", { "dependencies": { "@keyv/serialize": "^1.1.1" } }, "sha512-FA5LmZVF1VziNc0bIdCSA1IoSVnDCqE8HJIZZv2/W8YmoAM50+tnUgJR/gQZwEeIMleuIOnRnHA/UaZRNeV4iQ=="], + "@keyv/redis/@redis/client": ["@redis/client@5.11.0", "", { "dependencies": { "cluster-key-slot": "1.1.2" }, "peerDependencies": { "@node-rs/xxhash": "^1.1.0" }, "optionalPeers": ["@node-rs/xxhash"] }, "sha512-GHoprlNQD51Xq2Ztd94HHV94MdFZQ3CVrpA04Fz8MVoHM0B7SlbmPEVIjwTbcv58z8QyjnrOuikS0rWF03k5dQ=="], "@metamask/json-rpc-engine/@metamask/utils": ["@metamask/utils@8.5.0", "", { "dependencies": { "@ethereumjs/tx": "^4.2.0", "@metamask/superstruct": "^3.0.0", "@noble/hashes": "^1.3.1", "@scure/base": "^1.1.3", "@types/debug": "^4.1.7", "debug": "^4.3.4", "pony-cause": "^2.1.10", "semver": "^7.5.4", "uuid": "^9.0.1" } }, "sha512-I6bkduevXb72TIM9q2LRO63JSsF9EXduh3sBr9oybNX2hNNpr/j1tEjXrsG0Uabm4MJ1xkGAQEMwifvKZIkyxQ=="], @@ -2215,6 +2218,8 @@ "finalhandler/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "flat-cache/keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], + "foreground-child/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], "glob/minimatch": ["minimatch@5.1.6", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g=="], @@ -2299,6 +2304,8 @@ "@apollo/utils.keyvadapter/@apollo/utils.keyvaluecache/@apollo/utils.logger": ["@apollo/utils.logger@3.0.0", "", {}, "sha512-M8V8JOTH0F2qEi+ktPfw4RL7MvUycDfKp7aEap2eWXfL5SqWHN6jTLbj5f5fj1cceHpyaUSOZlvlaaryaxZAmg=="], + "@apollo/utils.keyvadapter/@apollo/utils.keyvaluecache/lru-cache": ["lru-cache@11.2.6", "", {}, "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ=="], + "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], "@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.1.0", "", {}, "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA=="], diff --git a/packages/web/app/api/rest/list/redis.ts b/packages/web/app/api/rest/cache.ts similarity index 75% rename from packages/web/app/api/rest/list/redis.ts rename to packages/web/app/api/rest/cache.ts index 3d9f41d1..dad42f97 100644 --- a/packages/web/app/api/rest/list/redis.ts +++ b/packages/web/app/api/rest/cache.ts @@ -1,8 +1,9 @@ import { createKeyv } from '@keyv/redis' -export function createListsKeyv(namespace?: string) { +export function createKeyvClient(namespace?: string) { const redisUrl = process.env.REST_CACHE_REDIS_URL || 'redis://localhost:6379' return createKeyv(redisUrl, { namespace, }) } + diff --git a/packages/web/app/api/rest/list/refresh.ts b/packages/web/app/api/rest/list/refresh.ts index 4bdeb94a..834a3e0e 100644 --- a/packages/web/app/api/rest/list/refresh.ts +++ b/packages/web/app/api/rest/list/refresh.ts @@ -1,10 +1,10 @@ +import { createKeyvClient } from '../cache' import { getVaultsList } from './db' -import { createListsKeyv } from './redis' -async function refresh(): Promise { - console.time('refresh') +const keyv = createKeyvClient('list:vaults') - const keyv = createListsKeyv('list:vaults') +async function refresh(): Promise { + console.time('refresh list:vaults') const vaults = await getVaultsList() @@ -17,24 +17,28 @@ async function refresh(): Promise { }, {} as Record) const chainIds = Object.keys(vaultsByChain).map(Number) - for (const chainId of chainIds) { - const chainVaults = vaultsByChain[chainId] - await keyv.set(String(chainId), JSON.stringify(chainVaults)) - } + const entries = chainIds.map((chainId) => ({ + key: String(chainId), + value: vaultsByChain[chainId], + })) + + entries.push({ key: 'all', value: vaults }) - await keyv.set('all', JSON.stringify(vaults)) + await keyv.setMany(entries) console.log(`✓ Completed: ${vaults.length} vaults cached across ${chainIds.length} chains`) - console.timeEnd('refresh') + console.timeEnd('refresh list:vaults') } if (require.main === module) { refresh() - .then(() => { + .then(async () => { + await keyv.disconnect() process.exit(0) }) - .catch(err => { + .catch(async (err) => { console.error(err) + await keyv.disconnect() process.exit(1) }) } diff --git a/packages/web/app/api/rest/list/vaults/[chainId]/route.ts b/packages/web/app/api/rest/list/vaults/[chainId]/route.ts index 96fd7a99..0d9370ac 100644 --- a/packages/web/app/api/rest/list/vaults/[chainId]/route.ts +++ b/packages/web/app/api/rest/list/vaults/[chainId]/route.ts @@ -1,8 +1,9 @@ import { NextResponse } from 'next/server' -import { createListsKeyv } from '../../redis' - +import { createKeyvClient } from '../../../cache' import type { VaultListItem } from '../../db' +const keyv = createKeyvClient('list:vaults') + export const runtime = 'nodejs' const corsHeaders = { @@ -10,8 +11,6 @@ const corsHeaders = { 'access-control-allow-methods': 'GET,OPTIONS', } -const listsKeyv = createListsKeyv('list:vaults') - type RouteParams = { chainId?: string | string[] } @@ -30,26 +29,18 @@ export async function GET( return new NextResponse('Invalid chainId', { status: 400, headers: corsHeaders }) } - let cached + let vaults: VaultListItem[] | undefined try { - cached = await listsKeyv.get(String(chainId)) + vaults = await keyv.get(String(chainId)) as VaultListItem[] | undefined } catch (err) { console.error(`Redis read failed for chainId ${chainId}:`, err) throw err } - if (!cached) { + if (!vaults) { return new NextResponse('Not found', { status: 404, headers: corsHeaders }) } - let vaults: VaultListItem[] - try { - vaults = JSON.parse(cached as string) - } catch (e) { - console.error(`Failed to parse vault list for chain ${chainId}:`, e) - return new NextResponse('Internal Server Error', { status: 500, headers: corsHeaders }) - } - const filtered = origin ? vaults.filter(v => v.origin === origin) : vaults diff --git a/packages/web/app/api/rest/list/vaults/route.ts b/packages/web/app/api/rest/list/vaults/route.ts index e2e0ec93..e5ca3e1e 100644 --- a/packages/web/app/api/rest/list/vaults/route.ts +++ b/packages/web/app/api/rest/list/vaults/route.ts @@ -1,7 +1,9 @@ import { NextResponse } from 'next/server' -import { createListsKeyv } from '../redis' +import { createKeyvClient } from '../../cache' import type { VaultListItem } from '../db' +const keyv = createKeyvClient('list:vaults') + export const runtime = 'nodejs' const corsHeaders = { @@ -9,27 +11,17 @@ const corsHeaders = { 'access-control-allow-methods': 'GET,OPTIONS', } -const listsKeyv = createListsKeyv('list:vaults') - export async function GET(request: Request) { const { searchParams } = new URL(request.url) const origin = searchParams.get('origin') try { - const cached = await listsKeyv.get('all') + const allVaults = await keyv.get('all') as VaultListItem[] | undefined - if (!cached) { + if (!allVaults) { return new NextResponse('Not found', { status: 404, headers: corsHeaders }) } - let allVaults: VaultListItem[] - try { - allVaults = JSON.parse(cached as string) - } catch (e) { - console.error('Failed to parse vault list from Redis:', e) - return new NextResponse('Internal Server Error', { status: 500, headers: corsHeaders }) - } - const filtered = origin ? allVaults.filter(v => v.origin === origin) : allVaults diff --git a/packages/web/app/api/rest/reports/[chainId]/[address]/route.ts b/packages/web/app/api/rest/reports/[chainId]/[address]/route.ts index 8e038cfc..191039c8 100644 --- a/packages/web/app/api/rest/reports/[chainId]/[address]/route.ts +++ b/packages/web/app/api/rest/reports/[chainId]/[address]/route.ts @@ -1,6 +1,14 @@ import { NextRequest, NextResponse } from 'next/server' -import { createReportsKeyv, getReportKey} from '../../redis' +import { createKeyvClient } from '../../../cache' import { VaultReport } from '../../db' +import { getReportKey } from '../../redis' + +const keyv = createKeyvClient() + +const corsHeaders = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, OPTIONS', +} export async function GET( request: NextRequest, @@ -17,43 +25,28 @@ export async function GET( ) } - const keyv = createReportsKeyv() const key = getReportKey(chainId, address) + const data = await keyv.get(key) as VaultReport[] | undefined - const cached = await keyv.get(key) - - if (!cached) { + if (!data) { return NextResponse.json( { error: 'Not found' }, { status: 404 } ) } - let data - try { - data = typeof cached === 'string' ? JSON.parse(cached) : cached - - return NextResponse.json(data, { - headers: { - 'Cache-Control': 'public, max-age=900, s-maxage=900, stale-while-revalidate=600', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': 'GET, OPTIONS' - } - }) - } catch (e) { - console.error('Failed to parse cached report', e) - return NextResponse.json( - { error: 'Internal Server Error' }, - { status: 500 } - ) - } + return NextResponse.json(data, { + headers: { + 'Cache-Control': 'public, max-age=900, s-maxage=900, stale-while-revalidate=600', + ...corsHeaders, + } + }) } export async function OPTIONS() { return new NextResponse(null, { headers: { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': 'GET, OPTIONS', + ...corsHeaders, 'Access-Control-Allow-Headers': 'Content-Type, Authorization', }, }) diff --git a/packages/web/app/api/rest/reports/redis.ts b/packages/web/app/api/rest/reports/redis.ts index 90318c6d..05d03fb7 100644 --- a/packages/web/app/api/rest/reports/redis.ts +++ b/packages/web/app/api/rest/reports/redis.ts @@ -1,10 +1,3 @@ -import { createKeyv } from '@keyv/redis' - -export function createReportsKeyv() { - const redisUrl = process.env.REST_CACHE_REDIS_URL || 'redis://localhost:6379' - return createKeyv(redisUrl) -} - export function getReportKey( chainId: number, address: string, diff --git a/packages/web/app/api/rest/reports/refresh.ts b/packages/web/app/api/rest/reports/refresh.ts index f26e1337..23a7e1a1 100644 --- a/packages/web/app/api/rest/reports/refresh.ts +++ b/packages/web/app/api/rest/reports/refresh.ts @@ -1,12 +1,14 @@ import 'lib/global' -import { getVaults, getStrategyReports } from './db' -import { createReportsKeyv, getReportKey } from './redis' +import { createKeyvClient } from '../cache' +import { getStrategyReports, getVaults } from './db' +import { getReportKey } from './redis' + +const keyv = createKeyvClient() const BATCH_SIZE = parseInt(process.env.REFRESH_BATCH_SIZE || '10', 10) async function refreshReports(): Promise { - console.time('refreshReports') - const keyv = createReportsKeyv() + console.time('refresh vault_reports') console.log('Fetching vaults...') const vaults = await getVaults() @@ -14,40 +16,41 @@ async function refreshReports(): Promise { let processed = 0 - async function processVault(vault: { chainId: number; address: string }) { - const addressLower = vault.address.toLowerCase() - - const reports = await getStrategyReports(vault.chainId, vault.address) - - if (!reports || reports.length === 0) { - return + for (let i = 0; i < vaults.length; i += BATCH_SIZE) { + const batch = vaults.slice(i, i + BATCH_SIZE) + const results = await Promise.all(batch.map(async (vault) => { + const reports = await getStrategyReports(vault.chainId, vault.address) + if (!reports || reports.length === 0) return null + return { + key: getReportKey(vault.chainId, vault.address.toLowerCase()), + value: reports, + } + })) + + const entries = results.filter((r): r is NonNullable => r !== null) + if (entries.length > 0) { + await keyv.setMany(entries) } - const cacheKey = getReportKey(vault.chainId, addressLower) - await keyv.set(cacheKey, JSON.stringify(reports)) - - processed++ + processed += entries.length if (processed % 10 === 0) { console.log(`Processed ${processed}/${vaults.length} vaults`) } } - for (let i = 0; i < vaults.length; i += BATCH_SIZE) { - const batch = vaults.slice(i, i + BATCH_SIZE) - await Promise.all(batch.map(processVault)) - } - console.log(`✓ Completed: ${processed} vaults processed`) - console.timeEnd('refreshReports') + console.timeEnd('refresh vault_reports') } if (require.main === module) { refreshReports() - .then(() => { + .then(async () => { + await keyv.disconnect() process.exit(0) }) - .catch(err => { + .catch(async (err) => { console.error(err) + await keyv.disconnect() process.exit(1) }) } diff --git a/packages/web/app/api/rest/snapshot/[chainId]/[address]/route.ts b/packages/web/app/api/rest/snapshot/[chainId]/[address]/route.ts index 07c95051..4f6c60ea 100644 --- a/packages/web/app/api/rest/snapshot/[chainId]/[address]/route.ts +++ b/packages/web/app/api/rest/snapshot/[chainId]/[address]/route.ts @@ -1,6 +1,9 @@ import { NextRequest, NextResponse } from 'next/server' -import { createSnapshotKeyv, getSnapshotKey } from '../../redis' +import { createKeyvClient } from '../../../cache' import type { VaultSnapshot } from '../../db' +import { getSnapshotKey } from '../../redis' + +const keyv = createKeyvClient() export const runtime = 'nodejs' @@ -14,8 +17,6 @@ const corsHeaders = { 'access-control-allow-methods': 'GET,OPTIONS', } -const snapshotKeyv = createSnapshotKeyv() - export async function GET( request: NextRequest, context: { params: Promise }, @@ -31,20 +32,18 @@ export async function GET( const addressLower = address.toLowerCase() const cacheKey = getSnapshotKey(Number(chainId), addressLower) - let cached: string | undefined + let parsed: VaultSnapshot | undefined try { - cached = await snapshotKeyv.get(cacheKey) + parsed = await keyv.get(cacheKey) } catch (err) { console.error(`Redis read failed for ${cacheKey}:`, err) throw err } - if (!cached) { + if (!parsed) { return new NextResponse('Not found', { status: 404, headers: corsHeaders }) } - const parsed: VaultSnapshot = JSON.parse(cached as string) - return NextResponse.json(parsed, { status: 200, headers: { diff --git a/packages/web/app/api/rest/snapshot/redis.ts b/packages/web/app/api/rest/snapshot/redis.ts index 4ad3ae97..c9f48f34 100644 --- a/packages/web/app/api/rest/snapshot/redis.ts +++ b/packages/web/app/api/rest/snapshot/redis.ts @@ -1,10 +1,3 @@ -import { createKeyv } from '@keyv/redis' - -export function createSnapshotKeyv() { - const redisUrl = process.env.REST_CACHE_REDIS_URL || 'redis://localhost:6379' - return createKeyv(redisUrl) -} - export function getSnapshotKey( chainId: number, address: string, diff --git a/packages/web/app/api/rest/snapshot/refresh-snapshot.ts b/packages/web/app/api/rest/snapshot/refresh-snapshot.ts index 64a9a8dc..1b66b226 100644 --- a/packages/web/app/api/rest/snapshot/refresh-snapshot.ts +++ b/packages/web/app/api/rest/snapshot/refresh-snapshot.ts @@ -1,11 +1,13 @@ +import { createKeyvClient } from '../cache' import { getVaults, getVaultSnapshot } from './db' -import { createSnapshotKeyv, getSnapshotKey } from './redis' +import { getSnapshotKey } from './redis' + +const keyv = createKeyvClient() const BATCH_SIZE = 10 async function refresh(): Promise { console.time('refresh') - const keyv = createSnapshotKeyv() console.log('Fetching vaults...') const vaults = await getVaults() @@ -13,40 +15,41 @@ async function refresh(): Promise { let processed = 0 - async function processVault(vault: { chainId: number; address: string }) { - const addressLower = vault.address.toLowerCase() - - const snapshot = await getVaultSnapshot(vault.chainId, vault.address) - - if (!snapshot) { - return + for (let i = 0; i < vaults.length; i += BATCH_SIZE) { + const batch = vaults.slice(i, i + BATCH_SIZE) + const snapshots = await Promise.all(batch.map(async (vault) => { + const snapshot = await getVaultSnapshot(vault.chainId, vault.address) + if (!snapshot) return null + return { + key: getSnapshotKey(vault.chainId, vault.address.toLowerCase()), + value: snapshot, + } + })) + + const entries = snapshots.filter((s): s is NonNullable => s !== null) + if (entries.length > 0) { + await keyv.setMany(entries) } - const cacheKey = getSnapshotKey(vault.chainId, addressLower) - await keyv.set(cacheKey, JSON.stringify(snapshot)) - - processed++ + processed += entries.length if (processed % 10 === 0) { console.log(`Processed ${processed}/${vaults.length} vaults`) } } - for (let i = 0; i < vaults.length; i += BATCH_SIZE) { - const batch = vaults.slice(i, i + BATCH_SIZE) - await Promise.all(batch.map(processVault)) - } - console.log(`✓ Completed: ${processed} vaults processed`) console.timeEnd('refresh') } if (require.main === module) { refresh() - .then(() => { + .then(async () => { + await keyv.disconnect() process.exit(0) }) - .catch(err => { + .catch(async (err) => { console.error(err) + await keyv.disconnect() process.exit(1) }) } diff --git a/packages/web/app/api/rest/timeseries/[segment]/[chainId]/[address]/route.ts b/packages/web/app/api/rest/timeseries/[segment]/[chainId]/[address]/route.ts index 0714fb5f..4f4cc2a9 100644 --- a/packages/web/app/api/rest/timeseries/[segment]/[chainId]/[address]/route.ts +++ b/packages/web/app/api/rest/timeseries/[segment]/[chainId]/[address]/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server' +import { createKeyvClient } from '../../../../cache' import { labels } from '../../../labels' -import { createTimeseriesKeyv, getTimeseriesKey } from '../../../redis' +import { getTimeseriesKey } from '../../../redis' export const runtime = 'nodejs' @@ -15,7 +16,7 @@ const corsHeaders = { 'access-control-allow-methods': 'GET,OPTIONS', } -const timeseriesKeyv = createTimeseriesKeyv() +const timeseriesKeyv = createKeyvClient() export async function GET( request: NextRequest, @@ -53,7 +54,7 @@ export async function GET( throw err } const parsed: Array<{ time: number; component: string; value: number }> = cached - ? JSON.parse(cached as string) + ? (cached as Array<{ time: number; component: string; value: number }>) : [] const filtered = parsed.filter((row) => components.includes(row.component)) diff --git a/packages/web/app/api/rest/timeseries/redis.ts b/packages/web/app/api/rest/timeseries/redis.ts index ebca8a45..ca97fc35 100644 --- a/packages/web/app/api/rest/timeseries/redis.ts +++ b/packages/web/app/api/rest/timeseries/redis.ts @@ -1,10 +1,3 @@ -import { createKeyv } from '@keyv/redis' - -export function createTimeseriesKeyv() { - const redisUrl = process.env.REST_CACHE_REDIS_URL || 'redis://localhost:6379' - return createKeyv(redisUrl) -} - export function getTimeseriesKey( label: string, chainId: number, diff --git a/packages/web/app/api/rest/timeseries/refresh-timeseries.ts b/packages/web/app/api/rest/timeseries/refresh-timeseries.ts index 1e83c86a..a4facd49 100644 --- a/packages/web/app/api/rest/timeseries/refresh-timeseries.ts +++ b/packages/web/app/api/rest/timeseries/refresh-timeseries.ts @@ -1,12 +1,14 @@ -import { labels } from './labels' +import { createKeyvClient } from '../cache' import { getFullTimeseries, getVaults, TimeseriesRow } from './db' -import { createTimeseriesKeyv, getTimeseriesKey } from './redis' +import { labels } from './labels' +import { getTimeseriesKey } from './redis' + +const keyv = createKeyvClient() const BATCH_SIZE = 10 async function refresh24hr(): Promise { console.time('refresh24hr') - const keyv = createTimeseriesKeyv() console.log('Fetching vaults...') const vaults = await getVaults() @@ -14,48 +16,54 @@ async function refresh24hr(): Promise { let processed = 0 - async function processVault(vault: { chainId: number; address: string }) { - const addressLower = vault.address.toLowerCase() + for (let i = 0; i < vaults.length; i += BATCH_SIZE) { + const batch = vaults.slice(i, i + BATCH_SIZE) + const entries: Array<{ key: string; value: unknown }> = [] - await Promise.all(labels.map(async ({ label }) => { - const rows: TimeseriesRow[] = await getFullTimeseries( - vault.chainId, - vault.address, - label, - ) + await Promise.all(batch.map(async (vault) => { + const addressLower = vault.address.toLowerCase() - const minimal = rows.map(row => ({ - time: Number(row.time), - component: row.component, - value: row.value, - })) + await Promise.all(labels.map(async ({ label }) => { + const rows: TimeseriesRow[] = await getFullTimeseries( + vault.chainId, + vault.address, + label, + ) + + const minimal = rows.map(row => ({ + time: Number(row.time), + component: row.component, + value: row.value, + })) - const cacheKey = getTimeseriesKey(label, vault.chainId, addressLower) - await keyv.set(cacheKey, JSON.stringify(minimal)) + entries.push({ + key: getTimeseriesKey(label, vault.chainId, addressLower), + value: minimal, + }) + })) })) - processed++ + await keyv.setMany(entries) + + processed += batch.length if (processed % 10 === 0) { console.log(`Processed ${processed}/${vaults.length} vaults`) } } - for (let i = 0; i < vaults.length; i += BATCH_SIZE) { - const batch = vaults.slice(i, i + BATCH_SIZE) - await Promise.all(batch.map(processVault)) - } - console.log(`✓ Completed: ${processed} vaults processed`) console.timeEnd('refresh24hr') } if (require.main === module) { refresh24hr() - .then(() => { + .then(async () => { + await keyv.disconnect() process.exit(0) }) - .catch(err => { + .catch(async (err) => { console.error(err) + await keyv.disconnect() process.exit(1) }) } diff --git a/packages/web/package.json b/packages/web/package.json index 0f06e9da..b91598c9 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -12,9 +12,9 @@ }, "dependencies": { "@apollo/server": "^4.9.5", - "@apollo/utils.keyvadapter": "^3.1.0", + "@apollo/utils.keyvadapter": "4.0.1", "@as-integrations/next": "^3.0.0", - "@keyv/redis": "^4.4.1", + "@keyv/redis": "5.1.6", "autoprefixer": "10.4.15", "bullmq": "^5.21.2", "chart.js": "^4.4.0",