From 157b833bc4a9c4c0c1f22a2e90e8754c8809eaec Mon Sep 17 00:00:00 2001 From: Cho Young-Hwi Date: Fri, 13 Mar 2026 20:35:52 +0000 Subject: [PATCH 1/4] [#12] Add cron backfill endpoint for missed events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement GET /api/cron/backfill — scans last ~200 blocks for StorylineCreated, PlotChained, and Donation events from StoryFactory, fetches content from IPFS, verifies hashes, and upserts missing records to Supabase. Deduplication via (tx_hash, log_index) means existing records are safely skipped. Includes block timestamp caching, CRON_SECRET auth, and graceful skip when StoryFactory is not yet deployed. Add vercel.json with 5-minute cron schedule. Fixes #12 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/app/api/cron/backfill/route.ts | 279 +++++++++++++++++++++++++++++ vercel.json | 8 + 2 files changed, 287 insertions(+) create mode 100644 src/app/api/cron/backfill/route.ts create mode 100644 vercel.json diff --git a/src/app/api/cron/backfill/route.ts b/src/app/api/cron/backfill/route.ts new file mode 100644 index 00000000..a4c2d8ee --- /dev/null +++ b/src/app/api/cron/backfill/route.ts @@ -0,0 +1,279 @@ +import { NextResponse } from "next/server"; +import { decodeEventLog, type Log } from "viem"; +import { publicClient } from "../../../../../lib/viem"; +import { createServerClient } from "../../../../../lib/supabase"; +import { + storyFactoryAbi, + plotChainedEvent, + storylineCreatedEvent, + donationEvent, +} from "../../../../../lib/contracts/abi"; +import { STORY_FACTORY } from "../../../../../lib/contracts/constants"; +import { hashContent } from "../../../../../lib/content"; +import { detectWriterType } from "../../../../../lib/contracts/erc8004"; +import type { Database } from "../../../../../lib/supabase"; + +const IPFS_GATEWAY = "https://ipfs.filebase.io/ipfs/"; +const IPFS_TIMEOUT_MS = 10_000; + +/** + * How many blocks to scan per cron run (~5 min on Base = ~150 blocks at 2s/block). + * Slightly over-scan to handle timing variance. + */ +const SCAN_BLOCKS = BigInt(200); + +/** Cron authorization — set CRON_SECRET env var to protect this endpoint */ +function verifyCron(req: Request): boolean { + const secret = process.env.CRON_SECRET; + if (!secret) return true; // no secret configured = open (dev mode) + const authHeader = req.headers.get("authorization"); + return authHeader === `Bearer ${secret}`; +} + +async function fetchIPFSContent(cid: string): Promise { + try { + const res = await fetch(`${IPFS_GATEWAY}${cid}`, { + signal: AbortSignal.timeout(IPFS_TIMEOUT_MS), + }); + if (!res.ok) return null; + return await res.text(); + } catch { + return null; + } +} + +async function getBlockTimestamp(blockNumber: bigint): Promise { + const block = await publicClient.getBlock({ blockNumber }); + return new Date(Number(block.timestamp) * 1000).toISOString(); +} + +type PlotInsert = Database["public"]["Tables"]["plots"]["Insert"]; +type StorylineInsert = Database["public"]["Tables"]["storylines"]["Insert"]; +type DonationInsert = Database["public"]["Tables"]["donations"]["Insert"]; + +export async function GET(req: Request) { + if (!verifyCron(req)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const supabase = createServerClient(); + if (!supabase) { + return NextResponse.json( + { error: "Supabase not configured" }, + { status: 500 } + ); + } + + // Skip if StoryFactory not yet deployed + if (STORY_FACTORY === "0x0000000000000000000000000000000000000000") { + return NextResponse.json({ + skipped: true, + reason: "StoryFactory not deployed yet", + }); + } + + const currentBlock = await publicClient.getBlockNumber(); + const fromBlock = currentBlock > SCAN_BLOCKS ? currentBlock - SCAN_BLOCKS : BigInt(0); + + // Fetch all StoryFactory logs in the scan range + const logs = await publicClient.getLogs({ + address: STORY_FACTORY, + fromBlock, + toBlock: currentBlock, + }); + + let storylinesInserted = 0; + let plotsInserted = 0; + let donationsInserted = 0; + let errors = 0; + + // Cache block timestamps to avoid redundant RPC calls + const blockTimestampCache = new Map(); + async function getCachedBlockTimestamp(blockNumber: bigint): Promise { + const cached = blockTimestampCache.get(blockNumber); + if (cached) return cached; + const ts = await getBlockTimestamp(blockNumber); + blockTimestampCache.set(blockNumber, ts); + return ts; + } + + for (const log of logs) { + try { + const decoded = decodeEventLog({ + abi: storyFactoryAbi, + data: log.data, + topics: log.topics, + }); + + const txHash = log.transactionHash!.toLowerCase(); + const logIndex = log.logIndex!; + + if (decoded.eventName === "StorylineCreated") { + await processStorylineCreated( + decoded, + log, + txHash, + logIndex, + supabase, + getCachedBlockTimestamp + ); + storylinesInserted++; + } else if (decoded.eventName === "PlotChained") { + await processPlotChained( + decoded, + log, + txHash, + logIndex, + supabase, + getCachedBlockTimestamp + ); + plotsInserted++; + } else if (decoded.eventName === "Donation") { + await processDonation( + decoded, + log, + txHash, + logIndex, + supabase, + getCachedBlockTimestamp + ); + donationsInserted++; + } + } catch { + errors++; + } + } + + return NextResponse.json({ + scanned: { fromBlock: Number(fromBlock), toBlock: Number(currentBlock) }, + upserted: { + storylines: storylinesInserted, + plots: plotsInserted, + donations: donationsInserted, + }, + errors, + }); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type DecodedEvent = any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type SupabaseClient = any; + +async function processStorylineCreated( + decoded: DecodedEvent, + log: Log, + txHash: string, + logIndex: number, + supabase: SupabaseClient, + getTimestamp: (blockNumber: bigint) => Promise +) { + const { + storylineId, + writer, + tokenAddress, + title, + hasDeadline, + openingCID, + openingHash, + } = decoded.args; + + const timestampISO = await getTimestamp(log.blockNumber!); + const writerType = await detectWriterType(writer); + + const storylineRow: StorylineInsert = { + storyline_id: Number(storylineId), + writer_address: writer.toLowerCase(), + token_address: tokenAddress.toLowerCase(), + title, + plot_count: 1, + has_deadline: hasDeadline, + writer_type: writerType, + last_plot_time: timestampISO, + block_timestamp: timestampISO, + tx_hash: txHash, + log_index: logIndex, + }; + + await supabase + .from("storylines") + .upsert(storylineRow, { onConflict: "tx_hash,log_index" }); + + // Insert genesis plot + const content = await fetchIPFSContent(openingCID); + if (content !== null && hashContent(content) === openingHash) { + const plotRow: PlotInsert = { + storyline_id: Number(storylineId), + plot_index: 0, + writer_address: writer.toLowerCase(), + content, + content_cid: openingCID, + content_hash: openingHash as string, + block_timestamp: timestampISO, + tx_hash: txHash, + log_index: logIndex, + }; + await supabase + .from("plots") + .upsert(plotRow, { onConflict: "tx_hash,log_index" }); + } +} + +async function processPlotChained( + decoded: DecodedEvent, + log: Log, + txHash: string, + logIndex: number, + supabase: SupabaseClient, + getTimestamp: (blockNumber: bigint) => Promise +) { + const { storylineId, plotIndex, writer, contentCID, contentHash } = + decoded.args; + + const content = await fetchIPFSContent(contentCID); + if (content === null) return; // skip if content unavailable + if (hashContent(content) !== contentHash) return; // skip if hash mismatch + + const timestampISO = await getTimestamp(log.blockNumber!); + + const row: PlotInsert = { + storyline_id: Number(storylineId), + plot_index: Number(plotIndex), + writer_address: writer.toLowerCase(), + content, + content_cid: contentCID, + content_hash: contentHash as string, + block_timestamp: timestampISO, + tx_hash: txHash, + log_index: logIndex, + }; + + await supabase + .from("plots") + .upsert(row, { onConflict: "tx_hash,log_index" }); +} + +async function processDonation( + decoded: DecodedEvent, + log: Log, + txHash: string, + logIndex: number, + supabase: SupabaseClient, + getTimestamp: (blockNumber: bigint) => Promise +) { + const { storylineId, donor, amount } = decoded.args; + const timestampISO = await getTimestamp(log.blockNumber!); + + const row: DonationInsert = { + storyline_id: Number(storylineId), + donor_address: donor.toLowerCase(), + amount: amount.toString(), + block_timestamp: timestampISO, + tx_hash: txHash, + log_index: logIndex, + }; + + await supabase + .from("donations") + .upsert(row, { onConflict: "tx_hash,log_index" }); +} diff --git a/vercel.json b/vercel.json new file mode 100644 index 00000000..c9963fd9 --- /dev/null +++ b/vercel.json @@ -0,0 +1,8 @@ +{ + "crons": [ + { + "path": "/api/cron/backfill", + "schedule": "*/5 * * * *" + } + ] +} From 7c9545103cff0540521c1cf09de6e32eb0601f00 Mon Sep 17 00:00:00 2001 From: Cho Young-Hwi Date: Fri, 13 Mar 2026 20:38:08 +0000 Subject: [PATCH 2/4] [#12] Add persistent start-block tracking for cron backfill Add backfill_cursor table (singleton row) to persist last processed block number across cron runs. Backfill now reads cursor, scans from last_block+1 to currentBlock (capped at 200-block max range for first run), and advances the cursor after processing. This ensures events older than a single scan window are not permanently missed. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/app/api/cron/backfill/route.ts | 24 ++++++++++++++++++- supabase/migrations/00003_backfill_cursor.sql | 10 ++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 supabase/migrations/00003_backfill_cursor.sql diff --git a/src/app/api/cron/backfill/route.ts b/src/app/api/cron/backfill/route.ts index a4c2d8ee..14e3320a 100644 --- a/src/app/api/cron/backfill/route.ts +++ b/src/app/api/cron/backfill/route.ts @@ -73,7 +73,23 @@ export async function GET(req: Request) { } const currentBlock = await publicClient.getBlockNumber(); - const fromBlock = currentBlock > SCAN_BLOCKS ? currentBlock - SCAN_BLOCKS : BigInt(0); + + // Read last processed block from persistent cursor + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { data: cursor } = await (supabase.from("backfill_cursor") as any) + .select("last_block") + .eq("id", 1) + .single(); + const lastBlock = cursor?.last_block ? BigInt(cursor.last_block) : BigInt(0); + + // Start from the block after last processed, but cap scan range + const idealFrom = lastBlock > BigInt(0) ? lastBlock + BigInt(1) : BigInt(0); + const maxFrom = currentBlock > SCAN_BLOCKS ? currentBlock - SCAN_BLOCKS : BigInt(0); + const fromBlock = idealFrom > maxFrom ? maxFrom : idealFrom; + + if (fromBlock > currentBlock) { + return NextResponse.json({ skipped: true, reason: "Already up to date" }); + } // Fetch all StoryFactory logs in the scan range const logs = await publicClient.getLogs({ @@ -144,6 +160,12 @@ export async function GET(req: Request) { } } + // Persist cursor — advance to currentBlock + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (supabase.from("backfill_cursor") as any) + .update({ last_block: Number(currentBlock), updated_at: new Date().toISOString() }) + .eq("id", 1); + return NextResponse.json({ scanned: { fromBlock: Number(fromBlock), toBlock: Number(currentBlock) }, upserted: { diff --git a/supabase/migrations/00003_backfill_cursor.sql b/supabase/migrations/00003_backfill_cursor.sql new file mode 100644 index 00000000..7bdb192a --- /dev/null +++ b/supabase/migrations/00003_backfill_cursor.sql @@ -0,0 +1,10 @@ +-- Persistent cursor for cron backfill — tracks last processed block. +create table if not exists backfill_cursor ( + id integer primary key default 1 check (id = 1), -- singleton row + last_block bigint not null default 0, + updated_at timestamptz not null default now() +); + +-- Seed the singleton row +insert into backfill_cursor (id, last_block) values (1, 0) + on conflict (id) do nothing; From d64acbae72fc94586f01a181a49a91fef56586c4 Mon Sep 17 00:00:00 2001 From: Cho Young-Hwi Date: Fri, 13 Mar 2026 20:39:08 +0000 Subject: [PATCH 3/4] [#12] Rename response counter from 'upserted' to 'processed' Co-Authored-By: Claude Opus 4.6 (1M context) --- src/app/api/cron/backfill/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/api/cron/backfill/route.ts b/src/app/api/cron/backfill/route.ts index 14e3320a..277b9d25 100644 --- a/src/app/api/cron/backfill/route.ts +++ b/src/app/api/cron/backfill/route.ts @@ -168,7 +168,7 @@ export async function GET(req: Request) { return NextResponse.json({ scanned: { fromBlock: Number(fromBlock), toBlock: Number(currentBlock) }, - upserted: { + processed: { storylines: storylinesInserted, plots: plotsInserted, donations: donationsInserted, From 7c5d49a5593b4263b1f6387d954e34b2f28f4945 Mon Sep 17 00:00:00 2001 From: Cho Young-Hwi Date: Fri, 13 Mar 2026 20:39:57 +0000 Subject: [PATCH 4/4] [#12] Fix cursor logic: cap toBlock per run, never skip blocks Keep fromBlock = last_block+1 (no forward clamping). Cap toBlock to fromBlock+200 to limit scan range per run. Advance cursor only to toBlock (highest block actually scanned). Backlog is processed incrementally across successive cron runs without skipping. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/app/api/cron/backfill/route.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/app/api/cron/backfill/route.ts b/src/app/api/cron/backfill/route.ts index 277b9d25..4a3a2b32 100644 --- a/src/app/api/cron/backfill/route.ts +++ b/src/app/api/cron/backfill/route.ts @@ -82,20 +82,23 @@ export async function GET(req: Request) { .single(); const lastBlock = cursor?.last_block ? BigInt(cursor.last_block) : BigInt(0); - // Start from the block after last processed, but cap scan range - const idealFrom = lastBlock > BigInt(0) ? lastBlock + BigInt(1) : BigInt(0); - const maxFrom = currentBlock > SCAN_BLOCKS ? currentBlock - SCAN_BLOCKS : BigInt(0); - const fromBlock = idealFrom > maxFrom ? maxFrom : idealFrom; + // Start from block after last processed; cap toBlock to limit scan per run + const fromBlock = lastBlock > BigInt(0) ? lastBlock + BigInt(1) : BigInt(0); if (fromBlock > currentBlock) { return NextResponse.json({ skipped: true, reason: "Already up to date" }); } + // Cap scan range per run to avoid timeouts on large backlogs + const toBlock = (fromBlock + SCAN_BLOCKS) < currentBlock + ? fromBlock + SCAN_BLOCKS + : currentBlock; + // Fetch all StoryFactory logs in the scan range const logs = await publicClient.getLogs({ address: STORY_FACTORY, fromBlock, - toBlock: currentBlock, + toBlock, }); let storylinesInserted = 0; @@ -160,14 +163,14 @@ export async function GET(req: Request) { } } - // Persist cursor — advance to currentBlock + // Persist cursor — advance to highest block actually scanned // eslint-disable-next-line @typescript-eslint/no-explicit-any await (supabase.from("backfill_cursor") as any) - .update({ last_block: Number(currentBlock), updated_at: new Date().toISOString() }) + .update({ last_block: Number(toBlock), updated_at: new Date().toISOString() }) .eq("id", 1); return NextResponse.json({ - scanned: { fromBlock: Number(fromBlock), toBlock: Number(currentBlock) }, + scanned: { fromBlock: Number(fromBlock), toBlock: Number(toBlock) }, processed: { storylines: storylinesInserted, plots: plotsInserted,