From e1a0e55620c0ffeb09bbf3727b04af7789035ea7 Mon Sep 17 00:00:00 2001 From: Khoirul Asfian Date: Tue, 31 Mar 2026 16:38:31 +0700 Subject: [PATCH 1/3] feat: improve logging accross the apps --- Dockerfile | 8 +- apps/content-proxy/package.json | 1 + apps/content-proxy/src/index.ts | 13 +- apps/web/package.json | 1 + apps/web/server.ts | 193 +++++------------- .../src/lib/middleware/logger-middleware.ts | 7 +- apps/web/src/lib/server/entry-sfn.ts | 30 +-- bun.lock | 52 ++++- packages/external-script/package.json | 1 + .../external-script/src/feed/refetch-feeds.ts | 47 ++--- packages/logger/package.json | 22 ++ packages/logger/src/index.ts | 97 +++++++++ packages/logger/tsconfig.json | 21 ++ 13 files changed, 287 insertions(+), 206 deletions(-) create mode 100644 packages/logger/package.json create mode 100644 packages/logger/src/index.ts create mode 100644 packages/logger/tsconfig.json diff --git a/Dockerfile b/Dockerfile index 3b6f7b6..3f217b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ COPY apps/content-proxy/package.json ./apps/content-proxy/ COPY packages/database/package.json ./packages/database/ COPY packages/feed-utils/package.json ./packages/feed-utils/ COPY packages/external-script/package.json ./packages/external-script/ +COPY packages/logger/package.json ./packages/logger/ # Install dependencies (including devDependencies for build) RUN bun install --linker hoisted --frozen-lockfile @@ -27,6 +28,7 @@ COPY apps/content-proxy/ ./apps/content-proxy/ COPY packages/database/ ./packages/database/ COPY packages/feed-utils/ ./packages/feed-utils/ COPY packages/external-script/ ./packages/external-script/ +COPY packages/logger/ ./packages/logger/ # Build application with version injection ARG BUILD_VERSION @@ -34,7 +36,7 @@ ARG BUILD_DATE ARG VITE_APP_VERSION ENV NODE_ENV=production ENV VITE_APP_VERSION=${VITE_APP_VERSION:-dev} -RUN bun run build --filter=@reafrac/web --filter=@reafrac/database --filter=@reafrac/feed-utils +RUN bun run build --filter=@reafrac/web --filter=@reafrac/database --filter=@reafrac/feed-utils --filter=@reafrac/logger FROM base AS dependencies @@ -47,6 +49,7 @@ COPY apps/content-proxy/package.json ./apps/content-proxy/ COPY packages/database/package.json ./packages/database/ COPY packages/feed-utils/package.json ./packages/feed-utils/ COPY packages/external-script/package.json ./packages/external-script/ +COPY packages/logger/package.json ./packages/logger/ RUN bun install --linker hoisted --frozen-lockfile --production @@ -70,10 +73,11 @@ COPY --from=builder /app/turbo.json ./ # Copy node_modules from dependencies stage COPY --from=dependencies /app/node_modules ./node_modules -# Copy workspace packages (needed for workspace symlinks) COPY --from=builder /app/packages ./packages COPY --from=builder /app/tsconfig.base.json ./tsconfig.base.json +RUN rm -rf packages/*/node_modules packages/*/*/node_modules + # Switch to non-root user (bun user already exists in image) USER bun diff --git a/apps/content-proxy/package.json b/apps/content-proxy/package.json index 0b2421d..7772953 100644 --- a/apps/content-proxy/package.json +++ b/apps/content-proxy/package.json @@ -14,6 +14,7 @@ "@elysiajs/openapi": "^1.4.11", "@reafrac/external-script": "workspace:*", "@reafrac/feed-utils": "workspace:*", + "@reafrac/logger": "workspace:*", "elysia": "~1.4.16", "zod": "^4.1.12" }, diff --git a/apps/content-proxy/src/index.ts b/apps/content-proxy/src/index.ts index a247ea7..123de87 100644 --- a/apps/content-proxy/src/index.ts +++ b/apps/content-proxy/src/index.ts @@ -2,6 +2,9 @@ import { extractFeed, parsedFeedSchema, extractArticle } from '@reafrac/feed-uti import { Elysia } from 'elysia'; import { openapi } from '@elysiajs/openapi'; import { z } from 'zod'; +import { createLogger } from '@reafrac/logger'; + +const log = createLogger({ name: 'content-proxy' }); const app = new Elysia() .use( @@ -20,9 +23,9 @@ const app = new Elysia() .post( '/extract-feed', async ({ body }) => { - console.log('extracting feed: ', body.url); + log.debug({ url: body.url }, 'Extracting feed'); const validated = await extractFeed(body.url); - + log.info({ url: body.url }, 'Feed extracted'); return validated; }, { @@ -33,9 +36,9 @@ const app = new Elysia() .post( '/extract-article', async ({ body }) => { - console.log('extracting article: ', body.url); + log.debug({ url: body.url }, 'Extracting article'); const validated = await extractArticle(body.url); - + log.info({ url: body.url }, 'Article extracted'); return validated; }, { @@ -60,4 +63,4 @@ const app = new Elysia() ) .listen(3001); -console.log(`šŸ¦Š Elysia is running at ${app.server?.hostname}:${app.server?.port}`); +log.info({ host: app.server?.hostname, port: app.server?.port }, 'Content proxy server running'); diff --git a/apps/web/package.json b/apps/web/package.json index 2f14a2f..988c542 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -25,6 +25,7 @@ "@reafrac/database": "workspace:*", "@reafrac/external-script": "workspace:*", "@reafrac/feed-utils": "workspace:*", + "@reafrac/logger": "workspace:*", "@sentry/tanstackstart-react": "^10.22.0", "@tailwindcss/vite": "^4.1.16", "@tanstack/react-devtools": "^0.7.8", diff --git a/apps/web/server.ts b/apps/web/server.ts index ab49c71..14dabd2 100644 --- a/apps/web/server.ts +++ b/apps/web/server.ts @@ -76,35 +76,18 @@ import path from 'node:path'; import * as Sentry from '@sentry/tanstackstart-react'; import { refetchFeeds } from '@reafrac/external-script'; import { runMigrations } from '@reafrac/database'; +import { createLogger, type Logger } from '@reafrac/logger'; const ENABLE_FEED_CRON = process.env.ENABLE_FEED_CRON === 'true'; const FEED_CRON_INTERVAL_MS = Number(process.env.FEED_CRON_INTERVAL_MS ?? 30 * 60 * 1000); let feedCronRunning = false; -// Configuration const SERVER_PORT = Number(process.env.PORT ?? 3000); const CLIENT_DIRECTORY = './dist/client'; const SERVER_ENTRY_POINT = './dist/server/server.js'; -// Logging utilities for professional output -const log = { - info: (message: string) => { - console.log(`[INFO] ${message}`); - }, - success: (message: string) => { - console.log(`[SUCCESS] ${message}`); - }, - warning: (message: string) => { - console.log(`[WARNING] ${message}`); - }, - error: (message: string) => { - console.log(`[ERROR] ${message}`); - }, - header: (message: string) => { - console.log(`\n${message}\n`); - } -}; +const log: Logger = createLogger({ name: 'server' }); // Preloading configuration from environment variables const MAX_PRELOAD_BYTES = Number( @@ -292,15 +275,16 @@ async function initializeStaticRoutes(clientDirectory: string): Promise 0) { - console.log(`Include patterns: ${process.env.ASSET_PRELOAD_INCLUDE_PATTERNS ?? ''}`); - } - if (EXCLUDE_PATTERNS.length > 0) { - console.log(`Exclude patterns: ${process.env.ASSET_PRELOAD_EXCLUDE_PATTERNS ?? ''}`); - } + log.debug( + { + maxSizeMB: (MAX_PRELOAD_BYTES / 1024 / 1024).toFixed(2), + includePatterns: process.env.ASSET_PRELOAD_INCLUDE_PATTERNS ?? '', + excludePatterns: process.env.ASSET_PRELOAD_EXCLUDE_PATTERNS ?? '' + }, + 'Asset preload configuration' + ); } let totalPreloadedBytes = 0; @@ -368,110 +352,47 @@ async function initializeStaticRoutes(clientDirectory: string): Promise 0 || skipped.length > 0)) { - const allFiles = [...loaded, ...skipped].sort((a, b) => a.route.localeCompare(b.route)); - - // Calculate max path length for alignment - const maxPathLength = Math.min(Math.max(...allFiles.map((f) => f.route.length)), 60); - - // Format file size with KB and actual gzip size - const formatFileSize = (bytes: number, gzBytes?: number) => { - const kb = bytes / 1024; - const sizeStr = kb < 100 ? kb.toFixed(2) : kb.toFixed(1); - - if (gzBytes !== undefined) { - const gzKb = gzBytes / 1024; - const gzStr = gzKb < 100 ? gzKb.toFixed(2) : gzKb.toFixed(1); - return { - size: sizeStr, - gzip: gzStr - }; - } - - // Rough gzip estimation (typically 30-70% compression) if no actual gzip data - const gzipKb = kb * 0.35; - return { - size: sizeStr, - gzip: gzipKb < 100 ? gzipKb.toFixed(2) : gzipKb.toFixed(1) - }; - }; - - if (loaded.length > 0) { - console.log('\nšŸ“ Preloaded into memory:'); - console.log('Path ā”‚ Size ā”‚ Gzip Size'); - loaded - .sort((a, b) => a.route.localeCompare(b.route)) - .forEach((file) => { - const { size, gzip } = formatFileSize(file.size); - const paddedPath = file.route.padEnd(maxPathLength); - const sizeStr = `${size.padStart(7)} kB`; - const gzipStr = `${gzip.padStart(7)} kB`; - console.log(`${paddedPath} ā”‚ ${sizeStr} ā”‚ ${gzipStr}`); - }); - } - - if (skipped.length > 0) { - console.log('\nšŸ’¾ Served on-demand:'); - console.log('Path ā”‚ Size ā”‚ Gzip Size'); - skipped - .sort((a, b) => a.route.localeCompare(b.route)) - .forEach((file) => { - const { size, gzip } = formatFileSize(file.size); - const paddedPath = file.route.padEnd(maxPathLength); - const sizeStr = `${size.padStart(7)} kB`; - const gzipStr = `${gzip.padStart(7)} kB`; - console.log(`${paddedPath} ā”‚ ${sizeStr} ā”‚ ${gzipStr}`); - }); - } - } - - // Show detailed verbose info if enabled if (VERBOSE) { if (loaded.length > 0 || skipped.length > 0) { - const allFiles = [...loaded, ...skipped].sort((a, b) => a.route.localeCompare(b.route)); - console.log('\nšŸ“Š Detailed file information:'); - console.log( - 'Status ā”‚ Path ā”‚ MIME Type ā”‚ Reason' + log.debug( + { + preloaded: loaded.map((f) => ({ + route: f.route, + sizeKB: (f.size / 1024).toFixed(2), + type: f.type + })), + onDemand: skipped.map((f) => ({ + route: f.route, + sizeKB: (f.size / 1024).toFixed(2), + type: f.type + })) + }, + 'Asset preload details' ); - allFiles.forEach((file) => { - const isPreloaded = loaded.includes(file); - const status = isPreloaded ? 'MEMORY' : 'ON-DEMAND'; - const reason = - !isPreloaded && file.size > MAX_PRELOAD_BYTES - ? 'too large' - : !isPreloaded - ? 'filtered' - : 'preloaded'; - const route = file.route.length > 30 ? file.route.substring(0, 27) + '...' : file.route; - console.log( - `${status.padEnd(12)} ā”‚ ${route.padEnd(30)} ā”‚ ${file.type.padEnd(28)} ā”‚ ${reason.padEnd(10)}` - ); - }); } else { - console.log('\nšŸ“Š No files found to display'); + log.debug('No files found to preload'); } } - // Log summary after the file list - console.log(); // Empty line for separation if (loaded.length > 0) { - log.success( - `Preloaded ${String(loaded.length)} files (${(totalPreloadedBytes / 1024 / 1024).toFixed(2)} MB) into memory` + log.info( + { + count: loaded.length, + totalSizeMB: (totalPreloadedBytes / 1024 / 1024).toFixed(2) + }, + 'Static assets preloaded' ); } else { - log.info('No files preloaded into memory'); + log.info('No static assets preloaded into memory'); } if (skipped.length > 0) { const tooLarge = skipped.filter((f) => f.size > MAX_PRELOAD_BYTES).length; const filtered = skipped.length - tooLarge; - log.info( - `${String(skipped.length)} files will be served on-demand (${String(tooLarge)} too large, ${String(filtered)} filtered)` - ); + log.info({ count: skipped.length, tooLarge, filtered }, 'Static assets served on-demand'); } } catch (error) { - log.error(`Failed to load static files from ${clientDirectory}: ${String(error)}`); + log.error({ directory: clientDirectory, error: String(error) }, 'Failed to load static assets'); } return { routes, loaded, skipped }; @@ -481,82 +402,78 @@ async function initializeStaticRoutes(clientDirectory: string): Promise Response | Promise }; try { const serverModule = (await import(SERVER_ENTRY_POINT)) as { default: { fetch: (request: Request) => Response | Promise }; }; handler = serverModule.default; - log.success('TanStack Start application handler initialized'); + log.info('Application handler initialized'); } catch (error) { - log.error(`Failed to load server handler: ${String(error)}`); + log.error({ error: String(error) }, 'Failed to load server handler'); process.exit(1); } - // Build static routes with intelligent preloading const { routes } = await initializeStaticRoutes(CLIENT_DIRECTORY); - // Create Bun server const server = Bun.serve({ port: SERVER_PORT, routes: { - // Serve static assets (preloaded or on-demand) ...routes, - // Fallback to TanStack Start handler for all other routes '/*': (req: Request) => { try { return handler.fetch(req); } catch (error) { - log.error(`Server handler error: ${String(error)}`); + log.error({ error: String(error) }, 'Server handler error'); return new Response('Internal Server Error', { status: 500 }); } } }, - // Global error handler error(error) { - log.error(`Uncaught server error: ${error instanceof Error ? error.message : String(error)}`); + log.error( + { error: error instanceof Error ? error.message : String(error) }, + 'Uncaught server error' + ); return new Response('Internal Server Error', { status: 500 }); } }); - log.success(`Server listening on http://localhost:${String(server.port)}`); + log.info({ url: `http://localhost:${server.port}`, port: server.port }, 'Server listening'); if (ENABLE_FEED_CRON) { const runFeedRefetch = async () => { if (feedCronRunning) { - log.warning('Feed refetch already in progress, skipping this run'); + log.warn('Feed refetch already in progress, skipping'); return; } feedCronRunning = true; const startTime = Date.now(); - log.info('Starting scheduled feed refetch...'); + log.info('Starting scheduled feed refetch'); try { await Sentry.startSpan({ op: 'cron', name: 'feed-refetch' }, async () => { await refetchFeeds(); }); const duration = ((Date.now() - startTime) / 1000).toFixed(1); - log.success(`Feed refetch completed in ${duration}s`); + log.info({ durationSeconds: duration }, 'Feed refetch completed'); } catch (error) { const duration = ((Date.now() - startTime) / 1000).toFixed(1); - log.error(`Feed refetch failed after ${duration}s: ${String(error)}`); + log.error({ durationSeconds: duration, error: String(error) }, 'Feed refetch failed'); Sentry.captureException(error, { tags: { component: 'feed-cron' } }); @@ -566,14 +483,14 @@ async function initializeServer() { }; setInterval(runFeedRefetch, FEED_CRON_INTERVAL_MS); - log.success( - `Feed cron scheduled (every ${(FEED_CRON_INTERVAL_MS / 1000 / 60).toFixed(0)} minutes)` + log.info( + { intervalMinutes: (FEED_CRON_INTERVAL_MS / 1000 / 60).toFixed(0) }, + 'Feed cron scheduled' ); } } -// Initialize the server initializeServer().catch((error: unknown) => { - log.error(`Failed to start server: ${String(error)}`); + log.error({ error: String(error) }, 'Failed to start server'); process.exit(1); }); diff --git a/apps/web/src/lib/middleware/logger-middleware.ts b/apps/web/src/lib/middleware/logger-middleware.ts index 8b1dece..fab6a5f 100644 --- a/apps/web/src/lib/middleware/logger-middleware.ts +++ b/apps/web/src/lib/middleware/logger-middleware.ts @@ -1,8 +1,9 @@ import { createMiddleware } from '@tanstack/react-start'; +import { createLogger } from '@reafrac/logger'; + +const log = createLogger({ name: 'http' }); export const requestLoggerMiddleware = createMiddleware().server(async ({ next, request }) => { - if (process.env.NODE_ENV?.toLowerCase() !== 'production') { - console.log(`[${request.method}] - ${request.url}`); - } + log.info({ method: request.method, url: request.url }, 'Incoming request'); return await next(); }); diff --git a/apps/web/src/lib/server/entry-sfn.ts b/apps/web/src/lib/server/entry-sfn.ts index 466b883..7058113 100644 --- a/apps/web/src/lib/server/entry-sfn.ts +++ b/apps/web/src/lib/server/entry-sfn.ts @@ -9,6 +9,9 @@ import { db, entries, feeds, userEntries, userFeedSubscriptions } from '@reafrac import { eq, and, desc, count, gte } from '@reafrac/database'; import { ofetch } from 'ofetch'; import { htmlSanitizer } from '../utils/entry-utils'; +import { createLogger } from '@reafrac/logger'; + +const log = createLogger({ name: 'entry-sfn' }); export const extractEntryContentServerFn = createServerFn({ method: 'GET' }) .middleware([sentryMiddleware, authFnMiddleware]) @@ -21,11 +24,10 @@ export const extractEntryContentServerFn = createServerFn({ method: 'GET' }) span.setAttribute('user_id', context.user.id); span.setAttribute('entry_url', data.entryUrl); - // check if user has proxy setup const proxyUrl = process.env.PROXY_URL; span.setAttribute('proxy_url', proxyUrl); - console.log({ proxyUrl, url: data.entryUrl }); + log.debug({ proxyUrl, entryUrl: data.entryUrl }, 'Extracting entry content'); let validated: ArticleData | undefined = undefined; if (proxyUrl) { // if user has set proxy settings, use it to extract feed @@ -154,30 +156,6 @@ export const getEntriesServerFn = createServerFn({ method: 'GET' }) span.setAttribute('starred', data.starred || false); span.setAttribute('force_refetch', data.forceRefetch || false); - // Refetch entries data from feeds link stored in db - // Only refetch for specific feed if feedId is provided, otherwise refetch all feeds - Sentry.startSpan({ op: 'function', name: 'refetchBeforeGetEntries' }, async () => { - try { - // await refetchFeedEntries( - // context.user.id, - // data.feedId ? [data.feedId] : undefined, - // data.forceRefetch - // ); - } catch (error) { - console.error('Error refetching feed entries:', error); - // Don't let refetch errors block the main request - just log them - Sentry.captureException(error, { - tags: { function: 'refetchBeforeGetEntries' }, - extra: { - userId: context.user.id, - feedId: data.feedId, - forceRefetch: data.forceRefetch, - errorMessage: error instanceof Error ? error.message : 'Unknown error' - } - }); - } - }); - // Build query conditions - ensure user only sees entries from feeds they're subscribed to // Using inner joins guarantees we only get entries that have user entries const conditions = [ diff --git a/bun.lock b/bun.lock index 71226f7..9215df6 100644 --- a/bun.lock +++ b/bun.lock @@ -21,6 +21,7 @@ "@elysiajs/openapi": "^1.4.11", "@reafrac/external-script": "workspace:*", "@reafrac/feed-utils": "workspace:*", + "@reafrac/logger": "workspace:*", "elysia": "~1.4.16", "zod": "^4.1.12", }, @@ -55,6 +56,7 @@ "@reafrac/database": "workspace:*", "@reafrac/external-script": "workspace:*", "@reafrac/feed-utils": "workspace:*", + "@reafrac/logger": "workspace:*", "@sentry/tanstackstart-react": "^10.22.0", "@tailwindcss/vite": "^4.1.16", "@tanstack/react-devtools": "^0.7.8", @@ -127,6 +129,7 @@ "dependencies": { "@reafrac/database": "workspace:*", "@reafrac/feed-utils": "workspace:*", + "@reafrac/logger": "workspace:*", "drizzle-orm": "^0.44.5", "nanoid": "^5.1.6", "postgres": "^3.4.7", @@ -151,6 +154,19 @@ "vitest": "^3.0.5", }, }, + "packages/logger": { + "name": "@reafrac/logger", + "version": "0.1.0", + "dependencies": { + "pino": "^9.6.0", + "pino-pretty": "^13.0.0", + }, + "devDependencies": { + "@types/bun": "^1.3.3", + "oxlint": "^1.25.0", + "typescript": "^5.7.2", + }, + }, }, "packages": { "@apm-js-collab/code-transformer": ["@apm-js-collab/code-transformer@0.8.2", "", {}, "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA=="], @@ -609,6 +625,8 @@ "@reafrac/feed-utils": ["@reafrac/feed-utils@workspace:packages/feed-utils"], + "@reafrac/logger": ["@reafrac/logger@workspace:packages/logger"], + "@reafrac/web": ["@reafrac/web@workspace:apps/web"], "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.47", "", {}, "sha512-8QagwMH3kNCuzD8EWL8R2YPW5e4OrHNSAHRFDdmFqEwEaD/KcNKjVoumo+gP2vW5eKB2UPbM6vTYiGZX0ixLnw=="], @@ -965,6 +983,8 @@ "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], + "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], @@ -991,6 +1011,8 @@ "data-urls": ["data-urls@5.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0" } }, "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg=="], + "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="], + "dayjs": ["dayjs@1.11.19", "", {}, "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1043,6 +1065,8 @@ "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "enhanced-resolve": ["enhanced-resolve@5.18.3", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww=="], "enquirer": ["enquirer@2.4.1", "", { "dependencies": { "ansi-colors": "^4.1.1", "strip-ansi": "^6.0.1" } }, "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ=="], @@ -1087,6 +1111,8 @@ "extendable-error": ["extendable-error@0.1.7", "", {}, "sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg=="], + "fast-copy": ["fast-copy@4.0.2", "", {}, "sha512-ybA6PDXIXOXivLJK/z9e+Otk7ve13I4ckBvGO5I2RRmBU1gMHLVDJYEuJYhGwez7YNlYji2M2DvVU+a9mSFDlw=="], + "fast-decode-uri-component": ["fast-decode-uri-component@1.0.1", "", {}, "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg=="], "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], @@ -1097,6 +1123,8 @@ "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], + "fast-safe-stringify": ["fast-safe-stringify@2.1.1", "", {}, "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="], + "fast-xml-parser": ["fast-xml-parser@5.3.2", "", { "dependencies": { "strnum": "^2.1.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-n8v8b6p4Z1sMgqRmqLJm3awW4NX7NkaKPfb3uJIBTSH7Pdvufi3PQ3/lJLQrvxcMYl7JI2jnDO90siPEpD8JBA=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], @@ -1147,6 +1175,8 @@ "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + "help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="], + "hoist-non-react-statics": ["hoist-non-react-statics@3.3.2", "", { "dependencies": { "react-is": "^16.7.0" } }, "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw=="], "html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="], @@ -1199,6 +1229,8 @@ "jose": ["jose@6.1.2", "", {}, "sha512-MpcPtHLE5EmztuFIqB0vzHAWJPpmN1E6L4oo+kze56LIs3MyXIj9ZHMDxqOvkP38gBR7K1v3jqd4WU2+nrfONQ=="], + "joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="], + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], "js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], @@ -1273,6 +1305,8 @@ "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], "mri": ["mri@1.2.0", "", {}, "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="], @@ -1301,6 +1335,8 @@ "on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="], + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], @@ -1355,10 +1391,12 @@ "pify": ["pify@4.0.1", "", {}, "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g=="], - "pino": ["pino@10.1.0", "", { "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-0zZC2ygfdqvqK8zJIr1e+wT1T/L+LF6qvqvbzEQ6tiMAoTqEVK9a1K3YRu8HEUvGEvNqZyPJTtb2sNIoTkB83w=="], + "pino": ["pino@9.14.0", "", { "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w=="], "pino-abstract-transport": ["pino-abstract-transport@2.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw=="], + "pino-pretty": ["pino-pretty@13.1.3", "", { "dependencies": { "colorette": "^2.0.7", "dateformat": "^4.6.3", "fast-copy": "^4.0.0", "fast-safe-stringify": "^2.1.1", "help-me": "^5.0.0", "joycon": "^3.1.1", "minimist": "^1.2.6", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^3.0.0", "pump": "^3.0.0", "secure-json-parse": "^4.0.0", "sonic-boom": "^4.0.1", "strip-json-comments": "^5.0.2" }, "bin": { "pino-pretty": "bin.js" } }, "sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg=="], + "pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="], "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], @@ -1385,6 +1423,8 @@ "process-warning": ["process-warning@5.0.0", "", {}, "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA=="], + "pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="], + "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], "quansync": ["quansync@0.2.11", "", {}, "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA=="], @@ -1445,6 +1485,8 @@ "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + "secure-json-parse": ["secure-json-parse@4.1.0", "", {}, "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA=="], + "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], "seroval": ["seroval@1.4.0", "", {}, "sha512-BdrNXdzlofomLTiRnwJTSEAaGKyHHZkbMXIywOh7zlzp4uZnXErEwl9XZ+N1hJSNpeTtNxWvVwN0wUzAIQ4Hpg=="], @@ -1491,7 +1533,7 @@ "strip-bom": ["strip-bom@3.0.0", "", {}, "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA=="], - "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], + "strip-json-comments": ["strip-json-comments@5.0.3", "", {}, "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw=="], "strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="], @@ -1629,6 +1671,8 @@ "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="], "xml-name-validator": ["xml-name-validator@5.0.0", "", {}, "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg=="], @@ -1663,6 +1707,8 @@ "@eslint/eslintrc/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + "@eslint/eslintrc/strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], + "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="], "@manypkg/find-root/find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="], @@ -1735,6 +1781,8 @@ "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + "pino-pretty/pino-abstract-transport": ["pino-abstract-transport@3.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg=="], + "postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], "readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], diff --git a/packages/external-script/package.json b/packages/external-script/package.json index 8cdcc55..57ae290 100644 --- a/packages/external-script/package.json +++ b/packages/external-script/package.json @@ -12,6 +12,7 @@ "dependencies": { "@reafrac/database": "workspace:*", "@reafrac/feed-utils": "workspace:*", + "@reafrac/logger": "workspace:*", "drizzle-orm": "^0.44.5", "nanoid": "^5.1.6", "postgres": "^3.4.7", diff --git a/packages/external-script/src/feed/refetch-feeds.ts b/packages/external-script/src/feed/refetch-feeds.ts index a740c34..4efbfda 100644 --- a/packages/external-script/src/feed/refetch-feeds.ts +++ b/packages/external-script/src/feed/refetch-feeds.ts @@ -1,26 +1,27 @@ import { db, feeds, entries, userFeedSubscriptions, userEntries } from '@reafrac/database'; import { eq, inArray, and } from 'drizzle-orm'; import { extractFeed, ParsedFeed } from '@reafrac/feed-utils'; +import { createLogger } from '@reafrac/logger'; +const log = createLogger({ name: 'feed-refetch' }); const proxyUrl = process.env.PROXY_URL; -// Main function to refetch feeds and update entries + export async function refetchFeeds() { - console.log('Starting feed refetch process...'); + log.info('Starting feed refetch process'); try { - const feedsToRefetch = await db.select().from(feeds).limit(50); // Limit to prevent overwhelming the system + const feedsToRefetch = await db.select().from(feeds).limit(50); - console.log(`>>> Found ${feedsToRefetch.length} feeds to refetch`); + log.info({ count: feedsToRefetch.length }, 'Found feeds to refetch'); - // Create async functions for each feed to be executed in parallel const feedTasks = feedsToRefetch.map((feed, idx) => async () => { + const feedLog = log.child({ feedId: feed.id, feedTitle: feed.title, feedLink: feed.link }); + try { - const i = String(idx + 1).padStart(2, '0'); - console.log(`${i}. Refetching feed: ${feed.title} (${feed.link})`); + feedLog.debug('Refetching feed'); let feedData: ParsedFeed | undefined = undefined; if (proxyUrl) { - // if user has set proxy settings, use it to extract feed const httpResponse = await fetch(`${proxyUrl}/extract-feed`, { method: 'POST', body: JSON.stringify({ url: feed.link }), @@ -33,11 +34,9 @@ export async function refetchFeeds() { feedData = (await httpResponse.json()) as ParsedFeed; } else { - // otherwise, do the feed extraction in this server feedData = await extractFeed(feed.link); } - // Update feed's last fetched timestamp await db .update(feeds) .set({ @@ -46,20 +45,17 @@ export async function refetchFeeds() { }) .where(eq(feeds.id, feed.id)); - // Get users subscribed to this feed const subscriptions = await db .select({ userId: userFeedSubscriptions.userId }) .from(userFeedSubscriptions) .where(eq(userFeedSubscriptions.feedId, feed.id)); if (subscriptions.length === 0) { - console.log(`${i}. No users subscribed to feed ${feed.title}, skipping entry insertion`); + feedLog.debug('No users subscribed, skipping entry insertion'); return; } - // Process new entries if (feedData.entries.length > 0) { - // Get only titles from new entries to check against existing const newTitles = feedData.entries.map((entry) => entry.title); const existingEntries = await db .select({ title: entries.title }) @@ -67,18 +63,15 @@ export async function refetchFeeds() { .where(and(eq(entries.feedId, feed.id), inArray(entries.title, newTitles))); const existingTitles = new Set(existingEntries.map((entry) => entry.title)); - - // Filter only new entries const newEntries = feedData.entries.filter((entry) => !existingTitles.has(entry.title)); if (newEntries.length === 0) { - console.log(`${i}. No new entries for feed ${feed.title}`); + feedLog.debug('No new entries found'); return; } - console.log(`${i}. Processing ${newEntries.length} new entries for feed ${feed.title}`); + feedLog.info({ count: newEntries.length }, 'Processing new entries'); - // Insert new entries const insertedEntries = await db .insert(entries) .values( @@ -96,10 +89,7 @@ export async function refetchFeeds() { ) .returning({ id: entries.id }); - // Create user entries for all subscribed users const userIds = subscriptions.map((sub) => sub.userId); - - // Batch inserts to avoid memory/query size issues const BATCH_SIZE = 1000; let totalCreated = 0; @@ -111,7 +101,6 @@ export async function refetchFeeds() { starred: false })); - // Insert in chunks for (let j = 0; j < batch.length; j += BATCH_SIZE) { const chunk = batch.slice(j, j + BATCH_SIZE); await db.insert(userEntries).values(chunk); @@ -120,18 +109,16 @@ export async function refetchFeeds() { } if (totalCreated > 0) { - console.log(`${i}. Created ${totalCreated} u-entries for ${feed.title}`); + feedLog.info({ userEntryCount: totalCreated }, 'Created user entries'); } } - console.log(`${i}. Successfully refetched feed: ${feed.title}`); + feedLog.info('Feed refetched successfully'); } catch (error) { - console.error(`Error refetching feed ${feed.id} (${feed.link}):`, error); - // Continue with other feeds even if one fails + feedLog.error({ error: String(error) }, 'Error refetching feed'); } }); - // Execute feed processing in parallel with a concurrency limit const FEED_CONCURRENCY = 10; let finishedFeedCount = 0; @@ -142,10 +129,10 @@ export async function refetchFeeds() { } if (finishedFeedCount > 0) { - console.log(`Feed refetch process completed successfully: ${finishedFeedCount} `); + log.info({ count: finishedFeedCount }, 'Feed refetch process completed'); } } catch (error) { - console.error('Error in feed refetch process:', error); + log.error({ error: String(error) }, 'Error in feed refetch process'); throw error; } } diff --git a/packages/logger/package.json b/packages/logger/package.json new file mode 100644 index 0000000..4dd0c62 --- /dev/null +++ b/packages/logger/package.json @@ -0,0 +1,22 @@ +{ + "name": "@reafrac/logger", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "scripts": { + "build": "tsc", + "lint": "oxlint", + "tsc": "tsc --noEmit" + }, + "dependencies": { + "pino": "^9.6.0", + "pino-pretty": "^13.0.0" + }, + "devDependencies": { + "@types/bun": "^1.3.3", + "oxlint": "^1.25.0", + "typescript": "^5.7.2" + } +} diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts new file mode 100644 index 0000000..ef54a40 --- /dev/null +++ b/packages/logger/src/index.ts @@ -0,0 +1,97 @@ +import pino, { type Logger, type LoggerOptions } from 'pino'; + +export type { Logger } from 'pino'; + +export interface CreateLoggerOptions { + name: string; + level?: 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal'; + pretty?: boolean; + context?: Record; +} + +type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal'; + +const LOG_LEVEL_ENV = 'LOG_LEVEL'; +const LOG_FORMAT_ENV = 'LOG_FORMAT'; +const NODE_ENV_ENV = 'NODE_ENV'; + +function getEnvOrDefault(key: string, defaultValue: T, parser?: (v: string) => T): T { + const value = process.env[key]; + if (!value) return defaultValue; + if (parser) { + try { + return parser(value); + } catch { + return defaultValue; + } + } + return value as T; +} + +function isProduction(): boolean { + const nodeEnv = process.env[NODE_ENV_ENV]?.toLowerCase(); + return nodeEnv === 'production'; +} + +function resolveLogLevel(explicit?: LogLevel): LogLevel { + if (explicit) return explicit; + return getEnvOrDefault(LOG_LEVEL_ENV, isProduction() ? 'info' : 'debug'); +} + +function resolvePrettyPrint(explicit?: boolean): boolean { + if (explicit !== undefined) return explicit; + const format = process.env[LOG_FORMAT_ENV]?.toLowerCase(); + if (format === 'json') return false; + if (format === 'pretty') return true; + return !isProduction(); +} + +function createPinoOptions(options: CreateLoggerOptions): LoggerOptions { + const level = resolveLogLevel(options.level); + const pretty = resolvePrettyPrint(options.pretty); + + const pinoOptions: LoggerOptions = { + level, + name: options.name, + timestamp: pino.stdTimeFunctions.isoTime, + formatters: { + level: (label) => ({ level: label }) + } + }; + + if (pretty) { + pinoOptions.transport = { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + singleLine: false + } + }; + } + + return pinoOptions; +} + +export function createLogger(options: CreateLoggerOptions): Logger { + const pinoOptions = createPinoOptions(options); + const baseLogger = pino(pinoOptions); + + if (options.context) { + return baseLogger.child(options.context); + } + + return baseLogger; +} + +export const logLevels = { + trace: 10, + debug: 20, + info: 30, + warn: 40, + error: 50, + fatal: 60 +} as const; + +export { pino }; diff --git a/packages/logger/tsconfig.json b/packages/logger/tsconfig.json new file mode 100644 index 0000000..01d062a --- /dev/null +++ b/packages/logger/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["bun"], + "moduleResolution": "bundler", + "baseUrl": ".", + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "composite": true, + "noEmit": false, + "emitDeclarationOnly": true, + "strict": true, + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} From 029eec4e83b150d3b0c37caad153b68535442a90 Mon Sep 17 00:00:00 2001 From: Khoirul Asfian Date: Tue, 31 Mar 2026 16:41:35 +0700 Subject: [PATCH 2/3] fix: oc review workflow --- .github/workflows/opencode-review.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 9ab5501..1cb589b 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -10,8 +10,8 @@ jobs: permissions: id-token: write contents: read - pull-requests: read - issues: read + pull-requests: write + issues: write steps: - uses: actions/checkout@v6 with: From 1e806bb4399a7a4bfc2f0e4cda66f1ff503a2960 Mon Sep 17 00:00:00 2001 From: Khoirul Asfian Date: Tue, 31 Mar 2026 16:46:13 +0700 Subject: [PATCH 3/3] chore: add changeset --- .changeset/large-carpets-arrive.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/large-carpets-arrive.md diff --git a/.changeset/large-carpets-arrive.md b/.changeset/large-carpets-arrive.md new file mode 100644 index 0000000..0f27e00 --- /dev/null +++ b/.changeset/large-carpets-arrive.md @@ -0,0 +1,5 @@ +--- +'@reafrac/logger': major +--- + +add centralize logging package shared accross the apps