diff --git a/src/api/index.ts b/src/api/index.ts index 942bb6a..19e3f15 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -2,13 +2,21 @@ import { Hono } from 'hono'; import { createLanguagesCard } from '../cards/languages'; import { createRepoCard } from '../cards/repo'; import { createStatsCard } from '../cards/stats'; -import { fetchRepo, fetchTopLanguages, fetchUserStats } from '../fetchers/github'; +import { fetchRepo, fetchTopLanguages, fetchUserStats, setObservability } from '../fetchers/github'; import type { Env, LanguagesCardOptions, RepoCardOptions, StatsCardOptions } from '../types'; import { AnalyticsCollector } from '../utils/analytics'; import { CACHE_TTL_EXPORT, CacheManager, getCacheHeaders } from '../utils/cache'; +import { createObservability } from '../utils/observability'; const api = new Hono<{ Bindings: Env }>(); +// Initialize observability for GitHub API tracking +api.use('*', async (c, next) => { + const observability = createObservability(c); + setObservability(observability); + await next(); +}); + const parseBoolean = (value: string | undefined): boolean | undefined => { if (value === 'true') return true; if (value === 'false') return false; diff --git a/src/fetchers/github.ts b/src/fetchers/github.ts index 6970856..c45289a 100644 --- a/src/fetchers/github.ts +++ b/src/fetchers/github.ts @@ -1,8 +1,48 @@ import type { Env, GitHubRepo, GitHubUser, LanguageEdge, LanguageStats, UserStats } from '../types'; import { calculateRank } from '../utils'; +import type { Observability } from '../utils/observability'; const GITHUB_API = 'https://api.github.com/graphql'; +interface GitHubApiMetrics { + endpoint: string; + status: number; + duration: number; + rateLimitRemaining?: number; + rateLimitReset?: number; +} + +let observabilityInstance: Observability | null = null; + +export function setObservability(obs: Observability): void { + observabilityInstance = obs; +} + +function recordGitHubMetrics(metrics: GitHubApiMetrics): void { + if (observabilityInstance) { + observabilityInstance.recordGitHubApiCall( + metrics.endpoint, + metrics.status, + metrics.duration, + metrics.rateLimitRemaining + ); + } + + // Always log rate limit warnings + if (metrics.rateLimitRemaining !== undefined && metrics.rateLimitRemaining < 100) { + console.log( + JSON.stringify({ + level: 'warn', + type: 'github_rate_limit', + timestamp: new Date().toISOString(), + remaining: metrics.rateLimitRemaining, + reset: metrics.rateLimitReset, + endpoint: metrics.endpoint, + }) + ); + } +} + const USER_STATS_QUERY = ` query userStats($login: String!) { user(login: $login) { @@ -81,33 +121,57 @@ query repoInfo($owner: String!, $name: String!) { const fetchGitHub = async ( query: string, variables: Record, - token?: string + token?: string, + endpoint = 'graphql' ): Promise => { if (!token) { throw new Error('GitHub token is required'); } - const response = await fetch(GITHUB_API, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${token}`, - 'User-Agent': 'devcard', - }, - body: JSON.stringify({ query, variables }), - }); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); - } + const start = Date.now(); + let status = 0; + let rateLimitRemaining: number | undefined; + let rateLimitReset: number | undefined; - const data = (await response.json()) as { data: T; errors?: Array<{ message: string }> }; + try { + const response = await fetch(GITHUB_API, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + 'User-Agent': 'devcard', + }, + body: JSON.stringify({ query, variables }), + }); - if (data.errors) { - throw new Error(`GraphQL error: ${data.errors[0].message}`); - } + status = response.status; + + // Extract rate limit headers + rateLimitRemaining = + parseInt(response.headers.get('X-RateLimit-Remaining') || '', 10) || undefined; + rateLimitReset = parseInt(response.headers.get('X-RateLimit-Reset') || '', 10) || undefined; + + if (!response.ok) { + throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); + } - return data.data; + const data = (await response.json()) as { data: T; errors?: Array<{ message: string }> }; + + if (data.errors) { + throw new Error(`GraphQL error: ${data.errors[0].message}`); + } + + return data.data; + } finally { + const duration = Date.now() - start; + recordGitHubMetrics({ + endpoint, + status: status || 0, + duration, + rateLimitRemaining, + rateLimitReset, + }); + } }; export const fetchUserStats = async ( @@ -118,7 +182,8 @@ export const fetchUserStats = async ( const data = await fetchGitHub<{ user: GitHubUser }>( USER_STATS_QUERY, { login: username }, - env.GITHUB_TOKEN + env.GITHUB_TOKEN, + 'user_stats' ); const user = data.user; @@ -169,7 +234,7 @@ export const fetchTopLanguages = async ( }>; }; }; - }>(TOP_LANGUAGES_QUERY, { login: username, first: 100 }, env.GITHUB_TOKEN); + }>(TOP_LANGUAGES_QUERY, { login: username, first: 100 }, env.GITHUB_TOKEN, 'top_languages'); if (!data.user) { throw new Error(`User "${username}" not found`); @@ -212,7 +277,8 @@ export const fetchRepo = async (owner: string, name: string, env: Env): Promise< const data = await fetchGitHub<{ repository: GitHubRepo }>( REPO_QUERY, { owner, name }, - env.GITHUB_TOKEN + env.GITHUB_TOKEN, + 'repo_info' ); if (!data.repository) { diff --git a/src/middleware/logger.ts b/src/middleware/logger.ts index 7af7496..e855384 100644 --- a/src/middleware/logger.ts +++ b/src/middleware/logger.ts @@ -1,18 +1,34 @@ import type { Context, Next } from 'hono'; import type { Env } from '../types'; +import { createObservability, generateSpanId, generateTraceId } from '../utils/observability'; interface LogEntry { timestamp: string; + level: 'info' | 'warn' | 'error'; + type: 'request'; method: string; path: string; status: number; duration: number; - rayId?: string; - ip?: string; - country?: string; - userAgent?: string; - cacheStatus?: string; - error?: string; + trace: { + traceId: string; + spanId: string; + rayId?: string; + }; + client: { + ip?: string; + country?: string; + userAgent?: string; + colo?: string; + }; + cache: { + status?: string; + hit: boolean; + }; + error?: { + message: string; + type?: string; + }; } export const loggerMiddleware = async ( @@ -21,41 +37,118 @@ export const loggerMiddleware = async ( ): Promise => { const start = Date.now(); const method = c.req.method; - const path = new URL(c.req.url).pathname; + const url = new URL(c.req.url); + const path = url.pathname; // Cloudflare-specific headers const rayId = c.req.header('CF-Ray'); const ip = c.req.header('CF-Connecting-IP'); const country = c.req.header('CF-IPCountry'); const userAgent = c.req.header('User-Agent'); + const colo = c.req.header('CF-IPCity') || c.req.header('CF-Ray')?.split('-')[1]; - let error: string | undefined; + // W3C Trace Context + const incomingTraceParent = c.req.header('traceparent'); + let traceId: string; + let spanId: string; + + if (incomingTraceParent) { + const parts = incomingTraceParent.split('-'); + traceId = parts[1] || generateTraceId(); + spanId = generateSpanId(); + } else { + traceId = rayId?.split('-')[0] || generateTraceId(); + spanId = generateSpanId(); + } + + // Add trace context to response headers + c.header('X-Trace-Id', traceId); + c.header('X-Span-Id', spanId); + if (rayId) { + c.header('X-Ray-Id', rayId); + } + + // Create observability instance + const observability = createObservability(c); + + let errorMessage: string | undefined; + let errorType: string | undefined; try { await next(); } catch (e) { - error = e instanceof Error ? e.message : 'Unknown error'; + errorMessage = e instanceof Error ? e.message : 'Unknown error'; + errorType = e instanceof Error ? e.constructor.name : 'UnknownError'; + + // Record error to Analytics Engine + observability.recordError({ + path, + errorType: errorType, + errorMessage: errorMessage, + status: 500, + }); + throw e; } finally { const duration = Date.now() - start; const status = c.res?.status ?? 500; - const cacheStatus = c.res?.headers.get('CF-Cache-Status') ?? undefined; + const cacheStatus = + c.res?.headers.get('CF-Cache-Status') ?? c.res?.headers.get('X-Cache-Status'); + const cacheHit = cacheStatus === 'HIT'; + + // Determine log level based on status + let level: 'info' | 'warn' | 'error' = 'info'; + if (status >= 500) { + level = 'error'; + } else if (status >= 400) { + level = 'warn'; + } const logEntry: LogEntry = { timestamp: new Date().toISOString(), + level, + type: 'request', method, path, status, duration, - rayId, - ip, - country, - userAgent, - cacheStatus, - error, + trace: { + traceId, + spanId, + rayId, + }, + client: { + ip, + country, + userAgent, + colo, + }, + cache: { + status: cacheStatus ?? undefined, + hit: cacheHit, + }, }; + if (errorMessage) { + logEntry.error = { + message: errorMessage, + type: errorType, + }; + } + // Output structured JSON log for Cloudflare Workers console.log(JSON.stringify(logEntry)); + + // Record to Analytics Engine + observability.recordRequest({ + method, + path, + status, + duration, + cacheHit, + country, + endpoint: path.startsWith('/api/') ? path.split('?')[0] : undefined, + username: url.searchParams.get('username') ?? undefined, + }); } }; diff --git a/src/types.ts b/src/types.ts index a66f6cc..c1a36fa 100644 --- a/src/types.ts +++ b/src/types.ts @@ -148,4 +148,20 @@ export interface Env { GITHUB_TOKEN?: string; CACHE?: KVNamespace; ASSETS: Fetcher; + ANALYTICS?: AnalyticsEngineDataset; + LOG_LEVEL?: string; +} + +// Analytics Engine types +export interface AnalyticsEngineDataset { + writeDataPoint(event: AnalyticsEngineDataPoint): void; +} + +export interface AnalyticsEngineDataPoint { + // Blobs are indexed string fields (max 20, each max 1024 bytes) + blobs?: ArrayBuffer[]; + // Doubles are numeric fields for aggregation (max 20) + doubles?: number[]; + // Indexes are used for filtering/grouping (max 1, max 96 bytes) + indexes?: [string]; } diff --git a/src/utils/monitoring.ts b/src/utils/monitoring.ts index a1fa5cd..8661ae1 100644 --- a/src/utils/monitoring.ts +++ b/src/utils/monitoring.ts @@ -1,5 +1,6 @@ import type { Context } from 'hono'; import type { Env } from '../types'; +import { createObservability, type Observability } from './observability'; interface MetricData { endpoint: string; @@ -11,46 +12,119 @@ interface MetricData { timestamp: number; } +interface AggregatedMetrics { + count: number; + errors: number; + avgDuration: number; + totalDuration: number; + cacheHits: number; + cacheMisses: number; +} + export class Monitor { + private static observability: Observability | null = null; + + static init(c: Context<{ Bindings: Env }>): void { + Monitor.observability = createObservability(c); + } + static async logMetric(c: Context<{ Bindings: Env }>, data: Partial) { - // In production, you might want to send this to an analytics service - // For now, we'll just log to console in development - // Log in development (when running locally) - const isDev = c.req.url.includes('localhost') || c.req.url.includes('127.0.0.1'); - if (isDev) { - console.log('Metric:', { + // Initialize observability if not already done + if (!Monitor.observability) { + Monitor.init(c); + } + + // Log structured metric + console.log( + JSON.stringify({ + level: 'info', + type: 'metric', + timestamp: new Date().toISOString(), ...data, - timestamp: Date.now(), - ip: c.req.header('CF-Connecting-IP') || 'unknown', - country: c.req.header('CF-IPCountry') || 'unknown', + client: { + ip: c.req.header('CF-Connecting-IP') || 'unknown', + country: c.req.header('CF-IPCountry') || 'unknown', + }, + }) + ); + + // Record to Analytics Engine if available + if (Monitor.observability && data.endpoint && data.status !== undefined) { + Monitor.observability.recordRequest({ + method: c.req.method, + path: data.endpoint, + status: data.status, + duration: data.duration || 0, + cacheHit: data.cacheHit || false, + country: c.req.header('CF-IPCountry') || undefined, + endpoint: data.endpoint, + username: data.username, }); } - // Optionally store aggregated metrics in KV for analysis + // Store aggregated metrics in KV for historical analysis if (c.env.CACHE) { const key = `metrics:${Math.floor(Date.now() / 3600000)}`; // Hourly aggregation try { - const existing = (await c.env.CACHE.get<{ count: number; errors: number }>(key, { - type: 'json', - })) || { count: 0, errors: 0 }; + const existing = + (await c.env.CACHE.get(key, { type: 'json' })) || + ({ + count: 0, + errors: 0, + avgDuration: 0, + totalDuration: 0, + cacheHits: 0, + cacheMisses: 0, + } as AggregatedMetrics); + + const newCount = existing.count + 1; + const newTotalDuration = existing.totalDuration + (data.duration || 0); + await c.env.CACHE.put( key, JSON.stringify({ - count: existing.count + 1, + count: newCount, errors: existing.errors + (data.error ? 1 : 0), + avgDuration: newTotalDuration / newCount, + totalDuration: newTotalDuration, + cacheHits: existing.cacheHits + (data.cacheHit ? 1 : 0), + cacheMisses: existing.cacheMisses + (data.cacheHit ? 0 : 1), }), { expirationTtl: 86400 } // Keep for 24 hours ); } catch (e) { - console.error('Failed to store metrics:', e); + console.error( + JSON.stringify({ + level: 'error', + type: 'metric_storage_error', + timestamp: new Date().toISOString(), + error: e instanceof Error ? e.message : 'Unknown error', + }) + ); } } } - static trackTiming(_name: string): { end: () => number } { + static trackTiming(name: string): { end: () => number; log: () => void } { const start = Date.now(); return { end: () => Date.now() - start, + log: () => { + const duration = Date.now() - start; + console.log( + JSON.stringify({ + level: 'debug', + type: 'timing', + timestamp: new Date().toISOString(), + name, + duration, + }) + ); + }, }; } + + static getObservability(): Observability | null { + return Monitor.observability; + } } diff --git a/src/utils/observability.ts b/src/utils/observability.ts new file mode 100644 index 0000000..3005f15 --- /dev/null +++ b/src/utils/observability.ts @@ -0,0 +1,252 @@ +import type { Context } from 'hono'; +import type { Env } from '../types'; + +type LogLevel = 'debug' | 'info' | 'warn' | 'error'; + +interface LogContext { + requestId?: string; + traceId?: string; + spanId?: string; + [key: string]: unknown; +} + +interface StructuredLog { + level: LogLevel; + message: string; + timestamp: string; + context?: LogContext; + error?: { + name: string; + message: string; + stack?: string; + }; +} + +const LOG_LEVEL_PRIORITY: Record = { + debug: 0, + info: 1, + warn: 2, + error: 3, +}; + +function shouldLog(currentLevel: LogLevel, configuredLevel: string): boolean { + const configured = (configuredLevel?.toLowerCase() || 'info') as LogLevel; + return LOG_LEVEL_PRIORITY[currentLevel] >= LOG_LEVEL_PRIORITY[configured]; +} + +function createTextEncoder(): TextEncoder { + return new TextEncoder(); +} + +export class Logger { + private context: LogContext; + private logLevel: string; + + constructor(logLevel = 'info', context: LogContext = {}) { + this.logLevel = logLevel; + this.context = context; + } + + private log(level: LogLevel, message: string, extra?: Record): void { + if (!shouldLog(level, this.logLevel)) { + return; + } + + const logEntry: StructuredLog = { + level, + message, + timestamp: new Date().toISOString(), + context: { ...this.context, ...extra }, + }; + + console.log(JSON.stringify(logEntry)); + } + + debug(message: string, extra?: Record): void { + this.log('debug', message, extra); + } + + info(message: string, extra?: Record): void { + this.log('info', message, extra); + } + + warn(message: string, extra?: Record): void { + this.log('warn', message, extra); + } + + error(message: string, error?: Error, extra?: Record): void { + const logEntry: StructuredLog = { + level: 'error', + message, + timestamp: new Date().toISOString(), + context: { ...this.context, ...extra }, + }; + + if (error) { + logEntry.error = { + name: error.name, + message: error.message, + stack: error.stack, + }; + } + + console.log(JSON.stringify(logEntry)); + } + + withContext(additionalContext: LogContext): Logger { + return new Logger(this.logLevel, { ...this.context, ...additionalContext }); + } +} + +export interface RequestMetrics { + method: string; + path: string; + status: number; + duration: number; + cacheHit: boolean; + country?: string; + endpoint?: string; + username?: string; +} + +export interface ErrorMetrics { + path: string; + errorType: string; + errorMessage: string; + status: number; +} + +export class Observability { + private analytics: Env['ANALYTICS']; + private logger: Logger; + private encoder: TextEncoder; + + constructor(env: Env, logger?: Logger) { + this.analytics = env.ANALYTICS; + this.logger = logger || new Logger(env.LOG_LEVEL); + this.encoder = createTextEncoder(); + } + + private stringToBlob(str: string, maxLength = 1024): ArrayBuffer { + const truncated = str.slice(0, maxLength); + const encoded = this.encoder.encode(truncated); + // Create a new ArrayBuffer and copy the data to ensure correct type + const buffer = new ArrayBuffer(encoded.byteLength); + new Uint8Array(buffer).set(encoded); + return buffer; + } + + recordRequest(metrics: RequestMetrics): void { + if (!this.analytics) { + this.logger.debug('Analytics Engine not available, skipping metric recording'); + return; + } + + try { + this.analytics.writeDataPoint({ + indexes: [metrics.path.slice(0, 96)], + blobs: [ + this.stringToBlob(metrics.method), + this.stringToBlob(metrics.endpoint || metrics.path), + this.stringToBlob(metrics.country || 'unknown'), + this.stringToBlob(metrics.username || 'anonymous'), + this.stringToBlob(metrics.cacheHit ? 'HIT' : 'MISS'), + ], + doubles: [metrics.status, metrics.duration, metrics.cacheHit ? 1 : 0, Date.now()], + }); + } catch (error) { + this.logger.error('Failed to record request metrics', error as Error); + } + } + + recordError(metrics: ErrorMetrics): void { + if (!this.analytics) { + return; + } + + try { + this.analytics.writeDataPoint({ + indexes: ['error'], + blobs: [ + this.stringToBlob(metrics.path), + this.stringToBlob(metrics.errorType), + this.stringToBlob(metrics.errorMessage), + ], + doubles: [metrics.status, Date.now()], + }); + } catch (error) { + this.logger.error('Failed to record error metrics', error as Error); + } + } + + recordCacheOperation( + operation: 'get' | 'put', + key: string, + hit: boolean, + duration: number + ): void { + if (!this.analytics) { + return; + } + + try { + this.analytics.writeDataPoint({ + indexes: ['cache'], + blobs: [this.stringToBlob(operation), this.stringToBlob(key.slice(0, 100))], + doubles: [hit ? 1 : 0, duration, Date.now()], + }); + } catch (error) { + this.logger.error('Failed to record cache metrics', error as Error); + } + } + + recordGitHubApiCall( + endpoint: string, + status: number, + duration: number, + rateLimitRemaining?: number + ): void { + if (!this.analytics) { + return; + } + + try { + this.analytics.writeDataPoint({ + indexes: ['github_api'], + blobs: [this.stringToBlob(endpoint)], + doubles: [status, duration, rateLimitRemaining ?? -1, Date.now()], + }); + } catch (error) { + this.logger.error('Failed to record GitHub API metrics', error as Error); + } + } + + getLogger(): Logger { + return this.logger; + } +} + +export function createRequestLogger(c: Context<{ Bindings: Env }>): Logger { + const rayId = c.req.header('CF-Ray') || crypto.randomUUID(); + const traceId = c.req.header('X-Trace-Id') || rayId; + + return new Logger(c.env.LOG_LEVEL, { + requestId: rayId, + traceId, + path: new URL(c.req.url).pathname, + method: c.req.method, + }); +} + +export function createObservability(c: Context<{ Bindings: Env }>): Observability { + const logger = createRequestLogger(c); + return new Observability(c.env, logger); +} + +export function generateTraceId(): string { + return crypto.randomUUID().replace(/-/g, ''); +} + +export function generateSpanId(): string { + return crypto.randomUUID().replace(/-/g, '').slice(0, 16); +} diff --git a/wrangler.toml b/wrangler.toml index 636dfd3..0a0ffd2 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,15 +1,47 @@ name = "devcard" main = "src/index.ts" compatibility_date = "2025-07-18" +compatibility_flags = ["nodejs_compat"] +# ============================================ +# Observability Settings +# ============================================ +[observability] +enabled = true + +# Logs configuration - enables Workers Logs in Cloudflare Dashboard +[observability.logs] +enabled = true +# Sample rate: 1.0 = 100% of requests logged (adjust for high-traffic) +head_sampling_rate = 1.0 +# Invocation logs capture console.log output +invocation_logs = true + +# ============================================ +# Static Assets +# ============================================ [assets] directory = "./dist" +# ============================================ +# Environment Variables +# ============================================ [vars] # Set your GitHub token as a secret: wrangler secret put GITHUB_TOKEN # GITHUB_TOKEN = "" +LOG_LEVEL = "info" +# ============================================ +# KV Namespaces +# ============================================ [[kv_namespaces]] binding = "CACHE" id = "ee1f8586ea9e4fcfa28b82870a1a3550" preview_id = "607a7597605c4984aae071103c16a817" + +# ============================================ +# Analytics Engine (for metrics and observability) +# ============================================ +[[analytics_engine_datasets]] +binding = "ANALYTICS" +dataset = "devcard_metrics"