diff --git a/apps/web/src/components/settings/add-feed-dialog.tsx b/apps/web/src/components/settings/add-feed-dialog.tsx index b4d1cb3..42beee5 100644 --- a/apps/web/src/components/settings/add-feed-dialog.tsx +++ b/apps/web/src/components/settings/add-feed-dialog.tsx @@ -12,6 +12,7 @@ import { DialogTitle, DialogTrigger } from '../ui/dialog'; +import { Tabs, TabsList, TabsTrigger, TabsContent } from '../ui/tabs'; import { z } from 'zod/mini'; import { startTransition, useState } from 'react'; import { useServerFn } from '@tanstack/react-start'; @@ -21,6 +22,7 @@ import { Spinner } from '../ui/spinner'; import { Input } from '../ui/input'; import { toastManager } from '../ui/toast'; import { NewFeedPreview } from './new-feed-preview'; +import { FeedDiscovery } from './feed-discovery'; import type { ParsedFeed } from '@reafrac/feed-utils'; import { userFeedQueryOptions } from '@/lib/queries/feed-query'; import { useQuery } from '@tanstack/react-query'; @@ -44,6 +46,21 @@ export function AddFeedDialog() { const { data: feeds, refetch: invalidateFeeds } = useQuery(userFeedQueryOptions(user.id)); + const handleSelectFeedFromDiscovery = async (feedUrl: string) => { + try { + setFeed(null); + setErrors({}); + setIsSearching(true); + const res = await previewFeed({ data: { feedUrl } }); + setFeed({ ...res, feedUrl }); + } catch (error) { + console.error(error); + setErrors({ feedUrl: 'Feed not found!' }); + } finally { + setIsSearching(false); + } + }; + const submitHandler = async (event: React.FormEvent) => { event.preventDefault(); try { @@ -115,41 +132,59 @@ export function AddFeedDialog() { Follow RSS feed, Reddit, Youtube Channel, Newsletters, Podcasts, and more. -
- - Feed URL -
- - - {isSearching ? ( - - ) : ( - - )} -
- - -
+ + + + + Enter URL + + + Discover Feed + + + + +
+ + Feed URL +
+ + + {isSearching ? ( + + ) : ( + + )} +
+ + +
+ + + + + + diff --git a/apps/web/src/components/settings/feed-discovery.tsx b/apps/web/src/components/settings/feed-discovery.tsx new file mode 100644 index 0000000..31f8aba --- /dev/null +++ b/apps/web/src/components/settings/feed-discovery.tsx @@ -0,0 +1,224 @@ +'use client'; + +import { useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { feedDiscoveryQueryOptions, type DiscoveredFeed } from '@/lib/queries/feed-discovery-query'; +import { Input } from '../ui/input'; +import { Button } from '../ui/button'; +import { Spinner } from '../ui/spinner'; +import { PlusIcon, GlobeIcon, SearchIcon } from 'lucide-react'; +import { Field, FieldError, FieldLabel } from '../ui/field'; +import { Form } from '../ui/form'; + +interface FeedDiscoveryProps { + onSelectFeed: (feedUrl: string) => void; +} + +export function FeedDiscovery({ onSelectFeed }: FeedDiscoveryProps) { + const [query, setQuery] = useState(''); + const [searchQuery, setSearchQuery] = useState(''); + const [isSearching, setIsSearching] = useState(false); + + // React Query for discovery + const { + data: feeds, + isLoading, + error, + refetch + } = useQuery({ + ...feedDiscoveryQueryOptions(searchQuery, true), + enabled: false // Don't auto-fetch, only fetch on manual trigger + }); + + const handleSubmit = async (event: React.FormEvent) => { + event.preventDefault(); + if (!query.trim()) return; + + setIsSearching(true); + setSearchQuery(query); + try { + await refetch(); + } finally { + setIsSearching(false); + } + }; + + // Extract seconds from rate limit error message + const getRateLimitSeconds = (errorMessage: string): number | null => { + const match = errorMessage.match(/wait (\d+) seconds/); + return match ? parseInt(match[1]) : null; + }; + + // Limit number of feeds displayed + const MAX_FEEDS = 10; + const displayedFeeds = feeds?.slice(0, MAX_FEEDS); + const hasMoreFeeds = feeds && feeds.length > MAX_FEEDS; + + return ( +
+ {/* Search Form */} +
+ + Website URL +
+ setQuery(e.target.value)} + disabled={isSearching} + /> + + {isSearching ? ( + + ) : ( + + )} +
+ + +
+ + {/* Attribution Link */} +

+ Powered by{' '} + + Feedsearch + +

+ + {/* Error State */} + {error && ( +
+ {error instanceof Error && error.message.includes('Rate limit') ? ( +
+

Too many searches

+

+ Please wait {getRateLimitSeconds(error.message) ?? 60} seconds before searching + again. +

+
+ ) : ( +

+ Failed to discover feeds. Please check the URL and try again. +

+ )} +
+ )} + + {/* Results */} + {displayedFeeds && displayedFeeds.length > 0 && ( +
+
+

+ Found {feeds?.length} feed{feeds && feeds.length !== 1 ? 's' : ''} +

+ {hasMoreFeeds && ( +

Showing top {MAX_FEEDS}

+ )} +
+
+ {displayedFeeds.map((feed) => ( + + ))} +
+
+ )} + + {/* No Results */} + {feeds && feeds.length === 0 && !isSearching && searchQuery && ( +
+

+ No feeds found for this URL. The website may not have RSS feeds or they may be hidden. +

+
+ )} + + {/* Empty State */} + {!searchQuery && !isSearching && ( +
+ +

+ Enter a website URL to discover available RSS feeds +

+
+ )} +
+ ); +} + +interface FeedDiscoveryCardProps { + feed: DiscoveredFeed; + onSelect: (feedUrl: string) => void; +} + +function FeedDiscoveryCard({ feed, onSelect }: FeedDiscoveryCardProps) { + return ( +
onSelect(feed.feedUrl)} + > +
+ {/* Favicon */} +
+ {feed.favicon ? ( + {`${feed.title} + ) : ( +
+ +
+ )} +
+ + {/* Content */} +
+

{feed.title}

+ {feed.description && ( +

{feed.description}

+ )} +
+ {feed.itemCount && {feed.itemCount} items} + {feed.isPodcast && ( + + Podcast + + )} +
+
+ + {/* Action */} + +
+
+ ); +} diff --git a/apps/web/src/components/ui/tabs.tsx b/apps/web/src/components/ui/tabs.tsx index 8bf9a8b..3750720 100644 --- a/apps/web/src/components/ui/tabs.tsx +++ b/apps/web/src/components/ui/tabs.tsx @@ -1,90 +1,80 @@ -"use client"; +'use client'; -import { Tabs as TabsPrimitive } from "@base-ui-components/react/tabs"; +import { Tabs as TabsPrimitive } from '@base-ui-components/react/tabs'; -import { cn } from "@/lib/utils/index"; +import { cn } from '@/lib/utils/index'; -type TabsVariant = "default" | "underline"; +type TabsVariant = 'default' | 'underline'; function Tabs({ className, ...props }: TabsPrimitive.Root.Props) { - return ( - - ); + return ( + + ); } function TabsList({ - variant = "default", - className, - children, - ...props + variant = 'default', + className, + children, + ...props }: TabsPrimitive.List.Props & { - variant?: TabsVariant; + variant?: TabsVariant; }) { - return ( - - {children} - - - ); + return ( + + {children} + + + ); } function TabsTab({ className, ...props }: TabsPrimitive.Tab.Props) { - return ( - - ); + return ( + + ); } function TabsPanel({ className, ...props }: TabsPrimitive.Panel.Props) { - return ( - - ); + return ( + + ); } -export { - Tabs, - TabsList, - TabsTab, - TabsTab as TabsTrigger, - TabsPanel, - TabsPanel as TabsContent, -}; +export { Tabs, TabsList, TabsTab, TabsTab as TabsTrigger, TabsPanel, TabsPanel as TabsContent }; diff --git a/apps/web/src/lib/queries/feed-discovery-query.ts b/apps/web/src/lib/queries/feed-discovery-query.ts new file mode 100644 index 0000000..4ba4d3c --- /dev/null +++ b/apps/web/src/lib/queries/feed-discovery-query.ts @@ -0,0 +1,17 @@ +import { queryOptions } from '@tanstack/react-query'; +import { discoverFeedsServerFn, type DiscoveredFeed } from '../server/feed-discovery-sfn'; + +export const feedDiscoveryQueryOptions = (query: string, skipCrawl?: boolean) => { + return queryOptions({ + queryKey: ['feed-discovery', query, skipCrawl], + queryFn: async () => + discoverFeedsServerFn({ + data: { query, skipCrawl } + }), + enabled: query.length > 0, + staleTime: 5 * 60 * 1000, + gcTime: 10 * 60 * 1000 + }); +}; + +export type { DiscoveredFeed }; diff --git a/apps/web/src/lib/rate-limiter.ts b/apps/web/src/lib/rate-limiter.ts new file mode 100644 index 0000000..3999a16 --- /dev/null +++ b/apps/web/src/lib/rate-limiter.ts @@ -0,0 +1,156 @@ +/** + * Simple in-memory rate limiter using sliding window algorithm + * Used to prevent API spam for feed discovery and other operations + */ + +interface RateLimitEntry { + timestamps: number[]; +} + +export class RateLimiter { + private requests: Map = new Map(); + private cleanupInterval: NodeJS.Timeout | null = null; + private maxEntries: number; + + constructor(cleanupIntervalMs: number = 5 * 60 * 1000, maxEntries: number = 10000) { + this.maxEntries = maxEntries; + this.startCleanup(cleanupIntervalMs); + } + + /** + * Check if a request is allowed based on rate limits + * @param key - Unique identifier for the rate limit (e.g., userId + operation) + * @param maxRequests - Maximum requests allowed in the window + * @param windowMs - Time window in milliseconds + * @returns true if request is allowed, false if rate limit exceeded + */ + checkLimit(key: string, maxRequests: number, windowMs: number): boolean { + const now = Date.now(); + const entry = this.requests.get(key); + + if (!entry) { + // Enforce max entries limit to prevent memory leak + if (this.requests.size >= this.maxEntries) { + this.evictOldestEntry(); + } + this.requests.set(key, { timestamps: [now] }); + return true; + } + + // Filter out timestamps outside the window + const validTimestamps = entry.timestamps.filter((timestamp) => now - timestamp < windowMs); + + if (validTimestamps.length >= maxRequests) { + // Update entry with filtered timestamps + entry.timestamps = validTimestamps; + return false; + } + + // Add current timestamp and update entry + validTimestamps.push(now); + entry.timestamps = validTimestamps; + return true; + } + + /** + * Evict the oldest entry when max entries limit is reached + */ + private evictOldestEntry(): void { + let oldestKey: string | null = null; + let oldestTimestamp = Infinity; + + for (const [key, entry] of this.requests.entries()) { + if (entry.timestamps.length > 0) { + const minTimestamp = Math.min(...entry.timestamps); + if (minTimestamp < oldestTimestamp) { + oldestTimestamp = minTimestamp; + oldestKey = key; + } + } + } + + if (oldestKey) { + this.requests.delete(oldestKey); + } + } + + /** + * Get the time until the rate limit resets (when oldest request expires) + * @param key - Unique identifier for the rate limit + * @param windowMs - Time window in milliseconds + * @returns milliseconds until reset, or 0 if not limited + */ + getResetTime(key: string, windowMs: number): number { + const entry = this.requests.get(key); + if (!entry || entry.timestamps.length === 0) { + return 0; + } + + const now = Date.now(); + const oldestTimestamp = Math.min(...entry.timestamps); + const resetTime = oldestTimestamp + windowMs - now; + + return Math.max(0, resetTime); + } + + /** + * Clear all rate limit entries + */ + clear(): void { + this.requests.clear(); + } + + /** + * Clear rate limit for a specific key + */ + clearKey(key: string): void { + this.requests.delete(key); + } + + /** + * Start periodic cleanup of expired entries + */ + private startCleanup(intervalMs: number): void { + this.cleanupInterval = setInterval(() => { + this.cleanupExpired(); + }, intervalMs); + } + + /** + * Cleanup expired entries to prevent memory leak + */ + private cleanupExpired(): void { + const now = Date.now(); + // Use the maximum window we expect (1 minute for feed discovery) + const maxWindowMs = 60 * 1000; + + for (const [key, entry] of this.requests.entries()) { + const validTimestamps = entry.timestamps.filter((timestamp) => now - timestamp < maxWindowMs); + + if (validTimestamps.length === 0) { + this.requests.delete(key); + } else { + entry.timestamps = validTimestamps; + } + } + } + + /** + * Stop the cleanup interval (for cleanup on server shutdown) + */ + stopCleanup(): void { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + this.cleanupInterval = null; + } + } +} + +// Global rate limiter instance for feed discovery +export const feedDiscoveryRateLimiter = new RateLimiter(); + +// Default rate limit configuration for feed discovery +export const FEED_DISCOVERY_RATE_LIMIT = { + maxRequests: 10, // 10 requests per minute per user + windowMs: 60 * 1000 // 1 minute window +}; diff --git a/apps/web/src/lib/server/feed-discovery-sfn.ts b/apps/web/src/lib/server/feed-discovery-sfn.ts new file mode 100644 index 0000000..cf8b148 --- /dev/null +++ b/apps/web/src/lib/server/feed-discovery-sfn.ts @@ -0,0 +1,212 @@ +import { createServerFn } from '@tanstack/react-start'; +import * as z from 'zod/mini'; +import { authFnMiddleware } from '../middleware/auth-middleware'; +import { sentryMiddleware } from '../middleware/sentry-middleware'; +import * as Sentry from '@sentry/tanstackstart-react'; +import { ofetch } from 'ofetch'; +import { SimpleCache } from '../cache'; +import { feedDiscoveryRateLimiter, FEED_DISCOVERY_RATE_LIMIT } from '../rate-limiter'; + +// Cache for discovered feeds (5 minute TTL) +const feedDiscoveryCache = new SimpleCache(5 * 60 * 1000); + +// Schema for discovered feed +export interface DiscoveredFeed { + feedUrl: string; + siteUrl: string; + title: string; + description: string; + favicon?: string; + itemCount?: number; + lastUpdated?: Date; + isPodcast: boolean; + version: string; +} + +// Feedsearch.dev API response format +interface FeedsearchResponse { + url: string; // Always present in valid responses + title?: string; + description?: string; + site_url?: string; + site_name?: string; + favicon?: string; + is_podcast?: boolean; + item_count?: number; + last_updated?: string; + version?: string; +} + +/** + * Normalize URL - ensure it has a scheme and sanitize input + */ +function normalizeUrl(query: string): string { + // Trim whitespace and remove potentially dangerous characters + const trimmed = query.trim(); + const sanitized = trimmed.replace(/[\r\n\t]/g, ''); + + // If already has scheme, return as is + if (sanitized.startsWith('http://') || sanitized.startsWith('https://')) { + return sanitized; + } + + // Add https:// prefix + return `https://${sanitized}`; +} + +/** + * Parse Feedsearch API response and transform to DiscoveredFeed + */ +function parseFeedsearchResponse(response: FeedsearchResponse[]): DiscoveredFeed[] { + return response.map((feed) => ({ + feedUrl: feed.url, + siteUrl: feed.site_url || '', + title: feed.title || feed.site_name || 'Untitled Feed', + description: feed.description || '', + favicon: feed.favicon, + itemCount: feed.item_count, + lastUpdated: feed.last_updated ? new Date(feed.last_updated) : undefined, + isPodcast: feed.is_podcast || false, + version: feed.version || 'unknown' + })); +} + +/** + * Discover RSS feeds for a given URL/domain using Feedsearch.dev API + * Implements rate limiting and caching to prevent API spam + */ +export const discoverFeedsServerFn = createServerFn({ method: 'GET' }) + .middleware([sentryMiddleware, authFnMiddleware]) + .inputValidator( + z.object({ + query: z.string(), + skipCrawl: z.optional(z.boolean()) + }) + ) + .handler(async ({ data, context }) => { + return Sentry.startSpan( + { + op: 'server_function', + name: 'discoverFeeds' + }, + async (span) => { + let normalizedQuery = ''; + + try { + span.setAttribute('user_id', context.user.id); + span.setAttribute('query', data.query); + + // 1. Rate limit check + const rateLimitKey = `${context.user.id}:feed-discovery`; + const allowed = feedDiscoveryRateLimiter.checkLimit( + rateLimitKey, + FEED_DISCOVERY_RATE_LIMIT.maxRequests, + FEED_DISCOVERY_RATE_LIMIT.windowMs + ); + + if (!allowed) { + const resetTime = feedDiscoveryRateLimiter.getResetTime( + rateLimitKey, + FEED_DISCOVERY_RATE_LIMIT.windowMs + ); + const seconds = Math.ceil(resetTime / 1000); + throw new Error( + `Rate limit exceeded. Please wait ${seconds} seconds before searching again.` + ); + } + + // 2. Normalize query (ensure URL format) + normalizedQuery = normalizeUrl(data.query); + span.setAttribute('normalized_url', normalizedQuery); + + // Validate URL format + try { + z.parse(z.url(), normalizedQuery); + } catch { + throw new Error('Invalid URL format. Please enter a valid website URL.'); + } + + // 3. Check cache + const cacheKey = `feed-discovery:${normalizedQuery}`; + const cachedFeeds = feedDiscoveryCache.get(cacheKey); + + if (cachedFeeds) { + span.setAttribute('cache_hit', true); + span.setAttribute('feeds_count', cachedFeeds.length); + span.setAttribute('status', 'success'); + return cachedFeeds; + } + + span.setAttribute('cache_hit', false); + + // 4. Call Feedsearch API + const apiUrl = process.env.FEEDSEARCH_API_URL || 'https://feedsearch.dev/api/v1/search'; + + span.setAttribute('api_url', apiUrl); + + const response = await ofetch(apiUrl, { + params: { + url: normalizedQuery, + info: true, + favicon: false, + skip_crawl: data.skipCrawl ?? false + }, + timeout: 10000 // 10 second timeout + }); + + // 5. Parse and transform response + const discoveredFeeds = parseFeedsearchResponse(response); + + span.setAttribute('feeds_count', discoveredFeeds.length); + + // 6. Cache results (5 minute TTL) + feedDiscoveryCache.set(cacheKey, discoveredFeeds); + + // 7. Return feeds + span.setAttribute('status', 'success'); + return discoveredFeeds; + } catch (error) { + span.setAttribute('status', 'error'); + + // Handle different error types with generic messages for users + let userMessage = 'Failed to discover feeds. Please check the URL and try again.'; + let shouldCaptureToSentry = true; + + // Don't capture rate limit errors to Sentry (expected behavior) + if (error instanceof Error && error.message.includes('Rate limit')) { + shouldCaptureToSentry = false; + userMessage = error.message; // Preserve rate limit message with seconds + } + // Handle timeout errors + else if (error instanceof Error && error.message.includes('timeout')) { + userMessage = 'Request timed out. Please try again.'; + } + // Handle network errors + else if (error instanceof Error && error.message.includes('network')) { + userMessage = 'Network error. Please check your connection and try again.'; + } + // Handle invalid URL errors + else if (error instanceof Error && error.message.includes('Invalid URL')) { + userMessage = error.message; // Preserve user-friendly URL error + } + + // Capture to Sentry if needed + if (shouldCaptureToSentry && error instanceof Error) { + Sentry.captureException(error, { + tags: { function: 'discoverFeeds' }, + extra: { + userId: context.user.id, + query: data.query, + normalizedQuery, + errorMessage: error.message, + errorStack: error.stack + } + }); + } + + // Throw error with user-friendly message + throw new Error(userMessage); + } + } + ); + });