diff --git a/core/feed.ts b/core/feed.ts index 27594738..f97dd0b8 100644 --- a/core/feed.ts +++ b/core/feed.ts @@ -28,6 +28,7 @@ export type FeedValue = { lastPublishedAt?: number loader: LoaderName reading: 'fast' | 'slow' + refreshedAt?: number slowReader?: UsefulReaderName title: string url: string @@ -99,7 +100,12 @@ export function getFeedLatestPosts( feed: FeedValue, task = createDownloadTask() ): PostsList { - return loaders[feed.loader].getPosts(task, feed.url) + return loaders[feed.loader].getPosts( + task, + feed.url, + undefined, + feed.refreshedAt + ) } let testFeedId = 0 diff --git a/core/lib/download.ts b/core/lib/download.ts index f0292344..cd82cf2a 100644 --- a/core/lib/download.ts +++ b/core/lib/download.ts @@ -222,7 +222,7 @@ export function createDownloadTask( if (controller.signal.aborted) { throw new DOMException('', 'AbortError') } - if (!response.ok) { + if (!response.ok && response.status !== 304) { throw new HTTPStatusError( response.status, url, @@ -246,6 +246,7 @@ export function createDownloadTask( if (controller.signal.aborted) { throw new DOMException('', 'AbortError') } + return createTextResponse(text, { headers: response.headers, redirected: response.redirected, diff --git a/core/loader/atom.ts b/core/loader/atom.ts index 1ec61ea0..6c66ab73 100644 --- a/core/loader/atom.ts +++ b/core/loader/atom.ts @@ -7,6 +7,7 @@ import { type OriginPost, type PostMedia, stringifyMedia } from '../post.ts' import { createPostsList, type PostsListResult } from '../posts-list.ts' import { buildFullURL, + fetchIfModified, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -122,9 +123,12 @@ function parseFeed( async function loadFeed( task: DownloadTask, - url: string + url: string, + refreshedAt?: number ): Promise { - return parseFeed(task, await task.text(url)) + return fetchIfModified(task, url, refreshedAt, response => + parseFeed(task, response) + ) } export const atom: Loader = { @@ -143,11 +147,11 @@ export const atom: Loader = { } }, - getPosts(task, url, text) { + getPosts(task, url, text, refreshedAt) { if (text) { return createPostsList(() => parseFeed(task, text)) } else { - return createPostsList(() => loadFeed(task, url)) + return createPostsList(() => loadFeed(task, url, refreshedAt)) } }, diff --git a/core/loader/common.ts b/core/loader/common.ts index d479c316..79e9c1b8 100644 --- a/core/loader/common.ts +++ b/core/loader/common.ts @@ -1,7 +1,7 @@ import type { FeedValue } from '../feed.ts' import type { DownloadTask, TextResponse } from '../lib/download.ts' import type { PostMedia } from '../post.ts' -import type { PostsList } from '../posts-list.ts' +import type { PostsList, PostsListResult } from '../posts-list.ts' export type Loader = { /** @@ -17,7 +17,12 @@ export type Loader = { * Task is a way to combine multiple HTTP requests (for instance, during * the feed search/preview) to cancel all of them fast. */ - getPosts(task: DownloadTask, url: string, text?: TextResponse): PostsList + getPosts( + task: DownloadTask, + url: string, + text?: TextResponse, + refreshedAt?: number + ): PostsList /** * Get source data of post from loader's API for debug purposes. @@ -183,3 +188,20 @@ export function findMediaInText( } satisfies PostMedia }) } + +/** + * Prevent heavy loading/parsing by `If-Modified-Since` header. + */ +export async function fetchIfModified( + task: DownloadTask, + url: string, + refreshedAt: number | undefined, + parseCb: (response: TextResponse) => PostsListResult +): Promise { + let headers = refreshedAt + ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } + : undefined + let response = await task.text(url, { headers }) + if (response.status === 304) return [[], undefined] + return parseCb(response) +} diff --git a/core/loader/json-feed.ts b/core/loader/json-feed.ts index 99c7256c..af5b5453 100644 --- a/core/loader/json-feed.ts +++ b/core/loader/json-feed.ts @@ -3,6 +3,7 @@ import { createDownloadTask, type TextResponse } from '../lib/download.ts' import { type OriginPost, type PostMedia, stringifyMedia } from '../post.ts' import { createPostsList } from '../posts-list.ts' import { + fetchIfModified, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -142,13 +143,16 @@ export const jsonFeed: Loader = { return [...linksByType, ...findAnchorHrefs(text, /feed\.json/i)] }, - getPosts(task, url, text) { + getPosts(task, url, text, refreshedAt) { if (text) { return createPostsList(() => [parsePosts(text), undefined]) } else { - return createPostsList(async () => { - return [parsePosts(await task.text(url)), undefined] - }) + return createPostsList(() => + fetchIfModified(task, url, refreshedAt, response => [ + parsePosts(response), + undefined + ]) + ) } }, diff --git a/core/loader/rss.ts b/core/loader/rss.ts index 6b43b1ec..f3f9c290 100644 --- a/core/loader/rss.ts +++ b/core/loader/rss.ts @@ -3,6 +3,7 @@ import { type OriginPost, type PostMedia, stringifyMedia } from '../post.ts' import { createPostsList } from '../posts-list.ts' import { findMRSS } from './atom.ts' import { + fetchIfModified, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -60,13 +61,16 @@ export const rss: Loader = { ] }, - getPosts(task, url, text) { + getPosts(task, url, text, refreshedAt) { if (text) { return createPostsList(() => [parsePosts(text), undefined]) } else { - return createPostsList(async () => { - return [parsePosts(await task.text(url)), undefined] - }) + return createPostsList(() => + fetchIfModified(task, url, refreshedAt, response => [ + parsePosts(response), + undefined + ]) + ) } }, diff --git a/core/refresh.ts b/core/refresh.ts index 97171116..87c18a22 100644 --- a/core/refresh.ts +++ b/core/refresh.ts @@ -115,6 +115,9 @@ async function checkForNextPage( if (!getFeed(feed.id).deleted) { await addPosts(feed, pages.get().list) } + await changeFeed(feed.id, { + refreshedAt: Math.round(Date.now() / 1000) + }) increaseKey(refreshStatistics, 'processedFeeds') } } diff --git a/core/request.ts b/core/request.ts index 20432cc5..6c3ad2dd 100644 --- a/core/request.ts +++ b/core/request.ts @@ -11,20 +11,20 @@ export function setRequestMethod(method: RequestMethod): void { } export interface RequestWaiter { - (status: number, body?: string, contentType?: string): Promise + (status: number, body?: null | string, contentType?: string): Promise aborted?: true } export interface RequestMock { andFail(): void - andRespond(status: number, body?: string, contentType?: string): void + andRespond(status: number, body?: null | string, contentType?: string): void andWait(): RequestWaiter } interface RequestExpect { contentType: string error: boolean - response: string + response: null | string status: number url: string wait: Promise diff --git a/core/test/loader/atom.test.ts b/core/test/loader/atom.test.ts index 8e166aa7..478b59d8 100644 --- a/core/test/loader/atom.test.ts +++ b/core/test/loader/atom.test.ts @@ -11,6 +11,7 @@ import { loaders, mockRequest, parseMedia, + setRequestMethod, setupEnvironment, testFeed, type TextResponse @@ -680,3 +681,56 @@ test('returns post source', async () => { undefined ) }) + +test('handles conditional request when server returns 304', async () => { + let callCount = 0 + let capturedOpts: RequestInit | undefined + + // Mock initial and refreshing requests + setRequestMethod((url, opts) => { + if (++callCount === 1) { + return Promise.resolve( + new Response( + ` + + Feed + + Test Post + test-post + + + `, + { + headers: { + 'Content-Type': 'application/atom+xml', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + } + ) + ) + } else { + capturedOpts = opts + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let task = createDownloadTask() + + // Initial request + let page1 = loaders.atom.getPosts(task, 'https://example.com/news/') + await page1.loading + equal(page1.get().list.length, 1) + + // Refreshing request + let page2 = loaders.atom.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + await page2.loading + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' + }) + equal(page2.get().list.length, 0) +}) diff --git a/core/test/loader/json-feed.test.ts b/core/test/loader/json-feed.test.ts index 704877cd..bd51b0c2 100644 --- a/core/test/loader/json-feed.test.ts +++ b/core/test/loader/json-feed.test.ts @@ -10,6 +10,7 @@ import { expectRequest, loaders, mockRequest, + setRequestMethod, setupEnvironment, testFeed, type TextResponse @@ -628,3 +629,58 @@ test('returns post source', async () => { undefined ) }) + +test('handles conditional request when server returns 304', async () => { + let callCount = 0 + let capturedOpts: RequestInit | undefined + + // Mock initial and refreshing requests + setRequestMethod((url, opts) => { + if (++callCount === 1) { + return Promise.resolve( + new Response( + JSON.stringify({ + items: [ + { + id: 'test-post', + title: 'Test Post', + url: 'https://example.com/1' + } + ], + title: 'Feed', + version: 'https://jsonfeed.org/version/1.1' + }), + { + headers: { + 'Content-Type': 'application/json', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + } + ) + ) + } else { + capturedOpts = opts + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let task = createDownloadTask() + + // Initial request + let page1 = loaders.jsonFeed.getPosts(task, 'https://example.com/news/') + await page1.loading + equal(page1.get().list.length, 1) + + // Refreshing request + let page2 = loaders.jsonFeed.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + await page2.loading + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' + }) + equal(page2.get().list.length, 0) +}) diff --git a/core/test/loader/rss.test.ts b/core/test/loader/rss.test.ts index 051c4525..3d99a317 100644 --- a/core/test/loader/rss.test.ts +++ b/core/test/loader/rss.test.ts @@ -10,6 +10,7 @@ import { expectRequest, loaders, mockRequest, + setRequestMethod, testFeed, type TextResponse } from '../../index.ts' @@ -449,3 +450,57 @@ test('returns post source', async () => { undefined ) }) + +test('handles conditional request when server returns 304', async () => { + let callCount = 0 + let capturedOpts: RequestInit | undefined + + // Mock initial and refreshing requests + setRequestMethod((url, opts) => { + if (++callCount === 1) { + return Promise.resolve( + new Response( + ` + + + Feed + + Test Post + https://example.com/1 + + + `, + { + headers: { + 'Content-Type': 'application/rss+xml', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + } + ) + ) + } else { + capturedOpts = opts + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let task = createDownloadTask() + + // Initial request + let page1 = loaders.rss.getPosts(task, 'https://example.com/news/') + await page1.loading + equal(page1.get().list.length, 1) + + // Refreshing request + let page2 = loaders.rss.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + await page2.loading + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' + }) + equal(page2.get().list.length, 0) +}) diff --git a/core/test/refresh.test.ts b/core/test/refresh.test.ts index 6c0c62d8..868721fb 100644 --- a/core/test/refresh.test.ts +++ b/core/test/refresh.test.ts @@ -1,3 +1,5 @@ +import './dom-parser.ts' + import { loadValue } from '@logux/client' import { restoreAll, spyOn } from 'nanospy' import { deepEqual, equal, fail, ok } from 'node:assert/strict' @@ -25,6 +27,7 @@ import { refreshProgress, refreshStatistics, refreshStatus, + setRequestMethod, stopRefreshing, testFeed, waitLoading @@ -522,3 +525,63 @@ test('retries with some delay', async () => { totalFeeds: 3 }) }) + +test('sends If-Modified-Since equal to refreshedAt of updated feed', async () => { + let feedId = await addFeed(testFeed()) + + let requestCount = 0 + let capturedOpts1: RequestInit | undefined + let capturedOpts2: RequestInit | undefined + + setRequestMethod((url, opts) => { + if (++requestCount === 1) { + capturedOpts1 = opts ?? {} + return Promise.resolve( + new Response( + ` + + Feed + `, + { + headers: { + 'Content-Type': 'application/rss+xml', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + }, + status: 200 + } + ) + ) + } else { + capturedOpts2 = opts ?? {} + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let getHeaders = (opts?: RequestInit): Record => + (opts?.headers ?? {}) as Record + + equal(requestCount, 0) + let feed0 = await loadValue(getFeed(feedId)) + ok(!feed0!.refreshedAt) + + refreshPosts() + await setTimeout(10) + + // First update: store current timestamp as `refreshedAt` + equal(requestCount, 1) + equal(getHeaders(capturedOpts1)['If-Modified-Since'], undefined) + let feed1 = await loadValue(getFeed(feedId)) + ok(feed1!.refreshedAt) + + refreshPosts() + await setTimeout(10) + + // Later update: send `refreshedAt` in `If-Modified-Since` header + equal(requestCount, 2) + equal( + getHeaders(capturedOpts2)['If-Modified-Since'], + new Date(feed1!.refreshedAt * 1000).toUTCString() + ) + let feed2 = await loadValue(getFeed(feedId)) + equal(feed2!.refreshedAt, feed1!.refreshedAt) +}) diff --git a/proxy/index.ts b/proxy/index.ts index 032beb91..89bb695e 100644 --- a/proxy/index.ts +++ b/proxy/index.ts @@ -127,6 +127,32 @@ export function createProxy( /* node:coverage enable */ } + if ( + req.headers['if-modified-since'] && + targetResponse.headers.has('last-modified') + ) { + try { + let cachedAt = new Date(req.headers['if-modified-since']) + let updatedAt = new Date(targetResponse.headers.get('last-modified')!) + + if (cachedAt.getTime() >= updatedAt.getTime()) { + res.setHeader('Last-Modified', updatedAt.toUTCString()) + res.writeHead(304) + return res.end() + } + /* node:coverage disable */ + } catch (e) { + let message = 'Skipping cache check due to malformed date headers' + if (e instanceof Error) { + message += `: ${e.stack ?? e.message}` + } else if (typeof e === 'string') { + message += `: ${e}` + } + process.stderr.write(styleText('yellow', message) + '\n') + } + /* node:coverage enable */ + } + let length: number | undefined if (targetResponse.headers.has('content-length')) { length = parseInt(targetResponse.headers.get('content-length')!) diff --git a/proxy/test/index.test.ts b/proxy/test/index.test.ts index 446be5af..25a9051b 100644 --- a/proxy/test/index.test.ts +++ b/proxy/test/index.test.ts @@ -40,10 +40,14 @@ let target = createServer(async (req, res) => { res.writeHead(500) res.end('Error') } else { - res.writeHead(200, { + let headers: Record = { 'Content-Type': 'text/json', 'Set-Cookie': 'test=1' - }) + } + if (queryParams.lastModified) { + headers['Last-Modified'] = queryParams.lastModified + } + res.writeHead(200, headers) res.end( JSON.stringify({ request: { @@ -259,3 +263,49 @@ test('is ready for errors', async () => { equal(response1.status, 500) equal(await response1.text(), 'Error') }) + +test('handles If-Modified-Since and Last-Modified', async () => { + let lastModified = new Date(Date.now() - 10e3).toUTCString() + + let futureTime = new Date(Date.now() + 20e3).toUTCString() + let pastTime = new Date(Date.now() - 20e3).toUTCString() + + let response1 = await request(`${targetUrl}?lastModified=${lastModified}`, { + headers: { + 'If-Modified-Since': futureTime + } + }) + equal(response1.status, 304) + equal(response1.headers.get('last-modified'), lastModified) + + let response2 = await request(`${targetUrl}?lastModified=${lastModified}`, { + headers: { + 'If-Modified-Since': pastTime + } + }) + equal(response2.status, 200) + let json2 = (await response2.json()) as EchoResponse + equal(json2.response, 'ok') +}) + +test('handles malformed If-Modified-Since and Last-Modified', async () => { + let time = new Date(Date.now()).toUTCString() + + let response1 = await request(`${targetUrl}?lastModified=invalid-date`, { + headers: { + 'If-Modified-Since': time + } + }) + equal(response1.status, 200) + let json1 = (await response1.json()) as EchoResponse + equal(json1.response, 'ok') + + let response2 = await request(`${targetUrl}?lastModified=${time}`, { + headers: { + 'If-Modified-Since': 'invalid-date' + } + }) + equal(response2.status, 200) + let json2 = (await response2.json()) as EchoResponse + equal(json2.response, 'ok') +})