From a2fbf100468bc5a4ad4a5b565ba554d6ab650060 Mon Sep 17 00:00:00 2001 From: immitsu Date: Fri, 2 Jan 2026 21:56:39 +0300 Subject: [PATCH 01/13] Support If-Modified-Since header for feed caching --- core/feed.ts | 19 +++++++++++++++++++ core/lib/download.ts | 3 ++- core/loader/rss.ts | 4 +++- core/refresh.ts | 3 +++ core/test/loader/rss.test.ts | 25 +++++++++++++++++++++++++ proxy/index.ts | 14 ++++++++++++++ proxy/test/index.test.ts | 32 ++++++++++++++++++++++++++++++-- 7 files changed, 96 insertions(+), 4 deletions(-) diff --git a/core/feed.ts b/core/feed.ts index 27594738..389e687d 100644 --- a/core/feed.ts +++ b/core/feed.ts @@ -28,6 +28,7 @@ export type FeedValue = { lastPublishedAt?: number loader: LoaderName reading: 'fast' | 'slow' + refreshedAt?: number slowReader?: UsefulReaderName title: string url: string @@ -99,6 +100,24 @@ export function getFeedLatestPosts( feed: FeedValue, task = createDownloadTask() ): PostsList { + if (feed.refreshedAt) { + let originalTask = task + task = { + ...task, + request(url, opts = {}) { + let headers = new Headers(opts.headers) + headers.set( + 'If-Modified-Since', + new Date(feed.refreshedAt! * 1000).toUTCString() + ) + return originalTask.request(url, { + ...opts, + headers + }) + } + } + } + return loaders[feed.loader].getPosts(task, feed.url) } diff --git a/core/lib/download.ts b/core/lib/download.ts index f0292344..cd82cf2a 100644 --- a/core/lib/download.ts +++ b/core/lib/download.ts @@ -222,7 +222,7 @@ export function createDownloadTask( if (controller.signal.aborted) { throw new DOMException('', 'AbortError') } - if (!response.ok) { + if (!response.ok && response.status !== 304) { throw new HTTPStatusError( response.status, url, @@ -246,6 +246,7 @@ export function createDownloadTask( if (controller.signal.aborted) { throw new DOMException('', 'AbortError') } + return createTextResponse(text, { headers: response.headers, redirected: response.redirected, diff --git a/core/loader/rss.ts b/core/loader/rss.ts index 6b43b1ec..84f71542 100644 --- a/core/loader/rss.ts +++ b/core/loader/rss.ts @@ -65,7 +65,9 @@ export const rss: Loader = { return createPostsList(() => [parsePosts(text), undefined]) } else { return createPostsList(async () => { - return [parsePosts(await task.text(url)), undefined] + let response = await task.text(url) + if (response.status === 304) return [[], undefined] + return [parsePosts(response), undefined] }) } }, diff --git a/core/refresh.ts b/core/refresh.ts index 97171116..87c18a22 100644 --- a/core/refresh.ts +++ b/core/refresh.ts @@ -115,6 +115,9 @@ async function checkForNextPage( if (!getFeed(feed.id).deleted) { await addPosts(feed, pages.get().list) } + await changeFeed(feed.id, { + refreshedAt: Math.round(Date.now() / 1000) + }) increaseKey(refreshStatistics, 'processedFeeds') } } diff --git a/core/test/loader/rss.test.ts b/core/test/loader/rss.test.ts index 051c4525..30766593 100644 --- a/core/test/loader/rss.test.ts +++ b/core/test/loader/rss.test.ts @@ -449,3 +449,28 @@ test('returns post source', async () => { undefined ) }) + +test('handles 304 not modified response', async () => { + expectRequest('https://example.com/news/').andRespond( + 304, + null as unknown as string, + null as unknown as string + ) + + let task = createDownloadTask() + let page = loaders.rss.getPosts(task, 'https://example.com/news/') + deepEqual(page.get(), { + error: undefined, + hasNext: true, + isLoading: true, + list: [] + }) + + await page.loading + deepEqual(page.get(), { + error: undefined, + hasNext: false, + isLoading: false, + list: [] + }) +}) diff --git a/proxy/index.ts b/proxy/index.ts index 032beb91..0efa76a3 100644 --- a/proxy/index.ts +++ b/proxy/index.ts @@ -127,6 +127,20 @@ export function createProxy( /* node:coverage enable */ } + if ( + req.headers['if-modified-since'] && + targetResponse.headers.has('last-modified') + ) { + let cachedAt = new Date(req.headers['if-modified-since']) + let updatedAt = new Date(targetResponse.headers.get('last-modified')!) + + if (cachedAt.getTime() >= updatedAt.getTime()) { + res.setHeader('Last-Modified', updatedAt.toUTCString()) + res.writeHead(304) + return res.end() + } + } + let length: number | undefined if (targetResponse.headers.has('content-length')) { length = parseInt(targetResponse.headers.get('content-length')!) diff --git a/proxy/test/index.test.ts b/proxy/test/index.test.ts index 446be5af..ccf8f045 100644 --- a/proxy/test/index.test.ts +++ b/proxy/test/index.test.ts @@ -40,10 +40,14 @@ let target = createServer(async (req, res) => { res.writeHead(500) res.end('Error') } else { - res.writeHead(200, { + let headers: Record = { 'Content-Type': 'text/json', 'Set-Cookie': 'test=1' - }) + } + if (queryParams.lastModified) { + headers['Last-Modified'] = queryParams.lastModified + } + res.writeHead(200, headers) res.end( JSON.stringify({ request: { @@ -259,3 +263,27 @@ test('is ready for errors', async () => { equal(response1.status, 500) equal(await response1.text(), 'Error') }) + +test('handles If-Modified-Since and Last-Modified', async () => { + let lastModified = new Date(Date.now() - 10e3).toUTCString() + + let futureTime = new Date(Date.now() + 20e3).toUTCString() + let pastTime = new Date(Date.now() - 20e3).toUTCString() + + let response1 = await request(`${targetUrl}?lastModified=${lastModified}`, { + headers: { + 'If-Modified-Since': futureTime + } + }) + equal(response1.status, 304) + equal(response1.headers.get('last-modified'), lastModified) + + let response2 = await request(`${targetUrl}?lastModified=${lastModified}`, { + headers: { + 'If-Modified-Since': pastTime + } + }) + equal(response2.status, 200) + let { response } = await response2.json() + equal(response, 'ok') +}) From ef22ed3c88b63bfbebc587efcc0f590443698d09 Mon Sep 17 00:00:00 2001 From: immitsu Date: Sat, 3 Jan 2026 22:51:05 +0300 Subject: [PATCH 02/13] Allow null for body in RequestMock.andRespond --- core/request.ts | 6 +++--- core/test/loader/rss.test.ts | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/core/request.ts b/core/request.ts index 20432cc5..6c3ad2dd 100644 --- a/core/request.ts +++ b/core/request.ts @@ -11,20 +11,20 @@ export function setRequestMethod(method: RequestMethod): void { } export interface RequestWaiter { - (status: number, body?: string, contentType?: string): Promise + (status: number, body?: null | string, contentType?: string): Promise aborted?: true } export interface RequestMock { andFail(): void - andRespond(status: number, body?: string, contentType?: string): void + andRespond(status: number, body?: null | string, contentType?: string): void andWait(): RequestWaiter } interface RequestExpect { contentType: string error: boolean - response: string + response: null | string status: number url: string wait: Promise diff --git a/core/test/loader/rss.test.ts b/core/test/loader/rss.test.ts index 30766593..de455957 100644 --- a/core/test/loader/rss.test.ts +++ b/core/test/loader/rss.test.ts @@ -451,11 +451,7 @@ test('returns post source', async () => { }) test('handles 304 not modified response', async () => { - expectRequest('https://example.com/news/').andRespond( - 304, - null as unknown as string, - null as unknown as string - ) + expectRequest('https://example.com/news/').andRespond(304, null) let task = createDownloadTask() let page = loaders.rss.getPosts(task, 'https://example.com/news/') From 21c5cadfed3cbef90eee7bef2d2a47da948b802b Mon Sep 17 00:00:00 2001 From: immitsu Date: Sat, 3 Jan 2026 23:13:44 +0300 Subject: [PATCH 03/13] Wrap date parsing in try-catch for proxy caching --- proxy/index.ts | 25 ++++++++++++++++++------- proxy/test/index.test.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/proxy/index.ts b/proxy/index.ts index 0efa76a3..cd72bb64 100644 --- a/proxy/index.ts +++ b/proxy/index.ts @@ -131,14 +131,25 @@ export function createProxy( req.headers['if-modified-since'] && targetResponse.headers.has('last-modified') ) { - let cachedAt = new Date(req.headers['if-modified-since']) - let updatedAt = new Date(targetResponse.headers.get('last-modified')!) - - if (cachedAt.getTime() >= updatedAt.getTime()) { - res.setHeader('Last-Modified', updatedAt.toUTCString()) - res.writeHead(304) - return res.end() + try { + let cachedAt = new Date(req.headers['if-modified-since']) + let updatedAt = new Date(targetResponse.headers.get('last-modified')!) + + if (cachedAt.getTime() >= updatedAt.getTime()) { + res.setHeader('Last-Modified', updatedAt.toUTCString()) + res.writeHead(304) + return res.end() + } + /* node:coverage disable */ + } catch { + process.stderr.write( + styleText( + 'yellow', + `Skipping cache check due to malformed date headers` + ) + '\n' + ) } + /* node:coverage enable */ } let length: number | undefined diff --git a/proxy/test/index.test.ts b/proxy/test/index.test.ts index ccf8f045..5cc8655b 100644 --- a/proxy/test/index.test.ts +++ b/proxy/test/index.test.ts @@ -284,6 +284,28 @@ test('handles If-Modified-Since and Last-Modified', async () => { } }) equal(response2.status, 200) - let { response } = await response2.json() - equal(response, 'ok') + let { response: response2Content } = await response2.json() + equal(response2Content, 'ok') +}) + +test('handles malformed If-Modified-Since and Last-Modified', async () => { + let time = new Date(Date.now()).toUTCString() + + let response1 = await request(`${targetUrl}?lastModified=invalid-date`, { + headers: { + 'If-Modified-Since': time + } + }) + equal(response1.status, 200) + let { response: response1Content } = await response1.json() + equal(response1Content, 'ok') + + let response2 = await request(`${targetUrl}?lastModified=${time}`, { + headers: { + 'If-Modified-Since': 'invalid-date' + } + }) + equal(response2.status, 200) + let { response: response2Content } = await response2.json() + equal(response2Content, 'ok') }) From 389db5f9ffdac214048fbc646677b18a0df81dbd Mon Sep 17 00:00:00 2001 From: immitsu Date: Sun, 4 Jan 2026 19:19:45 +0300 Subject: [PATCH 04/13] Add If-Modified-Since header at loader level --- core/feed.ts | 25 ++++------------ core/loader/common.ts | 7 ++++- core/loader/rss.ts | 7 +++-- core/test/loader/rss.test.ts | 56 ++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 22 deletions(-) diff --git a/core/feed.ts b/core/feed.ts index 389e687d..f97dd0b8 100644 --- a/core/feed.ts +++ b/core/feed.ts @@ -100,25 +100,12 @@ export function getFeedLatestPosts( feed: FeedValue, task = createDownloadTask() ): PostsList { - if (feed.refreshedAt) { - let originalTask = task - task = { - ...task, - request(url, opts = {}) { - let headers = new Headers(opts.headers) - headers.set( - 'If-Modified-Since', - new Date(feed.refreshedAt! * 1000).toUTCString() - ) - return originalTask.request(url, { - ...opts, - headers - }) - } - } - } - - return loaders[feed.loader].getPosts(task, feed.url) + return loaders[feed.loader].getPosts( + task, + feed.url, + undefined, + feed.refreshedAt + ) } let testFeedId = 0 diff --git a/core/loader/common.ts b/core/loader/common.ts index d479c316..7b68775c 100644 --- a/core/loader/common.ts +++ b/core/loader/common.ts @@ -17,7 +17,12 @@ export type Loader = { * Task is a way to combine multiple HTTP requests (for instance, during * the feed search/preview) to cancel all of them fast. */ - getPosts(task: DownloadTask, url: string, text?: TextResponse): PostsList + getPosts( + task: DownloadTask, + url: string, + text?: TextResponse, + refreshedAt?: number + ): PostsList /** * Get source data of post from loader's API for debug purposes. diff --git a/core/loader/rss.ts b/core/loader/rss.ts index 84f71542..db09be77 100644 --- a/core/loader/rss.ts +++ b/core/loader/rss.ts @@ -60,12 +60,15 @@ export const rss: Loader = { ] }, - getPosts(task, url, text) { + getPosts(task, url, text, refreshedAt) { if (text) { return createPostsList(() => [parsePosts(text), undefined]) } else { return createPostsList(async () => { - let response = await task.text(url) + let headers = refreshedAt + ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } + : undefined + let response = await task.text(url, { headers }) if (response.status === 304) return [[], undefined] return [parsePosts(response), undefined] }) diff --git a/core/test/loader/rss.test.ts b/core/test/loader/rss.test.ts index de455957..8bed4089 100644 --- a/core/test/loader/rss.test.ts +++ b/core/test/loader/rss.test.ts @@ -10,6 +10,7 @@ import { expectRequest, loaders, mockRequest, + setRequestMethod, testFeed, type TextResponse } from '../../index.ts' @@ -450,6 +451,61 @@ test('returns post source', async () => { ) }) +test('sends If-Modified-Since header when refreshedAt is provided', async () => { + let capturedOpts: RequestInit | undefined + + setRequestMethod((url, opts) => { + capturedOpts = opts + return Promise.resolve( + new Response( + ` + + + Feed + + Test Post + https://example.com/1 + + + `, + { + headers: { 'Content-Type': 'application/rss+xml' } + } + ) + ) + }) + + let task = createDownloadTask() + let page = loaders.rss.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + + await page.loading + + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' + }) + + deepEqual(page.get(), { + error: undefined, + hasNext: false, + isLoading: false, + list: [ + { + full: undefined, + media: undefined, + originId: 'https://example.com/1', + publishedAt: undefined, + title: 'Test Post', + url: 'https://example.com/1' + } + ] + }) +}) + test('handles 304 not modified response', async () => { expectRequest('https://example.com/news/').andRespond(304, null) From 8c02d565fc226dc552820c171627b8d757a58be5 Mon Sep 17 00:00:00 2001 From: immitsu Date: Sun, 4 Jan 2026 23:30:31 +0300 Subject: [PATCH 05/13] Rewrite 304 response test as behavior test --- core/test/loader/rss.test.ts | 63 +++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/core/test/loader/rss.test.ts b/core/test/loader/rss.test.ts index 8bed4089..777d0e2c 100644 --- a/core/test/loader/rss.test.ts +++ b/core/test/loader/rss.test.ts @@ -506,23 +506,56 @@ test('sends If-Modified-Since header when refreshedAt is provided', async () => }) }) -test('handles 304 not modified response', async () => { - expectRequest('https://example.com/news/').andRespond(304, null) +test('handles conditional request when server returns 304', async () => { + let callCount = 0 + let capturedOpts: RequestInit | undefined - let task = createDownloadTask() - let page = loaders.rss.getPosts(task, 'https://example.com/news/') - deepEqual(page.get(), { - error: undefined, - hasNext: true, - isLoading: true, - list: [] + // Mock initial and refreshing requests + setRequestMethod((url, opts) => { + if (++callCount === 1) { + return Promise.resolve( + new Response( + ` + + + Feed + + Test Post + https://example.com/1 + + + `, + { + headers: { + 'Content-Type': 'application/rss+xml', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + } + ) + ) + } else { + capturedOpts = opts + return Promise.resolve(new Response(null, { status: 304 })) + } }) - await page.loading - deepEqual(page.get(), { - error: undefined, - hasNext: false, - isLoading: false, - list: [] + let task = createDownloadTask() + + // Initial request + let page1 = loaders.rss.getPosts(task, 'https://example.com/news/') + await page1.loading + equal(page1.get().list.length, 1) + + // Refreshing request + let page2 = loaders.rss.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + await page2.loading + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' }) + equal(page2.get().list.length, 0) }) From f9df5c8e0ada72fcd4f770c602595245bee49e2f Mon Sep 17 00:00:00 2001 From: immitsu Date: Mon, 5 Jan 2026 00:08:47 +0300 Subject: [PATCH 06/13] Improve warning message for skipped cache checks --- proxy/index.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/proxy/index.ts b/proxy/index.ts index cd72bb64..89bb695e 100644 --- a/proxy/index.ts +++ b/proxy/index.ts @@ -141,13 +141,14 @@ export function createProxy( return res.end() } /* node:coverage disable */ - } catch { - process.stderr.write( - styleText( - 'yellow', - `Skipping cache check due to malformed date headers` - ) + '\n' - ) + } catch (e) { + let message = 'Skipping cache check due to malformed date headers' + if (e instanceof Error) { + message += `: ${e.stack ?? e.message}` + } else if (typeof e === 'string') { + message += `: ${e}` + } + process.stderr.write(styleText('yellow', message) + '\n') } /* node:coverage enable */ } From da57f01901aa07da6960eb61b929f53398d8689e Mon Sep 17 00:00:00 2001 From: immitsu Date: Mon, 5 Jan 2026 18:11:00 +0300 Subject: [PATCH 07/13] Support If-Modified-Since header for JSON feed caching --- core/loader/json-feed.ts | 9 +++-- core/test/loader/json-feed.test.ts | 56 ++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/core/loader/json-feed.ts b/core/loader/json-feed.ts index 99c7256c..dfb8a20b 100644 --- a/core/loader/json-feed.ts +++ b/core/loader/json-feed.ts @@ -142,12 +142,17 @@ export const jsonFeed: Loader = { return [...linksByType, ...findAnchorHrefs(text, /feed\.json/i)] }, - getPosts(task, url, text) { + getPosts(task, url, text, refreshedAt) { if (text) { return createPostsList(() => [parsePosts(text), undefined]) } else { return createPostsList(async () => { - return [parsePosts(await task.text(url)), undefined] + let headers = refreshedAt + ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } + : undefined + let response = await task.text(url, { headers }) + if (response.status === 304) return [[], undefined] + return [parsePosts(response), undefined] }) } }, diff --git a/core/test/loader/json-feed.test.ts b/core/test/loader/json-feed.test.ts index 704877cd..bd51b0c2 100644 --- a/core/test/loader/json-feed.test.ts +++ b/core/test/loader/json-feed.test.ts @@ -10,6 +10,7 @@ import { expectRequest, loaders, mockRequest, + setRequestMethod, setupEnvironment, testFeed, type TextResponse @@ -628,3 +629,58 @@ test('returns post source', async () => { undefined ) }) + +test('handles conditional request when server returns 304', async () => { + let callCount = 0 + let capturedOpts: RequestInit | undefined + + // Mock initial and refreshing requests + setRequestMethod((url, opts) => { + if (++callCount === 1) { + return Promise.resolve( + new Response( + JSON.stringify({ + items: [ + { + id: 'test-post', + title: 'Test Post', + url: 'https://example.com/1' + } + ], + title: 'Feed', + version: 'https://jsonfeed.org/version/1.1' + }), + { + headers: { + 'Content-Type': 'application/json', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + } + ) + ) + } else { + capturedOpts = opts + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let task = createDownloadTask() + + // Initial request + let page1 = loaders.jsonFeed.getPosts(task, 'https://example.com/news/') + await page1.loading + equal(page1.get().list.length, 1) + + // Refreshing request + let page2 = loaders.jsonFeed.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + await page2.loading + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' + }) + equal(page2.get().list.length, 0) +}) From 489db0dbff55dc103db8e279d3d60d406ec52e17 Mon Sep 17 00:00:00 2001 From: immitsu Date: Mon, 5 Jan 2026 18:26:30 +0300 Subject: [PATCH 08/13] Support If-Modified-Since header for Atom feed caching --- core/loader/atom.ts | 14 ++++++--- core/test/loader/atom.test.ts | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/core/loader/atom.ts b/core/loader/atom.ts index 1ec61ea0..4a38e6a8 100644 --- a/core/loader/atom.ts +++ b/core/loader/atom.ts @@ -122,9 +122,15 @@ function parseFeed( async function loadFeed( task: DownloadTask, - url: string + url: string, + refreshedAt?: number ): Promise { - return parseFeed(task, await task.text(url)) + let headers = refreshedAt + ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } + : undefined + let response = await task.text(url, { headers }) + if (response.status === 304) return [[], undefined] + return parseFeed(task, response) } export const atom: Loader = { @@ -143,11 +149,11 @@ export const atom: Loader = { } }, - getPosts(task, url, text) { + getPosts(task, url, text, refreshedAt) { if (text) { return createPostsList(() => parseFeed(task, text)) } else { - return createPostsList(() => loadFeed(task, url)) + return createPostsList(() => loadFeed(task, url, refreshedAt)) } }, diff --git a/core/test/loader/atom.test.ts b/core/test/loader/atom.test.ts index 8e166aa7..478b59d8 100644 --- a/core/test/loader/atom.test.ts +++ b/core/test/loader/atom.test.ts @@ -11,6 +11,7 @@ import { loaders, mockRequest, parseMedia, + setRequestMethod, setupEnvironment, testFeed, type TextResponse @@ -680,3 +681,56 @@ test('returns post source', async () => { undefined ) }) + +test('handles conditional request when server returns 304', async () => { + let callCount = 0 + let capturedOpts: RequestInit | undefined + + // Mock initial and refreshing requests + setRequestMethod((url, opts) => { + if (++callCount === 1) { + return Promise.resolve( + new Response( + ` + + Feed + + Test Post + test-post + + + `, + { + headers: { + 'Content-Type': 'application/atom+xml', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + } + ) + ) + } else { + capturedOpts = opts + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let task = createDownloadTask() + + // Initial request + let page1 = loaders.atom.getPosts(task, 'https://example.com/news/') + await page1.loading + equal(page1.get().list.length, 1) + + // Refreshing request + let page2 = loaders.atom.getPosts( + task, + 'https://example.com/news/', + undefined, + 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT + ) + await page2.loading + deepEqual(capturedOpts?.headers, { + 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' + }) + equal(page2.get().list.length, 0) +}) From d73a752d4ac34d7876b3c487b28fca45b954d6b4 Mon Sep 17 00:00:00 2001 From: immitsu Date: Mon, 5 Jan 2026 20:26:01 +0300 Subject: [PATCH 09/13] Fix types in proxy tests --- proxy/test/index.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/proxy/test/index.test.ts b/proxy/test/index.test.ts index 5cc8655b..25a9051b 100644 --- a/proxy/test/index.test.ts +++ b/proxy/test/index.test.ts @@ -284,8 +284,8 @@ test('handles If-Modified-Since and Last-Modified', async () => { } }) equal(response2.status, 200) - let { response: response2Content } = await response2.json() - equal(response2Content, 'ok') + let json2 = (await response2.json()) as EchoResponse + equal(json2.response, 'ok') }) test('handles malformed If-Modified-Since and Last-Modified', async () => { @@ -297,8 +297,8 @@ test('handles malformed If-Modified-Since and Last-Modified', async () => { } }) equal(response1.status, 200) - let { response: response1Content } = await response1.json() - equal(response1Content, 'ok') + let json1 = (await response1.json()) as EchoResponse + equal(json1.response, 'ok') let response2 = await request(`${targetUrl}?lastModified=${time}`, { headers: { @@ -306,6 +306,6 @@ test('handles malformed If-Modified-Since and Last-Modified', async () => { } }) equal(response2.status, 200) - let { response: response2Content } = await response2.json() - equal(response2Content, 'ok') + let json2 = (await response2.json()) as EchoResponse + equal(json2.response, 'ok') }) From 2a97ddf618392984c41001ad75a4b3dba915948d Mon Sep 17 00:00:00 2001 From: immitsu Date: Tue, 6 Jan 2026 23:23:28 +0300 Subject: [PATCH 10/13] Use `fetchIfModified` --- core/loader/atom.ts | 10 ++++------ core/loader/common.ts | 19 ++++++++++++++++++- core/loader/json-feed.ts | 15 +++++++-------- core/loader/rss.ts | 15 +++++++-------- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/core/loader/atom.ts b/core/loader/atom.ts index 4a38e6a8..6c66ab73 100644 --- a/core/loader/atom.ts +++ b/core/loader/atom.ts @@ -7,6 +7,7 @@ import { type OriginPost, type PostMedia, stringifyMedia } from '../post.ts' import { createPostsList, type PostsListResult } from '../posts-list.ts' import { buildFullURL, + fetchIfModified, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -125,12 +126,9 @@ async function loadFeed( url: string, refreshedAt?: number ): Promise { - let headers = refreshedAt - ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } - : undefined - let response = await task.text(url, { headers }) - if (response.status === 304) return [[], undefined] - return parseFeed(task, response) + return fetchIfModified(task, url, refreshedAt, response => + parseFeed(task, response) + ) } export const atom: Loader = { diff --git a/core/loader/common.ts b/core/loader/common.ts index 7b68775c..e449dc32 100644 --- a/core/loader/common.ts +++ b/core/loader/common.ts @@ -1,7 +1,7 @@ import type { FeedValue } from '../feed.ts' import type { DownloadTask, TextResponse } from '../lib/download.ts' import type { PostMedia } from '../post.ts' -import type { PostsList } from '../posts-list.ts' +import type { PostsList, PostsListResult } from '../posts-list.ts' export type Loader = { /** @@ -188,3 +188,20 @@ export function findMediaInText( } satisfies PostMedia }) } + +/** + * Fetches feed data if modified, handling conditional requests. + */ +export async function fetchIfModified( + task: DownloadTask, + url: string, + refreshedAt: number | undefined, + parseCb: (response: TextResponse) => PostsListResult +): Promise { + let headers = refreshedAt + ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } + : undefined + let response = await task.text(url, { headers }) + if (response.status === 304) return [[], undefined] + return parseCb(response) +} diff --git a/core/loader/json-feed.ts b/core/loader/json-feed.ts index dfb8a20b..af5b5453 100644 --- a/core/loader/json-feed.ts +++ b/core/loader/json-feed.ts @@ -3,6 +3,7 @@ import { createDownloadTask, type TextResponse } from '../lib/download.ts' import { type OriginPost, type PostMedia, stringifyMedia } from '../post.ts' import { createPostsList } from '../posts-list.ts' import { + fetchIfModified, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -146,14 +147,12 @@ export const jsonFeed: Loader = { if (text) { return createPostsList(() => [parsePosts(text), undefined]) } else { - return createPostsList(async () => { - let headers = refreshedAt - ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } - : undefined - let response = await task.text(url, { headers }) - if (response.status === 304) return [[], undefined] - return [parsePosts(response), undefined] - }) + return createPostsList(() => + fetchIfModified(task, url, refreshedAt, response => [ + parsePosts(response), + undefined + ]) + ) } }, diff --git a/core/loader/rss.ts b/core/loader/rss.ts index db09be77..f3f9c290 100644 --- a/core/loader/rss.ts +++ b/core/loader/rss.ts @@ -3,6 +3,7 @@ import { type OriginPost, type PostMedia, stringifyMedia } from '../post.ts' import { createPostsList } from '../posts-list.ts' import { findMRSS } from './atom.ts' import { + fetchIfModified, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -64,14 +65,12 @@ export const rss: Loader = { if (text) { return createPostsList(() => [parsePosts(text), undefined]) } else { - return createPostsList(async () => { - let headers = refreshedAt - ? { 'If-Modified-Since': new Date(refreshedAt * 1000).toUTCString() } - : undefined - let response = await task.text(url, { headers }) - if (response.status === 304) return [[], undefined] - return [parsePosts(response), undefined] - }) + return createPostsList(() => + fetchIfModified(task, url, refreshedAt, response => [ + parsePosts(response), + undefined + ]) + ) } }, From aaff8e0657c2dd8892bedefc8a346e811c2d1d86 Mon Sep 17 00:00:00 2001 From: immitsu Date: Wed, 7 Jan 2026 23:46:27 +0300 Subject: [PATCH 11/13] Verify If-Modified-Since usage with refreshedAt --- core/test/refresh.test.ts | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/core/test/refresh.test.ts b/core/test/refresh.test.ts index 6c0c62d8..868721fb 100644 --- a/core/test/refresh.test.ts +++ b/core/test/refresh.test.ts @@ -1,3 +1,5 @@ +import './dom-parser.ts' + import { loadValue } from '@logux/client' import { restoreAll, spyOn } from 'nanospy' import { deepEqual, equal, fail, ok } from 'node:assert/strict' @@ -25,6 +27,7 @@ import { refreshProgress, refreshStatistics, refreshStatus, + setRequestMethod, stopRefreshing, testFeed, waitLoading @@ -522,3 +525,63 @@ test('retries with some delay', async () => { totalFeeds: 3 }) }) + +test('sends If-Modified-Since equal to refreshedAt of updated feed', async () => { + let feedId = await addFeed(testFeed()) + + let requestCount = 0 + let capturedOpts1: RequestInit | undefined + let capturedOpts2: RequestInit | undefined + + setRequestMethod((url, opts) => { + if (++requestCount === 1) { + capturedOpts1 = opts ?? {} + return Promise.resolve( + new Response( + ` + + Feed + `, + { + headers: { + 'Content-Type': 'application/rss+xml', + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + }, + status: 200 + } + ) + ) + } else { + capturedOpts2 = opts ?? {} + return Promise.resolve(new Response(null, { status: 304 })) + } + }) + + let getHeaders = (opts?: RequestInit): Record => + (opts?.headers ?? {}) as Record + + equal(requestCount, 0) + let feed0 = await loadValue(getFeed(feedId)) + ok(!feed0!.refreshedAt) + + refreshPosts() + await setTimeout(10) + + // First update: store current timestamp as `refreshedAt` + equal(requestCount, 1) + equal(getHeaders(capturedOpts1)['If-Modified-Since'], undefined) + let feed1 = await loadValue(getFeed(feedId)) + ok(feed1!.refreshedAt) + + refreshPosts() + await setTimeout(10) + + // Later update: send `refreshedAt` in `If-Modified-Since` header + equal(requestCount, 2) + equal( + getHeaders(capturedOpts2)['If-Modified-Since'], + new Date(feed1!.refreshedAt * 1000).toUTCString() + ) + let feed2 = await loadValue(getFeed(feedId)) + equal(feed2!.refreshedAt, feed1!.refreshedAt) +}) From 5c2d60f817fbe3dfd080c7d30467e367e5b02d4e Mon Sep 17 00:00:00 2001 From: immitsu Date: Wed, 7 Jan 2026 23:47:36 +0300 Subject: [PATCH 12/13] Delete unit test from rss.test.ts --- core/test/loader/rss.test.ts | 55 ------------------------------------ 1 file changed, 55 deletions(-) diff --git a/core/test/loader/rss.test.ts b/core/test/loader/rss.test.ts index 777d0e2c..3d99a317 100644 --- a/core/test/loader/rss.test.ts +++ b/core/test/loader/rss.test.ts @@ -451,61 +451,6 @@ test('returns post source', async () => { ) }) -test('sends If-Modified-Since header when refreshedAt is provided', async () => { - let capturedOpts: RequestInit | undefined - - setRequestMethod((url, opts) => { - capturedOpts = opts - return Promise.resolve( - new Response( - ` - - - Feed - - Test Post - https://example.com/1 - - - `, - { - headers: { 'Content-Type': 'application/rss+xml' } - } - ) - ) - }) - - let task = createDownloadTask() - let page = loaders.rss.getPosts( - task, - 'https://example.com/news/', - undefined, - 1767225600 // Thu, 01 Jan 2026 00:00:00 GMT - ) - - await page.loading - - deepEqual(capturedOpts?.headers, { - 'If-Modified-Since': 'Thu, 01 Jan 2026 00:00:00 GMT' - }) - - deepEqual(page.get(), { - error: undefined, - hasNext: false, - isLoading: false, - list: [ - { - full: undefined, - media: undefined, - originId: 'https://example.com/1', - publishedAt: undefined, - title: 'Test Post', - url: 'https://example.com/1' - } - ] - }) -}) - test('handles conditional request when server returns 304', async () => { let callCount = 0 let capturedOpts: RequestInit | undefined From d06e0ce2dd514bf2b6a9338579bc7b742c46d3bd Mon Sep 17 00:00:00 2001 From: Andrey Sitnik Date: Wed, 7 Jan 2026 23:34:15 +0100 Subject: [PATCH 13/13] Apply suggestions from code review --- core/loader/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/loader/common.ts b/core/loader/common.ts index e449dc32..79e9c1b8 100644 --- a/core/loader/common.ts +++ b/core/loader/common.ts @@ -190,7 +190,7 @@ export function findMediaInText( } /** - * Fetches feed data if modified, handling conditional requests. + * Prevent heavy loading/parsing by `If-Modified-Since` header. */ export async function fetchIfModified( task: DownloadTask,