diff --git a/core/README.md b/core/README.md index 63938d0b..5ebb815f 100644 --- a/core/README.md +++ b/core/README.md @@ -7,6 +7,7 @@ _See the [full architecture guide](../README.md) first._ - [Client Environments](#client-environments) - [URL Routing](#url-routing) - [Test Strategy](#test-strategy) +- [Mocking Requests](#mocking-requests) ## Project Structure @@ -86,3 +87,22 @@ n bnt core/test/html.test.ts -t 'sanitizes HTML' In VS Code you can use [extension](https://marketplace.visualstudio.com/items?itemName=connor4312.nodejs-testing) to run specific test from UI. Open `core/coverage/lcov-report/index.html` to see coverage issues. + +## Mocking Requests + +To enable network request mocking in tests, you have to set up and tear down request mock before and after each test: + +```typescript +beforeEach(() => { + mockRequest() +}) + +afterEach(() => { + checkAndRemoveRequestMock() +}) +``` + +In the test itself, before making or triggering the request itself, use either: + +- `expectRequest(url).andRespond(...)` for simple mocking where the response is known upfront. +- or `expectRequest(url).andWait(...)` for complex scenarios where you need to control test loading states or simulate network delays. diff --git a/core/loader/atom.ts b/core/loader/atom.ts index 62a6eace..98b4eb95 100644 --- a/core/loader/atom.ts +++ b/core/loader/atom.ts @@ -1,8 +1,9 @@ -import type { TextResponse } from '../download.ts' +import type { DownloadTask, TextResponse } from '../download.ts' import type { OriginPost } from '../post.ts' -import { createPostsList } from '../posts-list.ts' +import { createPostsList, type PostsListLoader } from '../posts-list.ts' import type { Loader } from './index.ts' import { + buildFullURL, findAnchorHrefs, findDocumentLinks, findHeaderLinks, @@ -36,6 +37,66 @@ function parsePosts(text: TextResponse): OriginPost[] { }) } +/** + * Returns next or previous pagination url from feed xml, if present. + * See "paged feeds" https://www.rfc-editor.org/rfc/rfc5005#section-3 + */ +function getPaginationUrl( + xmlResponse: TextResponse, + rel: 'first' | 'last' | 'next' | 'previous' +): string | undefined { + let document = xmlResponse.parseXml() + if (!document) return undefined + let nextPageLink = [...document.querySelectorAll('link')].find( + link => link.getAttribute('rel') === rel + ) + return nextPageLink ? buildFullURL(nextPageLink, xmlResponse.url) : undefined +} + +type PostsCursor = + | [OriginPost[], PostsListLoader | undefined] + | [undefined, PostsListLoader] + +/** + * If xml response is ready, returns a tuple of posts and possibly + * the loader of the next portion of posts, if xml contains a link to them. + * If xml response is not yet ready, returns the recursive loader of posts. + */ +function getPostsCursor( + task: DownloadTask, + feedUrl: string, + feedResponse: TextResponse | undefined +): PostsCursor { + if (!feedResponse) { + return [ + undefined, + async () => { + let response = await task.text(feedUrl) + let [posts, loader] = getPostsCursor(task, feedUrl, response) + return [posts || [], loader] + } + ] + } + let nextPageUrl = getPaginationUrl(feedResponse, 'next') + let posts = parsePosts(feedResponse) + if (nextPageUrl) { + return [ + posts, + async () => { + let nextPageResponse = await task.text(nextPageUrl) + let [nextPosts, loader] = getPostsCursor( + task, + nextPageUrl, + nextPageResponse + ) + return [nextPosts || [], loader] + } + ] + } else { + return [posts, undefined] + } +} + export const atom: Loader = { getMineLinksFromText(text) { let type = 'application/atom+xml' @@ -54,12 +115,11 @@ export const atom: Loader = { }, getPosts(task, url, text) { - if (text) { - return createPostsList(parsePosts(text), undefined) + let [posts, nextLoader] = getPostsCursor(task, url, text) + if (!posts && nextLoader) { + return createPostsList(undefined, nextLoader) } else { - return createPostsList(undefined, async () => { - return [parsePosts(await task.text(url)), undefined] - }) + return createPostsList(posts || [], nextLoader) } }, diff --git a/core/loader/utils.ts b/core/loader/utils.ts index cc5d29f4..e049bc67 100644 --- a/core/loader/utils.ts +++ b/core/loader/utils.ts @@ -14,7 +14,7 @@ export function isHTML(text: TextResponse): boolean { * the explicitly provided base URL, but also the base URL specified * in the document. */ -function buildFullURL( +export function buildFullURL( link: HTMLAnchorElement | HTMLLinkElement, baseUrl: string ): string { diff --git a/core/request.ts b/core/request.ts index b3465e7b..c3267c6d 100644 --- a/core/request.ts +++ b/core/request.ts @@ -86,11 +86,13 @@ export function expectRequest(url: string): RequestMock { } requestExpects.push(expect) return { + /** Immediately sets up a mock response that will be returned synchronously */ andRespond(status, body = '', contentType = 'text/html') { expect.contentType = contentType expect.status = status expect.response = body }, + /** Returns a function that allows more control over the response */ andWait() { let { promise, resolve } = Promise.withResolvers() expect.wait = promise diff --git a/core/test/loader/atom.test.ts b/core/test/loader/atom.test.ts index 0d628ba9..6b61bd8a 100644 --- a/core/test/loader/atom.test.ts +++ b/core/test/loader/atom.test.ts @@ -2,13 +2,16 @@ import '../dom-parser.ts' import { spyOn } from 'nanospy' import { deepStrictEqual, equal } from 'node:assert' -import { test } from 'node:test' +import { afterEach, beforeEach, test } from 'node:test' import { setTimeout } from 'node:timers/promises' import { + checkAndRemoveRequestMock, createDownloadTask, createTextResponse, + expectRequest, loaders, + mockRequest, type TextResponse } from '../../index.ts' @@ -20,6 +23,14 @@ function exampleAtom(responseBody: string): TextResponse { }) } +beforeEach(() => { + mockRequest() +}) + +afterEach(() => { + checkAndRemoveRequestMock() +}) + test('detects xml:base attribute', () => { deepStrictEqual( loaders.atom.getMineLinksFromText( @@ -457,3 +468,93 @@ test('parses media', () => { } ) }) + +test('detects pagination with rel="next" link', () => { + let $store = loaders.atom.getPosts( + createDownloadTask(), + 'https://example.com/feed/', + exampleAtom( + ` + + + + ` + ) + ) + equal($store.get().hasNext, true) +}) + +test('detects when there is no pagination', () => { + let $store = loaders.atom.getPosts( + createDownloadTask(), + 'https://example.com/feed/', + exampleAtom( + ` + + ` + ) + ) + equal($store.get().hasNext, false) +}) + +test('loads first then second page', async () => { + let task = createDownloadTask() + + expectRequest('https://example.com/feed').andRespond( + 200, + ` + + + `, + 'application/atom+xml' + ) + let posts = loaders.atom.getPosts(task, 'https://example.com/feed') + await posts.loading + + expectRequest('https://example.com/feed?page=2').andRespond( + 200, + ` + + `, + 'application/atom+xml' + ) + await posts.next() +}) + +test('has posts from both pages', async () => { + let task = createDownloadTask() + + expectRequest('https://example.com/feed').andRespond( + 200, + ` + + + + Post on page 1 + 1 + 2023-01-01T00:00:00Z + + `, + 'application/atom+xml' + ) + let posts = loaders.atom.getPosts(task, 'https://example.com/feed') + await posts.loading + + expectRequest('https://example.com/feed?page=2').andRespond( + 200, + ` + + + Post on page 2 + 2 + 2023-01-02T00:00:00Z + + `, + 'application/atom+xml' + ) + await posts.next() + + equal(posts.get().list.length, 2) + equal(posts.get().list[0]?.title, 'Post on page 1') + equal(posts.get().list[1]?.title, 'Post on page 2') +})