From fca2797c87966d615c466fee52601f960bfc0f71 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 18:16:45 -0700 Subject: [PATCH 01/14] Add series types --- types/entities.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/types/entities.ts b/types/entities.ts index f2af572..dfb14fc 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -38,6 +38,34 @@ export interface User { bioHtml: string | null; } +export interface SeriesWorkSummary + extends Omit< + WorkSummary, + "category" | "publishedAt" | "rating" | "tags" | "stats" + > { + tags: Omit; + stats: Omit; +} + +export interface Series { + id: string; + title: string; + begunAt: string; + updatedAt: string; + authors: WorkSummary["authors"]; + + description: string; + notes: string; + words: number; + stats: { + works: number; + bookmarks: number; + }; + completed: boolean; + + works: WorkSummary[]; +} + export enum WorkRatings { NOT_RATED = "Not Rated", GENERAL_AUDIENCES = "General Audiences", From 60739ac41e15ae6228d80ad7940a0fd54d84ae96 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 18:27:19 -0700 Subject: [PATCH 02/14] Mark `description` and `notes` as optional --- types/entities.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/entities.ts b/types/entities.ts index dfb14fc..7e19533 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -54,8 +54,8 @@ export interface Series { updatedAt: string; authors: WorkSummary["authors"]; - description: string; - notes: string; + description: string | null; + notes: string | null; words: number; stats: { works: number; From 610a837acb4ff599269d8136a34fa37f99b28c91 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 18:30:24 -0700 Subject: [PATCH 03/14] Remove `notes` --- types/entities.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/types/entities.ts b/types/entities.ts index 7e19533..38f074b 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -55,7 +55,6 @@ export interface Series { authors: WorkSummary["authors"]; description: string | null; - notes: string | null; words: number; stats: { works: number; From 8ec9bf15304ad5842ec3d3f5ff33949b50784791 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 21:41:49 -0700 Subject: [PATCH 04/14] Add series getters --- src/series/getters.ts | 230 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 src/series/getters.ts diff --git a/src/series/getters.ts b/src/series/getters.ts new file mode 100644 index 0000000..3589d49 --- /dev/null +++ b/src/series/getters.ts @@ -0,0 +1,230 @@ +import { Author, Series, SeriesWorkSummary } from "types/entities"; +import { SeriesPage, WorkPage } from "../page-loaders"; +import { CheerioAPI, load } from "cheerio"; +import { getWorkDetailsFromUrl } from "src/urls"; +import { + getWorkBookmarkCount, + getWorkHits, + getWorkKudosCount, + getWorkLanguage, + getWorkPublishedChapters, + getWorkTotalChapters, + getWorkWordCount, +} from "src/works/work-getters"; + +const monthMap: { [month: string]: string } = { + Jan: "01", + Feb: "02", + Mar: "03", + Apr: "04", + May: "05", + Jun: "06", + Jul: "07", + Aug: "08", + Sep: "09", + Oct: "10", + Nov: "11", + Dec: "12", +}; + +export const getSeriesTitle = ($seriesPage: SeriesPage): string => { + return $seriesPage("h2.heading").text().trim(); +}; + +export const getSeriesAuthors = ( + $seriesPage: SeriesPage +): Series["authors"] => { + const authorLinks = $seriesPage("dl.series a[rel=author]"); + const authors: Author[] = []; + + if ( + $seriesPage("dl.series dd:nth-of-type(1)").text().trim() === "Anonymous" + ) { + return "Anonymous"; + } + + if (authorLinks.length !== 0) { + authorLinks.each((i, element) => { + const url = element.attribs.href; + const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; + + authors.push({ username: username, pseud: decodeURI(pseud) }); + }); + } + + return authors; +}; + +export const getSeriesDescription = ( + $seriesPage: SeriesPage +): string | null => { + const description = $seriesPage("dl.series blockquote.userstuff").html(); + return description ? description.trim() : null; +}; + +export const getSeriesPublishDate = ($seriesPage: SeriesPage): string => { + return $seriesPage("dl.series > dd:nth-of-type(2)").text().trim(); +}; + +export const getSeriesUpdateDate = ($seriesPage: SeriesPage): string => { + return $seriesPage("dl.series > dd:nth-of-type(3)").text().trim(); +}; + +export const getSeriesWordCount = ($seriesPage: SeriesPage): number => { + return parseInt( + $seriesPage("dl.meta dl.stats dd:nth-of-type(1)") + .text() + .replaceAll(",", "") + .trim() + ); +}; + +export const getSeriesWorkCount = ($seriesPage: SeriesPage): number => { + return parseInt( + $seriesPage("dl.meta dl.stats dd:nth-of-type(2)") + .text() + .replaceAll(",", "") + .trim() + ); +}; + +export const getSeriesCompletionStatus = ($seriesPage: SeriesPage): boolean => { + return $seriesPage("dl.stats dd:nth-of-type(3)").text().trim() === "Yes"; +}; + +export const getSeriesBookmarkCount = ($seriesPage: SeriesPage): number => { + return parseInt( + $seriesPage("dl.meta dl.stats dd:nth-of-type(4)") + .text() + .replaceAll(",", "") + .trim() + ); +}; + +export const getSeriesWorks = ( + $seriesPage: SeriesPage +): SeriesWorkSummary[] => { + const works: SeriesWorkSummary[] = []; + + $seriesPage("ul.index > li.work").each((index, element) => { + works[index] = getSeriesWork($seriesPage(element).html() as string); + }); + + return works; +}; + +// Helpers for series' works +interface SeriesWork extends CheerioAPI { + kind: "SeriesWork"; +} + +const getSeriesWork = (workHtml: string): SeriesWorkSummary => { + const work = load(workHtml); + const $work = work as SeriesWork, + $$work = work as WorkPage; + + const totalChapters = getWorkTotalChapters($$work); + const publishedChapters = getWorkPublishedChapters($$work); + + const url = $work("a[href*='/works/']").attr("href") as string; + + return { + id: getWorkDetailsFromUrl({ url }).workId, + title: getSeriesWorkTitle($work), + updatedAt: getSeriesWorkUpdateDate($work), + + summary: getSeriesWorkSummary($work), + adult: false, + fandoms: getSeriesWorkFandoms($work), + tags: { + characters: getSeriesWorkCharacters($work), + relationships: getSeriesWorkRelationships($work), + additional: getSeriesWorkAdditionalTags($work), + }, + authors: getSeriesWorkAuthors($work), + language: getWorkLanguage($$work), + words: getWorkWordCount($$work), + chapters: { + published: publishedChapters, + total: totalChapters, + }, + complete: totalChapters !== null && totalChapters === publishedChapters, + stats: { + bookmarks: getWorkBookmarkCount($$work), + kudos: getWorkKudosCount($$work), + hits: getWorkHits($$work), + }, + }; +}; + +const getSeriesWorkTitle = ($work: SeriesWork) => { + return $work("h4.heading a[href*='/works/']").text().trim(); +}; + +const getSeriesWorkUpdateDate = ($work: SeriesWork) => { + const [day, month, year] = $work("p.datetime").text().trim().split(" "); + return `${year}-${monthMap[month]}-${day}`; +}; + +const getSeriesWorkSummary = ($work: SeriesWork) => { + const summary = $work("blockquote.summary").html(); + return summary ? summary.trim() : null; +}; + +const getSeriesWorkFandoms = ($work: SeriesWork): string[] => { + const fandoms: string[] = []; + + $work("h5.fandoms a.tag").each(function (i, element) { + fandoms[i] = $work(element).text().trim(); + }); + return fandoms; +}; + +const getSeriesWorkCharacters = ($work: SeriesWork): string[] => { + const characters: string[] = []; + + $work("li.characters a.tag").each(function (i, character) { + characters[i] = $work(character).text().trim(); + }); + return characters; +}; + +const getSeriesWorkRelationships = ($work: SeriesWork): string[] => { + const ships: string[] = []; + + $work("li.relationships a.tag").each(function (i, ship) { + ships[i] = $work(ship).text().trim(); + }); + return ships; +}; + +const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => { + const tags: string[] = []; + + $work("li.freeforms a.tag").each(function (i) { + tags[i] = $work(this).text().trim(); + }); + return tags; +}; + +const getSeriesWorkAuthors = ( + $work: SeriesWork +): SeriesWorkSummary["authors"] => { + const authorLinks = $work("h4.heading a[rel='author']"); + const authors: Author[] = []; + + if ($work("h4.heading").text().split("by")[1].trim() === "Anonymous") { + return "Anonymous"; + } + + if (authorLinks.length !== 0) { + authorLinks.each((i, element) => { + const url = element.attribs.href; + const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; + + authors.push({ username: username, pseud: decodeURI(pseud) }); + }); + } + + return authors; +}; From a97d7b90b781399dc44f6eb1b57c4d383ef567e2 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 21:42:18 -0700 Subject: [PATCH 05/14] Add `getSeries` --- src/series/index.ts | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/series/index.ts diff --git a/src/series/index.ts b/src/series/index.ts new file mode 100644 index 0000000..ba0105c --- /dev/null +++ b/src/series/index.ts @@ -0,0 +1,38 @@ +import { loadSeriesPage } from "src/page-loaders"; +import { Series } from "types/entities"; +import { + getSeriesAuthors, + getSeriesBookmarkCount, + getSeriesCompletionStatus, + getSeriesDescription, + getSeriesPublishDate, + getSeriesTitle, + getSeriesUpdateDate, + getSeriesWordCount, + getSeriesWorkCount, + getSeriesWorks, +} from "./getters"; + +export const getSeries = async ({ + seriesId, +}: { + seriesId: string; +}): Promise => { + const seriesPage = await loadSeriesPage(seriesId); + + return { + id: seriesId, + title: getSeriesTitle(seriesPage), + begunAt: getSeriesPublishDate(seriesPage), + updatedAt: getSeriesUpdateDate(seriesPage), + authors: getSeriesAuthors(seriesPage), + description: getSeriesDescription(seriesPage), + words: getSeriesWordCount(seriesPage), + stats: { + works: getSeriesWorkCount(seriesPage), + bookmarks: getSeriesBookmarkCount(seriesPage), + }, + completed: getSeriesCompletionStatus(seriesPage), + works: getSeriesWorks(seriesPage), + }; +}; From c92aa87728b980eed658ccddabaf22287ac6408e Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 21:42:28 -0700 Subject: [PATCH 06/14] Add the page loader --- src/page-loaders.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/page-loaders.ts b/src/page-loaders.ts index 42bc973..4e16e24 100644 --- a/src/page-loaders.ts +++ b/src/page-loaders.ts @@ -90,3 +90,13 @@ export const loadChaptersIndexPage = async ({ workId }: { workId: string }) => { ).data ) as ChapterIndexPage; }; + +export interface SeriesPage extends CheerioAPI { + kind: "SeriesPage"; +} +export const loadSeriesPage = async (seriesId: string) => { + return load( + (await axios.get(`https://archiveofourown.org/series/${seriesId}`)) + .data + ) as SeriesPage; +}; From dfdf439c6f7f6658a707b3908dac135bcefc6b3d Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 21:43:01 -0700 Subject: [PATCH 07/14] Update types --- types/entities.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/types/entities.ts b/types/entities.ts index 38f074b..8ea5ab2 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -41,7 +41,7 @@ export interface User { export interface SeriesWorkSummary extends Omit< WorkSummary, - "category" | "publishedAt" | "rating" | "tags" | "stats" + "category" | "publishedAt" | "rating" | "tags" | "stats" | "locked" > { tags: Omit; stats: Omit; @@ -53,7 +53,6 @@ export interface Series { begunAt: string; updatedAt: string; authors: WorkSummary["authors"]; - description: string | null; words: number; stats: { @@ -62,7 +61,7 @@ export interface Series { }; completed: boolean; - works: WorkSummary[]; + works: SeriesWorkSummary[]; } export enum WorkRatings { From 486b1df7ca74489d885c7c8fbd0dba3ef57549f4 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 21:43:08 -0700 Subject: [PATCH 08/14] Add export to main --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index dff92ba..1d89a62 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ export * from "./users"; export * from "./tags"; export * from "./works"; +export * from "./series"; From d7d6c205a508f80929f6fe142ea9b08335b419cc Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 22:11:22 -0700 Subject: [PATCH 09/14] Update to partially match #36's `series` list --- src/series/getters.ts | 11 +++++++++++ src/series/index.ts | 21 +++++++++++++-------- types/entities.ts | 15 ++++++++------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 3589d49..7e1b1e2 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -62,6 +62,17 @@ export const getSeriesDescription = ( return description ? description.trim() : null; }; +export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => { + const notes = $seriesPage("dl.series dd:nth-of-type(5)"); + console.log(notes.prevAll().first().text()); + if (notes.prevAll().first().text().trim() === "Notes:") { + console.log("Yep"); + return notes.html().trim(); + } else { + return null; + } +}; + export const getSeriesPublishDate = ($seriesPage: SeriesPage): string => { return $seriesPage("dl.series > dd:nth-of-type(2)").text().trim(); }; diff --git a/src/series/index.ts b/src/series/index.ts index ba0105c..af80dc1 100644 --- a/src/series/index.ts +++ b/src/series/index.ts @@ -5,6 +5,7 @@ import { getSeriesBookmarkCount, getSeriesCompletionStatus, getSeriesDescription, + getSeriesNotes, getSeriesPublishDate, getSeriesTitle, getSeriesUpdateDate, @@ -12,6 +13,7 @@ import { getSeriesWorkCount, getSeriesWorks, } from "./getters"; +import { getWorkUrl } from "src/urls"; export const getSeries = async ({ seriesId, @@ -20,19 +22,22 @@ export const getSeries = async ({ }): Promise => { const seriesPage = await loadSeriesPage(seriesId); + const seriesWorks = getSeriesWorks(seriesPage); + return { id: seriesId, - title: getSeriesTitle(seriesPage), + name: getSeriesTitle(seriesPage), begunAt: getSeriesPublishDate(seriesPage), updatedAt: getSeriesUpdateDate(seriesPage), - authors: getSeriesAuthors(seriesPage), + creators: getSeriesAuthors(seriesPage), description: getSeriesDescription(seriesPage), + notes: getSeriesNotes(seriesPage), words: getSeriesWordCount(seriesPage), - stats: { - works: getSeriesWorkCount(seriesPage), - bookmarks: getSeriesBookmarkCount(seriesPage), - }, - completed: getSeriesCompletionStatus(seriesPage), - works: getSeriesWorks(seriesPage), + bookmarks: getSeriesBookmarkCount(seriesPage), + complete: getSeriesCompletionStatus(seriesPage), + workCount: getSeriesWorkCount(seriesPage), + works: seriesWorks, + workTitles: seriesWorks.map((work) => work.title), + workUrls: seriesWorks.map((work) => getWorkUrl({ workId: work.id })), }; }; diff --git a/types/entities.ts b/types/entities.ts index 8ea5ab2..4c9fd54 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -49,19 +49,20 @@ export interface SeriesWorkSummary export interface Series { id: string; - title: string; + name: string; begunAt: string; updatedAt: string; - authors: WorkSummary["authors"]; + creators: WorkSummary["authors"]; description: string | null; + notes: string | null; words: number; - stats: { - works: number; - bookmarks: number; - }; - completed: boolean; + bookmarks: number; + complete: boolean; + workCount: number; works: SeriesWorkSummary[]; + workTitles: string[] + workUrls: string[] } export enum WorkRatings { From f988712b923be379910afdb4c949cf6eaff6a1bb Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 22:16:35 -0700 Subject: [PATCH 10/14] Remove `console.log` --- src/series/getters.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 7e1b1e2..6deb5f8 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -64,7 +64,6 @@ export const getSeriesDescription = ( export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => { const notes = $seriesPage("dl.series dd:nth-of-type(5)"); - console.log(notes.prevAll().first().text()); if (notes.prevAll().first().text().trim() === "Notes:") { console.log("Yep"); return notes.html().trim(); From 155aa43760442dd24ccee653bb8d3609542d786e Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 22:40:31 -0700 Subject: [PATCH 11/14] Fix naming and handling of `Anonymous` --- src/series/getters.ts | 18 +++++++++--------- src/series/index.ts | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 6deb5f8..0156b9f 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -31,28 +31,28 @@ export const getSeriesTitle = ($seriesPage: SeriesPage): string => { return $seriesPage("h2.heading").text().trim(); }; -export const getSeriesAuthors = ( +export const getSeriesCreators = ( $seriesPage: SeriesPage -): Series["authors"] => { - const authorLinks = $seriesPage("dl.series a[rel=author]"); - const authors: Author[] = []; +): Series["creators"] => { + const creatorLinks = $seriesPage("dl.meta a[rel=author]"); + const creators: Author[] = []; if ( - $seriesPage("dl.series dd:nth-of-type(1)").text().trim() === "Anonymous" + $seriesPage("dl.meta > dd:nth-of-type(1)").text().trim() === "Anonymous" ) { return "Anonymous"; } - if (authorLinks.length !== 0) { - authorLinks.each((i, element) => { + if (creatorLinks.length !== 0) { + creatorLinks.each((i, element) => { const url = element.attribs.href; const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - authors.push({ username: username, pseud: decodeURI(pseud) }); + creators.push({ username: username, pseud: decodeURI(pseud) }); }); } - return authors; + return creators; }; export const getSeriesDescription = ( diff --git a/src/series/index.ts b/src/series/index.ts index af80dc1..742ad5f 100644 --- a/src/series/index.ts +++ b/src/series/index.ts @@ -1,7 +1,7 @@ import { loadSeriesPage } from "src/page-loaders"; import { Series } from "types/entities"; import { - getSeriesAuthors, + getSeriesCreators, getSeriesBookmarkCount, getSeriesCompletionStatus, getSeriesDescription, @@ -29,7 +29,7 @@ export const getSeries = async ({ name: getSeriesTitle(seriesPage), begunAt: getSeriesPublishDate(seriesPage), updatedAt: getSeriesUpdateDate(seriesPage), - creators: getSeriesAuthors(seriesPage), + creators: getSeriesCreators(seriesPage), description: getSeriesDescription(seriesPage), notes: getSeriesNotes(seriesPage), words: getSeriesWordCount(seriesPage), From 0b736a9f6bc1bc8ae79cb867176173acd7718c65 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Tue, 31 Oct 2023 22:40:45 -0700 Subject: [PATCH 12/14] Add some tests --- src/series/getters.ts | 1 - tests/series.test.ts | 406 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 tests/series.test.ts diff --git a/src/series/getters.ts b/src/series/getters.ts index 0156b9f..32539a7 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -65,7 +65,6 @@ export const getSeriesDescription = ( export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => { const notes = $seriesPage("dl.series dd:nth-of-type(5)"); if (notes.prevAll().first().text().trim() === "Notes:") { - console.log("Yep"); return notes.html().trim(); } else { return null; diff --git a/tests/series.test.ts b/tests/series.test.ts new file mode 100644 index 0000000..54d5963 --- /dev/null +++ b/tests/series.test.ts @@ -0,0 +1,406 @@ +import { getSeries } from "src/index"; + +// TODO: Add more tests + +describe("Fetches series information", () => { + test("Fetches series object in its entirety", async () => { + const series = await getSeries({ seriesId: "2270465" }); + + expect(series).toMatchObject({ + id: "2270465", + name: "OG Titan", + begunAt: "2021-04-11", + updatedAt: "2023-02-13", + creators: [{ username: "MyHero", pseud: "MyHero" }], + description: + "

My potentially related stories about the relationship between the OG Titans. Probably focused on Dick Grayson.

", + notes: null, + words: 30044, + bookmarks: 177, + complete: false, + workCount: 6, + works: [ + { + id: "30604247", + title: "Away from all of Reality", + updatedAt: "2021-04-11", + summary: + "

Donna Troy regularly dreams about kidnapping her Wonder Twin, Dick Grayson, and running away to a better life. She wants to put him in a safe little apartment in Paris, or drag him to London, or disappear into the wilderness, or move out to the Kent farm. She dreams of taking him away to New York, cutting all the toxic people from his life and helping him grow.

Donna dreams to give Dick a better life but knows in her soul that he would never go. She knows that Dick would never be able to walk away from the others because it goes against his very being.

Donna is terrified of the possible day where Dick honestly takes her up on the offer to run away. It terrifies her because deep in her heart, because she knows if he ever truly says yes, that means they have finally broken him. And for all her strength, Donna doesn't know if she would be able to put him back to together again.

But still she dreams, of them running away because it's those small moments that keeps her from burning the world as she follows him through this Hell.

", + adult: false, + fandoms: [ + "Batman - All Media Types", + "Teen Titans (Comics)", + "Nightwing (Comics)", + ], + tags: { + characters: ["Dick Grayson", "Donna Troy"], + relationships: [ + "Dick Grayson & Donna Troy", + "Dick Grayson & Damian Wayne", + ], + additional: [ + "Dick Grayson Deserves Better", + "Hurt No Comfort", + "Protective Donna Troy", + "BAMF Donna Troy", + "Dick Grayson doesn't have a super - He has Ma and Pa Kent", + "Dick Grayson is Not Okay", + "Donna Troy is president of the", + "#DickGraysonProtectionSquad", + "Bruce Wayne is a Bad Parent", + "Author might have some unresolved issues with Babs", + "You'll see what I mean", + "There are events in cannon that I can't get over and it shows", + "Same with Bruce", + "Wally West would be their secret keeper", + "Damian would have an open invite to visit", + "Daydreaming", + "Donna Troy POV", + "wonder twins", + "Hurt Dick Grayson", + "Dick Grayson-centric", + "Garth and Wally will still come for brunch twice a month", + ], + }, + authors: [{ username: "MyHero", pseud: "MyHero" }], + language: "English", + words: 2733, + chapters: { published: 1, total: 1 }, + complete: true, + stats: { bookmarks: 177, kudos: 1, hits: 8 }, + }, + { + id: "30794750", + title: "Code B", + updatedAt: "2023-02-01", + summary: + '

It started with a group text from Garth, between him, Wally, Roy, and Donna.

Tuesday 8:37pm
Fish-R-Us: Code Burgundy - D

 
It ended with Roy starting to realize just how far he has wondered from his family due to misguided anger.
And maybe he starts to do something about it, the first step is to find out find the missing pieces. He along with Jason (who was first reluctant then determined) are on a mission for the truth to the "death" of Dick Grayson.

', + adult: false, + fandoms: [ + "Batman - All Media Types", + "Nightwing (Comics)", + "Teen Titans (Comics)", + "Red Hood and the Outlaws (Comics)", + ], + tags: { + characters: [ + "Dick Grayson", + "Garth (DCU)", + "Donna Troy", + "Wally West", + "Roy Harper", + "Jason Todd", + "Alfred Pennyworth", + ], + relationships: [ + "Dick Grayson & Donna Troy", + "Dick Grayson & Wally West", + "Garth & Dick Grayson & Roy Harper & Donna Troy & Wally West", + "Garth & Dick Grayson", + "Roy Harper & Jason Todd", + "Dick Grayson & Damian Wayne", + "Alfred Pennyworth & Jason Todd", + "Dick Grayson & Roy Harper", + "Dick Grayson & Amy Rohrbach", + "Catalina Flores/Dick Grayson", + ], + additional: [ + "Hurt Dick Grayson", + "Dick Grayson Needs a Hug", + "Protective Wally West", + "Team as Family", + "Roy Harper Needs a Hug", + "Bruce Wayne is a Bad Parent", + "The problem with not getting all the information is you don't have all the information", + "A moment of clarity by Roy Harper", + "wonder twins", + "Jason Todd Needs A Hug", + "Dick Grayson Did Not Fake His Death", + "Dick Grayson is Not Okay", + "Dick Grayson-centric", + "Alfred Pennyworth isn’t perfect but is trying his best and is significantly better than Bruce", + "Former drug addict Roy Harper", + "Mentioned Catalina Flores", + "Dick Grayson & Wally West Friendship", + "Protective Team", + "Past Rape/Non-con", + "Amy Rohrbach is a good friend", + "Roy Harper needs a swear jar", + "Blink and you miss it reference to...", + "Evil Slade Wilson", + "Creepy Slade Wilson", + "Implied/Referenced Rape/Non-con", + "Like the city of Rome your positive mental health cant be built in a day", + "Also like Rome your mental health can be destroyed in hours", + "All about Dick Grayson and he doesn't even show up until chapter 11", + "Dick Grayson & Donna Troy Friendship", + ], + }, + authors: [{ username: "MyHero", pseud: "MyHero" }], + language: "English", + words: 17141, + chapters: { published: 12, total: null }, + complete: false, + stats: { bookmarks: 561, kudos: 2, hits: 51 }, + }, + { + id: "30914645", + title: "Donna Troy Loves You", + updatedAt: "2021-04-26", + summary: + "

Three times Cassandra witnessed the friendship between Dick and Donna without Donna even being there. And the one time Donna was there in person.

Wonder Twins might not be in the same city, but they aren't any less wonderful.

", + adult: false, + fandoms: [ + "Nightwing (Comics)", + "Batman - All Media Types", + "Teen Titans (Comics)", + ], + tags: { + characters: ["Dick Grayson", "Donna Troy", "Cassandra Cain"], + relationships: [ + "Dick Grayson & Donna Troy", + "Cassandra Cain & Dick Grayson", + "Dick Grayson & Alfred Pennyworth", + "Batfamily Members & Dick Grayson", + ], + additional: [ + "Protective Donna Troy", + "Good Sibling Cassandra Cain", + "Caring Cassandra Cain", + "Dick Grayson is Nightwing", + "Dick Grayson Needs a Hug", + "BAMF Dick Grayson", + "Dick Grayson Gets a Hug", + "Bruce Wayne Tries to Be a Good Parent", + "Good Grandparent Alfred Pennyworth", + "Mentioned Catalina Flores", + "Implied/Referenced Rape/Non-con", + "Jealous Barbara Gordon", + "Dick Grayson Needs Therapy", + "Hurt Dick Grayson", + "#DickGraysonProtectionSquad", + "Bruce Wayne C+ Parenting", + "Alfred Pennyworth is a Saint", + "Dick Grayson-centric", + ], + }, + authors: [{ username: "MyHero", pseud: "MyHero" }], + language: "English", + words: 3021, + chapters: { published: 1, total: 1 }, + complete: true, + stats: { bookmarks: 192, kudos: 1, hits: 9 }, + }, + { + id: "31221131", + title: "Rockin Robin", + updatedAt: "2021-05-11", + summary: + "

It started with a tweet. Actually it started with a Teen Titan case from back in the day that became a hobby and just a healthy outlit.  It came back into their lives with a tweet.

 

Inspired by Everyday one headcanon about Nightwing by TrikaLika (Chapter 38. Band)

", + adult: false, + fandoms: ["Teen Titans (Comics)", "Batman - All Media Types"], + tags: { + characters: [ + "Garth (DCU)", + "Dick Grayson", + "Donna Troy", + "Roy Harper", + "Wally West", + "Guest staring:", + "Barbara Gordon", + "Tim Drake", + ], + relationships: [ + "Garth & Dick Grayson & Roy Harper & Donna Troy & Wally West", + ], + additional: [ + "The Titans were a band", + "Dick on vocals and guitar", + "Donna on vocals and guitar", + "Roy on bass guitar", + "Wally on drums", + "Garth on piano", + "Twitter", + "Will there be more? Probably", + "Team as Family", + "wonder twins", + ], + }, + authors: [{ username: "MyHero", pseud: "MyHero" }], + language: "English", + words: 832, + chapters: { published: 1, total: 1 }, + complete: true, + stats: { bookmarks: 70, kudos: 567, hits: 4 }, + }, + { + id: "35757790", + title: "Realistic Exit Strategy", + updatedAt: "2023-02-13", + summary: + "

Dick knew about Donna's daydreams about him walking away. And maybe in his darkest moments, Dick has a few of his own. But now, as he and Donna board the plane to travel across the world, he wonders if he will ever make it back. This isn't how either of them expected it to go, but maybe that's the silver lining to the otherwise dark situation.

 

Aka... It didn't take his death to finally make Dick walk away from Gotham, but he might not be able to say the same in a year.

The cure was still early stages of development, considered experimental at best. The Bats were the least of their concerns. Dick knows things will never be the same, assuming he lives at all.

", + adult: false, + fandoms: [ + "Teen Titans (Comics)", + "Nightwing (Comics)", + "Batman - All Media Types", + ], + tags: { + characters: [ + "Dick Grayson", + "Donna Troy", + "Wally West", + "Garth (DCU)", + "Diana (Wonder Woman)", + "Clark Kent", + "Roy Harper", + ], + relationships: [ + "Dick Grayson & Donna Troy", + "Garth & Dick Grayson & Roy Harper & Donna Troy & Wally West", + "Diana (Wonder Woman) & Dick Grayson", + "Dick Grayson & Clark Kent", + ], + additional: [ + "Dick Grayson Needs a Hug", + "Dick Grayson Needs Help", + "Dick Grayson Has Issues", + "Illnesses", + "Hurt/Comfort", + "Unhealthy Coping Mechanisms", + "Dick Grayson Deserves Better", + "Protective Donna Troy", + "Good Friend Donna Troy", + "Jason Todd is Bad at Feelings", + "Tim Drake is Bad at Feelings", + "No Beta", + "no beta we die like robins", + "Lack of Communication", + "Hurt Dick Grayson", + "wonder twins", + "Dick Grayson-centric", + "Team as Family", + "Medical Inaccuracies", + "Protective Wally West", + "Protective Garth (DCU)", + "Good Uncle Clark Kent", + "Good Aunt Diana Prince", + "Roy Harper comes around", + "Good Friend Roy Harper", + "Medical inaccuracies like whoa", + "Don't need to read part one", + "but it might help.", + ], + }, + authors: [{ username: "MyHero", pseud: "MyHero" }], + language: "English", + words: 5727, + chapters: { published: 2, total: 3 }, + complete: false, + stats: { bookmarks: 150, kudos: 993, hits: 8 }, + }, + { + id: "44149795", + title: "You starting down the road leaving me again", + updatedAt: "2023-01-09", + summary: + "

Dick feels the distance between Bludhaven and Gotham like a crater in his chest. His family were light-years away, across the inkey Gotham Bay. The distant lights were sparkling stars in the smog.

He had hope. The Titans were his tethers, pulling him from the blackhole centered in his Bludhaven appartment.

Aka

Friends that sing together stay together

", + adult: false, + fandoms: [ + "Batman - All Media Types", + "Nightwing (Comics)", + "Titans (Comics)", + ], + tags: { + characters: [ + "Dick Grayson", + "Donna Troy", + "Garth (DCU)", + "Roy Harper", + "Wally West", + ], + relationships: [ + "Garth & Dick Grayson & Roy Harper & Donna Troy & Wally West", + ], + additional: [ + "Teen Titans as Family", + "Titans", + "Team as Family", + "Karaoke", + "Dick Grayson Needs a Hug", + "Blüdhaven", + "Hurt Dick Grayson", + "Emotional Hurt", + "Found Family", + "Parent Roy Harper", + "Mentioned Linda Park", + "would this count as", + "Dick Grayson-centric", + "What do you think they sing?", + "The boys probably do a boy band", + "They lost a bet to Donna", + "No actual singing", + "No Plot/Plotless", + "Random & Short", + "Author Is Sleep Deprived", + "Author needed something sweet", + ], + }, + authors: [{ username: "MyHero", pseud: "MyHero" }], + language: "English", + words: 590, + chapters: { published: 1, total: 1 }, + complete: true, + stats: { bookmarks: 28, kudos: 242, hits: 1 }, + }, + ], + workTitles: [ + "Away from all of Reality", + "Code B", + "Donna Troy Loves You", + "Rockin Robin", + "Realistic Exit Strategy", + "You starting down the road leaving me again", + ], + workUrls: [ + "https://archiveofourown.org/works/30604247", + "https://archiveofourown.org/works/30794750", + "https://archiveofourown.org/works/30914645", + "https://archiveofourown.org/works/31221131", + "https://archiveofourown.org/works/35757790", + "https://archiveofourown.org/works/44149795", + ], + }); + }); + + test("Fetches author with username Anonymous", async () => { + const series = await getSeries({ seriesId: "2946579" }); + expect(series.creators).toBe("Anonymous"); + }); + + describe("Fetches series title", () => { + test("Fetch series title with space character", async () => { + const series = await getSeries({ + seriesId: "2270465", + }); + + expect(series.name).toBe("OG Titan"); + }); + + test("Fetch series with slashes", async () => { + const series = await getSeries({ + seriesId: "1728802", + }); + + expect(series.name).toBe("angsty oneshots/short stories"); + }); + + test("Fetch series with non-letter characters", async () => { + const series = await getSeries({ + seriesId: "2817877", + }); + + expect(series.name).toBe("*Insert Fandom* but Social Media (one-shots)"); + }); + }); +}); From 388148f3251f4c05461ce6070321e0cf60643d01 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Wed, 1 Nov 2023 16:02:24 -0700 Subject: [PATCH 13/14] Apply suggestions from code review Rename `begunAt` to `startedAt` and `creators` to `authors` --- src/series/getters.ts | 16 ++++++++-------- src/series/index.ts | 6 +++--- tests/series.test.ts | 2 +- types/entities.ts | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 32539a7..34e1a4c 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -31,11 +31,11 @@ export const getSeriesTitle = ($seriesPage: SeriesPage): string => { return $seriesPage("h2.heading").text().trim(); }; -export const getSeriesCreators = ( +export const getSeriesAuthors = ( $seriesPage: SeriesPage -): Series["creators"] => { - const creatorLinks = $seriesPage("dl.meta a[rel=author]"); - const creators: Author[] = []; +): Series["authors"] => { + const authorLinks = $seriesPage("dl.meta a[rel=author]"); + const authors: Author[] = []; if ( $seriesPage("dl.meta > dd:nth-of-type(1)").text().trim() === "Anonymous" @@ -43,16 +43,16 @@ export const getSeriesCreators = ( return "Anonymous"; } - if (creatorLinks.length !== 0) { - creatorLinks.each((i, element) => { + if (authorLinks.length !== 0) { + authorLinks.each((i, element) => { const url = element.attribs.href; const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - creators.push({ username: username, pseud: decodeURI(pseud) }); + authors.push({ username: username, pseud: decodeURI(pseud) }); }); } - return creators; + return authors; }; export const getSeriesDescription = ( diff --git a/src/series/index.ts b/src/series/index.ts index 742ad5f..f142872 100644 --- a/src/series/index.ts +++ b/src/series/index.ts @@ -1,7 +1,7 @@ import { loadSeriesPage } from "src/page-loaders"; import { Series } from "types/entities"; import { - getSeriesCreators, + getSeriesAuthors, getSeriesBookmarkCount, getSeriesCompletionStatus, getSeriesDescription, @@ -27,9 +27,9 @@ export const getSeries = async ({ return { id: seriesId, name: getSeriesTitle(seriesPage), - begunAt: getSeriesPublishDate(seriesPage), + startedAt: getSeriesPublishDate(seriesPage), updatedAt: getSeriesUpdateDate(seriesPage), - creators: getSeriesCreators(seriesPage), + authors: getSeriesAuthors(seriesPage), description: getSeriesDescription(seriesPage), notes: getSeriesNotes(seriesPage), words: getSeriesWordCount(seriesPage), diff --git a/tests/series.test.ts b/tests/series.test.ts index 54d5963..5f935f1 100644 --- a/tests/series.test.ts +++ b/tests/series.test.ts @@ -375,7 +375,7 @@ describe("Fetches series information", () => { test("Fetches author with username Anonymous", async () => { const series = await getSeries({ seriesId: "2946579" }); - expect(series.creators).toBe("Anonymous"); + expect(series.authors).toBe("Anonymous"); }); describe("Fetches series title", () => { diff --git a/types/entities.ts b/types/entities.ts index 4c9fd54..2602368 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -50,9 +50,9 @@ export interface SeriesWorkSummary export interface Series { id: string; name: string; - begunAt: string; + startedAt: string; updatedAt: string; - creators: WorkSummary["authors"]; + authors: WorkSummary["authors"]; description: string | null; notes: string | null; words: number; From c251f87a6756c8c4918cb671c1b698e9c7be9573 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Wed, 1 Nov 2023 16:05:36 -0700 Subject: [PATCH 14/14] Remove duplicate data, add `url` to `SeriesWorkSummary` --- src/series/getters.ts | 6 ++++-- src/series/index.ts | 3 --- types/entities.ts | 4 +--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 34e1a4c..0d05030 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -1,7 +1,7 @@ import { Author, Series, SeriesWorkSummary } from "types/entities"; import { SeriesPage, WorkPage } from "../page-loaders"; import { CheerioAPI, load } from "cheerio"; -import { getWorkDetailsFromUrl } from "src/urls"; +import { getWorkDetailsFromUrl, getWorkUrl } from "src/urls"; import { getWorkBookmarkCount, getWorkHits, @@ -136,9 +136,11 @@ const getSeriesWork = (workHtml: string): SeriesWorkSummary => { const publishedChapters = getWorkPublishedChapters($$work); const url = $work("a[href*='/works/']").attr("href") as string; + const id = getWorkDetailsFromUrl({ url }).workId return { - id: getWorkDetailsFromUrl({ url }).workId, + id, + url: getWorkUrl({ workId: id }), title: getSeriesWorkTitle($work), updatedAt: getSeriesWorkUpdateDate($work), diff --git a/src/series/index.ts b/src/series/index.ts index f142872..31aa790 100644 --- a/src/series/index.ts +++ b/src/series/index.ts @@ -13,7 +13,6 @@ import { getSeriesWorkCount, getSeriesWorks, } from "./getters"; -import { getWorkUrl } from "src/urls"; export const getSeries = async ({ seriesId, @@ -37,7 +36,5 @@ export const getSeries = async ({ complete: getSeriesCompletionStatus(seriesPage), workCount: getSeriesWorkCount(seriesPage), works: seriesWorks, - workTitles: seriesWorks.map((work) => work.title), - workUrls: seriesWorks.map((work) => getWorkUrl({ workId: work.id })), }; }; diff --git a/types/entities.ts b/types/entities.ts index 2602368..e5cc60a 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -43,6 +43,7 @@ export interface SeriesWorkSummary WorkSummary, "category" | "publishedAt" | "rating" | "tags" | "stats" | "locked" > { + url: string; tags: Omit; stats: Omit; } @@ -58,11 +59,8 @@ export interface Series { words: number; bookmarks: number; complete: boolean; - workCount: number; works: SeriesWorkSummary[]; - workTitles: string[] - workUrls: string[] } export enum WorkRatings {