diff --git a/README.md b/README.md index 8f6d78a..e5af2fb 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,17 @@ Reddit saves will be synced to a list named `Reddit Saved` in your Karakeep inst Reddit sync will be skipped if any of `KS_REDDIT_CLIENTID`, `KS_REDDIT_CLIENTSECRET` or `KS_REDDIT_REFRESHTOKEN` is not set. +### GitHub Stars +| Variable | Required | Description | +|----------|----------|-------------| +| `KS_GITHUB_TOKEN` | ❌ | Your GitHub personal access token | +| `KS_GITHUB_SCHEDULE` | ❌ | Sync schedule in cron format (default: `@daily`) | + +To obtain a GitHub personal access token, you can visit [this link](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) and create a new token with `Starring` user permission (read). + +GitHub stars will be synced to a list named `GitHub Starred` in your Karakeep instance. +GitHub sync will be skipped if `KS_GITHUB_TOKEN` is not set. ## Deployment diff --git a/crates/sync/src/plugin.rs b/crates/sync/src/plugin.rs index aa869f7..c488fc7 100644 --- a/crates/sync/src/plugin.rs +++ b/crates/sync/src/plugin.rs @@ -1,3 +1,4 @@ +mod github_stars; mod hn_upvotes; mod reddit_saves; @@ -55,6 +56,12 @@ pub trait Plugin: Send + Sync + 'static { } } + tracing::info!( + "sync complete for list: {} (created={})", + list_name, + created_count + ); + Ok(created_count) } } @@ -63,5 +70,6 @@ pub fn get_plugins() -> Vec> { vec![ Box::new(hn_upvotes::HNUpvoted {}), Box::new(reddit_saves::RedditSaves {}), + Box::new(github_stars::GithubStars {}), ] } diff --git a/crates/sync/src/plugin/github_stars.rs b/crates/sync/src/plugin/github_stars.rs new file mode 100644 index 0000000..1d69c29 --- /dev/null +++ b/crates/sync/src/plugin/github_stars.rs @@ -0,0 +1,114 @@ +use crate::settings; +use async_trait::async_trait; +use futures::{Stream, stream}; +use karakeep_client::BookmarkCreate; +use reqwest::Url; +use std::pin::Pin; + +#[derive(Debug, Clone)] +pub struct GithubStars {} + +fn parse_next_link(link_header: &str) -> Option { + link_header + .split(',') + .find_map(|link| { + let parts: Vec<&str> = link.split(';').collect(); + if parts.len() == 2 && parts[1].trim() == r#"rel="next""# { + let url: Url = parts[0] + .trim() + .trim_start_matches('<') + .trim_end_matches('>') + .parse() + .ok()?; + + let query = url.query()?; + Some(format!("?{}", query)) + } else { + None + } + }) + .map(|s| s.to_string()) +} + +#[async_trait] +impl super::Plugin for GithubStars { + fn list_name(&self) -> &'static str { + "GitHub Starred" + } + + async fn to_bookmark_stream( + &self, + ) -> anyhow::Result> + Send>>> { + let stream = stream::unfold(Some("?page=2".to_string()), move |params| async move { + let params = params?; + + let settings = &settings::get_settings(); + let token = settings + .github + .token + .as_ref() + .expect("GitHub token must be set for GitHub Stars plugin"); + + tracing::info!("fetching GitHub stars with params: {}, {token}", params); + + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + "Authorization", + format!("Bearer {}", token).parse().unwrap(), + ); + headers.insert("User-Agent", "karakeep-sync/1.0".parse().unwrap()); + headers.insert("Accept", "application/vnd.github.v3+json".parse().unwrap()); + + let client = reqwest::Client::new(); + let url = format!("https://api.github.com/user/starred{}", params); + let resp = client.get(url).headers(headers).send().await.ok()?; + + let next_page = resp.headers().get("Link").and_then(|link_header| { + let link_str = link_header.to_str().ok()?; + parse_next_link(link_str) + }); + + let resp = resp.json::>().await.ok()?; + let bookmarks: Vec = resp + .into_iter() + .map(|item| BookmarkCreate { + url: item["html_url"].as_str().unwrap_or("").to_string(), + title: item["full_name"].as_str().unwrap_or("").to_string(), + }) + .collect(); + + Some((bookmarks, next_page)) + }); + Ok(Box::pin(stream)) + } + + fn is_activated(&self) -> bool { + let settings = &settings::get_settings(); + settings.github.token.is_some() && !settings.github.token.as_ref().unwrap().is_empty() + } + + fn recurring_schedule(&self) -> String { + let settings = &settings::get_settings(); + settings.github.schedule.clone() + } +} + +#[cfg(test)] +mod test { + use super::parse_next_link; + + #[test] + fn test_parse_next_link() { + let link_header = r#"; rel="next", ; rel="last""#; + let next_link = parse_next_link(link_header); + assert_eq!(next_link, Some("?page=2".to_string())); + + let link_header_no_next = r#"; rel="last""#; + let next_link = parse_next_link(link_header_no_next); + assert_eq!(next_link, None); + + let empty_link_header = ""; + let next_link = parse_next_link(empty_link_header); + assert_eq!(next_link, None); + } +} diff --git a/crates/sync/src/settings.rs b/crates/sync/src/settings.rs index b277126..dfa5c63 100644 --- a/crates/sync/src/settings.rs +++ b/crates/sync/src/settings.rs @@ -4,6 +4,12 @@ use std::sync::OnceLock; use crate::settings; +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct GitHubSettings { + pub token: Option, + pub schedule: String, +} + #[derive(Debug, Clone, Deserialize)] pub(crate) struct HNSettings { pub auth: Option, @@ -29,6 +35,7 @@ pub(crate) struct Settings { pub hn: HNSettings, pub karakeep: KarakeepSettings, pub reddit: RedditSettings, + pub github: GitHubSettings, } impl Settings { @@ -41,6 +48,8 @@ impl Settings { .unwrap() .set_override("reddit.schedule", "@daily") .unwrap() + .set_override("github.schedule", "@daily") + .unwrap() .build() .unwrap();