From 066148109406f29306ee07adbcbc3f42c305622c Mon Sep 17 00:00:00 2001 From: taoky Date: Tue, 8 Aug 2023 12:58:34 +0800 Subject: [PATCH 1/3] Add directory lister support (for fixing vyos syncing) --- fixtures/vyos/index.html | 409 +++++++++++++++++++++++++++++++++ src/parser/directory_lister.rs | 132 +++++++++++ src/parser/mod.rs | 7 + 3 files changed, 548 insertions(+) create mode 100644 fixtures/vyos/index.html create mode 100644 src/parser/directory_lister.rs diff --git a/fixtures/vyos/index.html b/fixtures/vyos/index.html new file mode 100644 index 0000000..9378fd2 --- /dev/null +++ b/fixtures/vyos/index.html @@ -0,0 +1,409 @@ + + + + + + + + + + + + + + + +repositories/current/dists/current • dev.packages.vyos.net + +
+
+
+
+
+ + + + +
+
+ + +
+
+ +
+
+ +
+
+ +
+
+
+ +
+
+
+
+
+ +
+
+
+ + Home + + + / repositories + / current + / dists +
+ + + + +
+
+
+ +
+
+
+
+ File Name +
+ + + + +
+ + +
+ +
+ + + +
+
+ +
+
+ +
+
+
+ + +
+ + +
+ + +
+

+ + + + + +
+
+ +
+ + +
+
+
diff --git a/src/parser/directory_lister.rs b/src/parser/directory_lister.rs new file mode 100644 index 0000000..2d84164 --- /dev/null +++ b/src/parser/directory_lister.rs @@ -0,0 +1,132 @@ +use crate::{ + listing::{FileSize, FileType, ListItem}, + utils::get, +}; + +use super::{ListResult, Parser}; +use anyhow::Result; +use chrono::NaiveDateTime; +use scraper::{Html, Selector}; + +#[derive(Debug, Clone, Default)] +pub struct DirectoryListerListingParser; + +impl Parser for DirectoryListerListingParser { + fn get_list(&self, client: &reqwest::blocking::Client, url: &url::Url) -> Result { + let resp = get(client, url.clone())?; + let url = resp.url().clone(); + let body = resp.text()?; + assert!( + url.path().ends_with('/'), + "URL for listing should have a trailing slash" + ); + let document = Html::parse_document(&body); + // https://github.com/DirectoryLister/DirectoryLister/blob/0283f14aa1fbd97796f753e8d6105c752546050f/app/views/components/file.twig + + // find