diff --git a/src/cli/sync.rs b/src/cli/sync.rs
index 8a5f940..5207385 100644
--- a/src/cli/sync.rs
+++ b/src/cli/sync.rs
@@ -380,35 +380,48 @@ pub fn sync(args: SyncArgs, bind_address: Option) -> ! {
exit_code = 1;
} else {
// Don't even walkdir when dry_run, to prevent no dir error
- if !args.dry_run {
- for entry in walkdir::WalkDir::new(download_dir).contents_first(true) {
- let entry = entry.unwrap();
- let path = entry.path();
- if !remote_list.contains(&path.to_path_buf()) {
- if !args.no_delete {
- // always make sure that we are deleting the right thing
- if del_cnt >= args.max_delete {
- info!("Exceeding max delete count, aborting");
- // exit with 25 to indicate that the deletion has been aborted
- // this is the same as rsync
- exit_code = 25;
- break;
- }
- del_cnt += 1;
- assert!(path.starts_with(download_dir));
+
+ for entry in walkdir::WalkDir::new(download_dir).contents_first(true) {
+ let entry = match entry {
+ Ok(entry) => entry,
+ Err(e) => {
+ error!("Failed to walkdir: {:?}", e);
+ if !args.dry_run {
+ exit_code = 1;
+ }
+ break;
+ }
+ };
+ let path = entry.path();
+ if !remote_list.contains(&path.to_path_buf()) {
+ if !args.no_delete {
+ // always make sure that we are deleting the right thing
+ if del_cnt >= args.max_delete {
+ info!("Exceeding max delete count, aborting");
+ // exit with 25 to indicate that the deletion has been aborted
+ // this is the same as rsync
+ exit_code = 25;
+ break;
+ }
+ del_cnt += 1;
+ assert!(path.starts_with(download_dir));
+ if args.dry_run {
+ info!("Dry run, not deleting {:?}", path);
+ continue;
+ } else {
info!("Deleting {:?}", path);
- if entry.file_type().is_dir() {
- if let Err(e) = std::fs::remove_dir(path) {
- error!("Failed to remove {:?}: {:?}", path, e);
- exit_code = 4;
- }
- } else if let Err(e) = std::fs::remove_file(path) {
+ }
+ if entry.file_type().is_dir() {
+ if let Err(e) = std::fs::remove_dir(path) {
error!("Failed to remove {:?}: {:?}", path, e);
exit_code = 4;
}
- } else {
- info!("{:?} not in remote", path);
+ } else if let Err(e) = std::fs::remove_file(path) {
+ error!("Failed to remove {:?}: {:?}", path, e);
+ exit_code = 4;
}
+ } else {
+ info!("{:?} not in remote", path);
}
}
}
diff --git a/src/parser/directory_lister.rs b/src/parser/directory_lister.rs
new file mode 100644
index 0000000..c5d4fd1
--- /dev/null
+++ b/src/parser/directory_lister.rs
@@ -0,0 +1,148 @@
+use crate::{
+ listing::{FileSize, FileType, ListItem},
+ utils::get,
+};
+
+use super::{ListResult, Parser};
+use anyhow::Result;
+use chrono::NaiveDateTime;
+use scraper::{Html, Selector};
+
+#[derive(Debug, Clone, Default)]
+pub struct DirectoryListerListingParser;
+
+impl Parser for DirectoryListerListingParser {
+ fn get_list(&self, client: &reqwest::blocking::Client, url: &url::Url) -> Result {
+ let resp = get(client, url.clone())?;
+ let url = resp.url().clone();
+ let body = resp.text()?;
+ assert!(
+ url.path().ends_with('/'),
+ "URL for listing should have a trailing slash"
+ );
+ let document = Html::parse_document(&body);
+ // https://github.com/DirectoryLister/DirectoryLister/blob/0283f14aa1fbd97796f753e8d6105c752546050f/app/views/components/file.twig
+
+ // find
which contains file index
+ let selector = Selector::parse("ul").unwrap();
+ let indexlist = document.select(&selector).next().unwrap();
+ // find second
+ let selector = Selector::parse("li").unwrap();
+ let indexlist = indexlist.select(&selector).nth(1).unwrap();
+ let selector = Selector::parse("a").unwrap();
+ let mut items = Vec::new();
+ for element in indexlist.select(&selector) {
+ let href = element.value().attr("href").unwrap();
+ let href = url.join(href)?;
+ // displayed file name, class = "flex-1 truncate"
+ let selector = Selector::parse("div.flex-1.truncate").unwrap();
+ let displayed_filename = element.select(&selector).next().unwrap().inner_html();
+ let displayed_filename = displayed_filename.trim();
+ // size, class = "hidden whitespace-nowrap text-right mx-2 w-1/6 sm:block"
+ let selector = Selector::parse("div.hidden.whitespace-nowrap.text-right.mx-2").unwrap();
+ let size = element.select(&selector).next().unwrap().inner_html();
+ let size = size.trim();
+ // mtime, class = "hidden whitespace-nowrap text-right truncate ml-2 w-1/4 sm:block"
+ let selector =
+ Selector::parse("div.hidden.whitespace-nowrap.text-right.truncate.ml-2").unwrap();
+ let mtime = element.select(&selector).next().unwrap().inner_html();
+ let mtime = mtime.trim();
+
+ if displayed_filename == ".." {
+ continue;
+ }
+ let type_ = if size == "—" {
+ FileType::Directory
+ } else {
+ FileType::File
+ };
+ let date = NaiveDateTime::parse_from_str(mtime, "%Y-%m-%d %H:%M:%S")?;
+ items.push(ListItem {
+ url: href,
+ name: displayed_filename.to_string(),
+ type_,
+ size: {
+ if size == "—" {
+ None
+ } else {
+ let (n_size, unit) = FileSize::get_humanized(size);
+ Some(FileSize::HumanizedBinary(n_size, unit))
+ }
+ },
+ mtime: date,
+ })
+ }
+
+ Ok(ListResult::List(items))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use url::Url;
+
+ use crate::listing::SizeUnit;
+
+ use super::*;
+
+ #[test]
+ fn test_vyos() {
+ let client = reqwest::blocking::Client::new();
+ let items = DirectoryListerListingParser::default()
+ .get_list(
+ &client,
+ &url::Url::parse("http://localhost:1921/vyos/").unwrap(),
+ )
+ .unwrap();
+ match items {
+ ListResult::List(items) => {
+ assert_eq!(items.len(), 7);
+ assert_eq!(items[0].name, "main");
+ assert_eq!(items[0].type_, FileType::Directory);
+ assert_eq!(items[0].size, None);
+ assert_eq!(
+ items[0].mtime,
+ NaiveDateTime::parse_from_str("2023-08-07 21:11:02", "%Y-%m-%d %H:%M:%S")
+ .unwrap()
+ );
+ assert_eq!(
+ items[0].url,
+ Url::parse(
+ "http://localhost:1921/vyos/?dir=repositories/current/dists/current/main"
+ )
+ .unwrap()
+ );
+ assert_eq!(items[4].name, "Contents-amd64.gz");
+ assert_eq!(items[4].type_, FileType::File);
+ assert_eq!(
+ items[4].size,
+ Some(FileSize::HumanizedBinary(1.80, SizeUnit::M))
+ );
+ assert_eq!(
+ items[4].mtime,
+ NaiveDateTime::parse_from_str("2023-08-07 21:10:57", "%Y-%m-%d %H:%M:%S")
+ .unwrap()
+ );
+ assert_eq!(items[4].url, Url::parse("http://localhost:1921/vyos/repositories/current/dists/current/Contents-amd64.gz").unwrap());
+ }
+ _ => unreachable!(),
+ }
+ }
+
+ #[test]
+ fn test_vyos_2() {
+ let client = reqwest::blocking::Client::new();
+ let items = DirectoryListerListingParser::default()
+ .get_list(
+ &client,
+ &url::Url::parse("http://localhost:1921/vyos/vyos-accel-ppp/").unwrap(),
+ )
+ .unwrap();
+ match items {
+ ListResult::List(items) => {
+ assert_eq!(items.len(), 3);
+ }
+ _ => unreachable!(),
+ }
+ }
+}
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index 6fabb58..9d9f0fd 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -1,11 +1,13 @@
use anyhow::Result;
use clap::ValueEnum;
use reqwest::blocking::Client;
+use tracing::warn;
use url::Url;
use crate::listing::ListItem;
pub mod apache_f2;
+pub mod directory_lister;
pub mod docker;
pub mod nginx;
@@ -27,6 +29,7 @@ pub enum ParserType {
Nginx,
ApacheF2,
Docker,
+ DirectoryLister,
}
impl ParserType {
@@ -35,6 +38,10 @@ impl ParserType {
Self::Nginx => Box::::default(),
Self::ApacheF2 => Box::::default(),
Self::Docker => Box::::default(),
+ Self::DirectoryLister => {
+ warn!("html5ever parser does not support foster parenting. The result may be incorrect.");
+ Box::::default()
+ }
}
}
}