Migrate in lib

Cargo.toml

@@ -34,3 +34,5 @@ jiff = "0.2.18"
 anyhow = "1.0.100"
 whoami = "2"
 uuid = "1.20.0"
+# deprecated but ok for our usecase
+serde_yaml = "0.9"

dvs/Cargo.toml

@@ -21,6 +21,7 @@ md5.workspace = true
 jiff.workspace = true
 anyhow.workspace = true
 whoami.workspace = true
+serde_yaml.workspace = true
 uuid = { version = "1.20.0", features = ["v4"] }
 
 [target.'cfg(unix)'.dependencies]

dvs/src/config.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use crate::backends::Backend as BackendTrait;
 use crate::backends::local::LocalBackend;
-use crate::paths::{CONFIG_FILE_NAME, DEFAULT_FOLDER_NAME, find_repo_root};
+use crate::paths::{CONFIG_FILE_NAME, DEFAULT_METADATA_FOLDER_NAME, find_repo_root};
 use anyhow::{Context, Result};
 use fs_err as fs;
 use serde::{Deserialize, Serialize};
@@ -19,7 +19,7 @@ pub struct Config {
     /// at the root of the repository
     /// If this option is set, dvs will use that folder name instead of `.dvs`
     metadata_folder_name: Option<String>,
-    backend: Backend,
+    pub backend: Backend,
 }
 
 impl Config {
@@ -70,7 +70,7 @@ impl Config {
         if let Some(name) = &self.metadata_folder_name {
             name.as_str()
         } else {
-            DEFAULT_FOLDER_NAME
+            DEFAULT_METADATA_FOLDER_NAME
         }
     }
 

dvs/src/file.rs

@@ -1,3 +1,4 @@
+use std::io::BufReader;
 use std::path::{Path, PathBuf};
 
 use crate::audit::{AuditEntry, AuditFile};
@@ -56,9 +57,9 @@ impl FileMetadata {
             bail!("Path {} is not a file", path.as_ref().display());
         }
 
-        let content = fs::read(path.as_ref())?;
-        let size = content.len() as u64;
-        let hashes = Hashes::from(content);
+        let file = fs::File::open(path.as_ref())?;
+        let size = file.metadata()?.len();
+        let hashes = Hashes::from_reader(BufReader::new(file))?;
         let created_by = whoami::username()?;
         let add_time = jiff::Zoned::now().to_string();
 

dvs/src/hashes.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use std::fmt::Display;
+use std::io::BufRead;
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)]
 #[serde(rename_all = "lowercase")]
@@ -17,19 +18,28 @@ pub struct Hashes {
     pub md5: String,
 }
 
-impl From<Vec<u8>> for Hashes {
-    fn from(bytes: Vec<u8>) -> Self {
-        let blake3_hash = format!("{}", blake3::hash(&bytes));
-        let md5_hash = format!("{:x}", md5::compute(&bytes));
+impl Hashes {
+    pub fn from_reader<R: BufRead>(mut reader: R) -> std::io::Result<Self> {
+        let mut blake3_hasher = blake3::Hasher::new();
+        let mut md5_context = md5::Context::new();
 
-        Self {
-            blake3: blake3_hash,
-            md5: md5_hash,
+        loop {
+            let buf = reader.fill_buf()?;
+            if buf.is_empty() {
+                break;
+            }
+            blake3_hasher.update(buf);
+            md5_context.consume(buf);
+            let len = buf.len();
+            reader.consume(len);
         }
+
+        Ok(Self {
+            blake3: blake3_hasher.finalize().to_string(),
+            md5: format!("{:x}", md5_context.finalize()),
+        })
     }
-}
 
-impl Hashes {
     pub fn get_by_alg(&self, alg: HashAlg) -> &str {
         match alg {
             HashAlg::Blake3 => &self.blake3,

dvs/src/lib.rs

@@ -4,6 +4,7 @@ pub mod config;
 pub mod file;
 mod hashes;
 pub mod init;
+pub mod migrate;
 pub mod paths;
 
 pub use backends::Backend;