Async FilesystemStore

fuzz/Cargo.toml

@@ -22,8 +22,10 @@ lightning = { path = "../lightning", features = ["regex", "_test_utils"] }
 lightning-invoice = { path = "../lightning-invoice" }
 lightning-liquidity = { path = "../lightning-liquidity" }
 lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" }
+lightning-persister = { path = "../lightning-persister", features = ["tokio"]}
 bech32 = "0.11.0"
 bitcoin = { version = "0.32.2", features = ["secp-lowmemory"] }
+tokio = { version = "~1.35", default-features = false, features = ["rt-multi-thread"] }
 
 afl = { version = "0.12", optional = true }
 honggfuzz = { version = "0.5", optional = true, default-features = false }

fuzz/ci-fuzz.sh

@@ -38,7 +38,7 @@ for TARGET in src/bin/*.rs; do
 		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -F 64 -N1000"
 	elif [ "$FILE" = "process_network_graph_target" -o "$FILE" = "full_stack_target" -o "$FILE" = "router_target" -o "$FILE" = "lsps_message_target" ]; then
 		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -N10000"
-	elif [ "$FILE" = "indexedmap_target" ]; then
+	elif [ "$FILE" = "indexedmap_target" -o "$FILE" = "fs_store_target" ]; then
 		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -N100000"
 	else
 		HFUZZ_RUN_ARGS="$HFUZZ_RUN_ARGS -N1000000"

fuzz/src/bin/fs_store_target.rs

@@ -0,0 +1,120 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+// This file is auto-generated by gen_target.sh based on target_template.txt
+// To modify it, modify target_template.txt and run gen_target.sh instead.
+
+#![cfg_attr(feature = "libfuzzer_fuzz", no_main)]
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+#[cfg(not(fuzzing))]
+compile_error!("Fuzz targets need cfg=fuzzing");
+
+#[cfg(not(hashes_fuzz))]
+compile_error!("Fuzz targets need cfg=hashes_fuzz");
+
+#[cfg(not(secp256k1_fuzz))]
+compile_error!("Fuzz targets need cfg=secp256k1_fuzz");
+
+extern crate lightning_fuzz;
+use lightning_fuzz::fs_store::*;
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+	fuzz!(|data| {
+		fs_store_run(data.as_ptr(), data.len());
+	});
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+	loop {
+		fuzz!(|data| {
+			fs_store_run(data.as_ptr(), data.len());
+		});
+	}
+}
+
+#[cfg(feature = "libfuzzer_fuzz")]
+#[macro_use] extern crate libfuzzer_sys;
+#[cfg(feature = "libfuzzer_fuzz")]
+fuzz_target!(|data: &[u8]| {
+	fs_store_run(data.as_ptr(), data.len());
+});
+
+#[cfg(feature = "stdin_fuzz")]
+fn main() {
+	use std::io::Read;
+
+	let mut data = Vec::with_capacity(8192);
+	std::io::stdin().read_to_end(&mut data).unwrap();
+	fs_store_run(data.as_ptr(), data.len());
+}
+
+#[test]
+fn run_test_cases() {
+	use std::fs;
+	use std::io::Read;
+	use lightning_fuzz::utils::test_logger::StringBuffer;
+
+	use std::sync::{atomic, Arc};
+	{
+		let data: Vec<u8> = vec![0];
+		fs_store_run(data.as_ptr(), data.len());
+	}
+	let mut threads = Vec::new();
+	let threads_running = Arc::new(atomic::AtomicUsize::new(0));
+	if let Ok(tests) = fs::read_dir("test_cases/fs_store") {
+		for test in tests {
+			let mut data: Vec<u8> = Vec::new();
+			let path = test.unwrap().path();
+			fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
+			threads_running.fetch_add(1, atomic::Ordering::AcqRel);
+
+			let thread_count_ref = Arc::clone(&threads_running);
+			let main_thread_ref = std::thread::current();
+			threads.push((path.file_name().unwrap().to_str().unwrap().to_string(),
+				std::thread::spawn(move || {
+					let string_logger = StringBuffer::new();
+
+					let panic_logger = string_logger.clone();
+					let res = if ::std::panic::catch_unwind(move || {
+						fs_store_test(&data, panic_logger);
+					}).is_err() {
+						Some(string_logger.into_string())
+					} else { None };
+					thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel);
+					main_thread_ref.unpark();
+					res
+				})
+			));
+			while threads_running.load(atomic::Ordering::Acquire) > 32 {
+				std::thread::park();
+			}
+		}
+	}
+	let mut failed_outputs = Vec::new();
+	for (test, thread) in threads.drain(..) {
+		if let Some(output) = thread.join().unwrap() {
+			println!("\nOutput of {}:\n{}\n", test, output);
+			failed_outputs.push(test);
+		}
+	}
+	if !failed_outputs.is_empty() {
+		println!("Test cases which failed: ");
+		for case in failed_outputs {
+			println!("{}", case);
+		}
+		panic!();
+	}
+}

fuzz/src/bin/gen_target.sh

@@ -28,6 +28,7 @@ GEN_TEST base32
 GEN_TEST fromstr_to_netaddress
 GEN_TEST feature_flags
 GEN_TEST lsps_message
+GEN_TEST fs_store
 
 GEN_TEST msg_accept_channel msg_targets::
 GEN_TEST msg_announcement_signatures msg_targets::

fuzz/src/fs_store.rs

@@ -0,0 +1,184 @@
+use core::hash::{BuildHasher, Hasher};
+use lightning::util::persist::{KVStore, KVStoreSync};
+use lightning_persister::fs_store::FilesystemStore;
+use std::fs;
+use tokio::runtime::Runtime;
+
+use crate::utils::test_logger;
+
+struct TempFilesystemStore {
+	temp_path: std::path::PathBuf,
+	inner: FilesystemStore,
+}
+
+impl TempFilesystemStore {
+	fn new() -> Self {
+		const SHM_PATH: &str = "/dev/shm";
+		let mut temp_path = if std::path::Path::new(SHM_PATH).exists() {
+			std::path::PathBuf::from(SHM_PATH)
+		} else {
+			std::env::temp_dir()
+		};
+
+		let random_number = std::collections::hash_map::RandomState::new().build_hasher().finish();
+		let random_folder_name = format!("fs_store_fuzz_{:016x}", random_number);
+		temp_path.push(random_folder_name);
+
+		let inner = FilesystemStore::new(temp_path.clone());
+		TempFilesystemStore { inner, temp_path }
+	}
+}
+
+impl Drop for TempFilesystemStore {
+	fn drop(&mut self) {
+		_ = fs::remove_dir_all(&self.temp_path)
+	}
+}
+
+/// Actual fuzz test, method signature and name are fixed
+fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
+	let rt = Runtime::new().unwrap();
+	rt.block_on(do_test_internal(data, out));
+}
+
+async fn do_test_internal<Out: test_logger::Output>(data: &[u8], _out: Out) {
+	let mut read_pos = 0;
+	macro_rules! get_slice {
+		($len: expr) => {{
+			let slice_len = $len as usize;
+			if data.len() < read_pos + slice_len {
+				None
+			} else {
+				read_pos += slice_len;
+				Some(&data[read_pos - slice_len..read_pos])
+			}
+		}};
+	}
+
+	let temp_fs_store = TempFilesystemStore::new();
+	let fs_store = &temp_fs_store.inner;
+
+	let primary_namespace = "primary";
+	let secondary_namespace = "secondary";
+	let key = "key";
+
+	let mut next_data_value = 0u64;
+	let mut get_next_data_value = || {
+		let data_value = next_data_value.to_be_bytes().to_vec();
+		next_data_value += 1;
+
+		data_value
+	};
+
+	let mut current_data = None;
+
+	let mut handles = Vec::new();
+	loop {
+		let v = match get_slice!(1) {
+			Some(b) => b[0],
+			None => break,
+		};
+		match v % 13 {
+			// Sync write
+			0 => {
+				let data_value = get_next_data_value();
+
+				KVStoreSync::write(
+					fs_store,
+					primary_namespace,
+					secondary_namespace,
+					key,
+					data_value.clone(),
+				)
+				.unwrap();
+
+				current_data = Some(data_value);
+			},
+			// Sync remove
+			1 => {
+				KVStoreSync::remove(fs_store, primary_namespace, secondary_namespace, key, false)
+					.unwrap();
+
+				current_data = None;
+			},
+			// Sync list
+			2 => {
+				KVStoreSync::list(fs_store, primary_namespace, secondary_namespace).unwrap();
+			},
+			// Sync read
+			3 => {
+				_ = KVStoreSync::read(fs_store, primary_namespace, secondary_namespace, key);
+			},
+			// Async write. Bias writes a bit.
+			4..=9 => {
+				let data_value = get_next_data_value();
+
+				let fut = KVStore::write(
+					fs_store,
+					primary_namespace,
+					secondary_namespace,
+					key,
+					data_value.clone(),
+				);
+
+				// Already set the current_data, even though writing hasn't finished yet. This supports the call-time
+				// ordering semantics.
+				current_data = Some(data_value);
+
+				let handle = tokio::task::spawn(fut);
+
+				// Store the handle to later await the result.
+				handles.push(handle);
+			},
+			// Async remove
+			10 | 11 => {
+				let lazy = v == 10;
+				let fut =
+					KVStore::remove(fs_store, primary_namespace, secondary_namespace, key, lazy);
+
+				// Already set the current_data, even though writing hasn't finished yet. This supports the call-time
+				// ordering semantics.
+				current_data = None;
+
+				let handle = tokio::task::spawn(fut);
+				handles.push(handle);
+			},
+			// Join tasks.
+			12 => {
+				for handle in handles.drain(..) {
+					let _ = handle.await.unwrap();
+				}
+			},
+			_ => unreachable!(),
+		}
+
+		// If no more writes are pending, we can reliably see if the data is consistent.
+		if handles.is_empty() {
+			let data_value =
+				KVStoreSync::read(fs_store, primary_namespace, secondary_namespace, key).ok();
+			assert_eq!(data_value, current_data);
+
+			let list = KVStoreSync::list(fs_store, primary_namespace, secondary_namespace).unwrap();
+			assert_eq!(list.is_empty(), current_data.is_none());
+
+			assert_eq!(0, fs_store.state_size());
+		}
+	}
+
+	// Always make sure that all async tasks are completed before returning. Otherwise the temporary storage dir could
+	// be removed, and then again recreated by unfinished tasks.
+	for handle in handles.drain(..) {
+		let _ = handle.await.unwrap();
+	}
+}
+
+/// Method that needs to be added manually, {name}_test
+pub fn fs_store_test<Out: test_logger::Output>(data: &[u8], out: Out) {
+	do_test(data, out);
+}
+
+/// Method that needs to be added manually, {name}_run
+#[no_mangle]
+pub extern "C" fn fs_store_run(data: *const u8, datalen: usize) {
+	do_test(unsafe { std::slice::from_raw_parts(data, datalen) }, test_logger::DevNull {});
+}

fuzz/src/lib.rs

@@ -9,6 +9,7 @@
 
 extern crate bitcoin;
 extern crate lightning;
+extern crate lightning_persister;
 extern crate lightning_rapid_gossip_sync;
 
 #[cfg(not(fuzzing))]
@@ -45,4 +46,5 @@ pub mod router;
 pub mod static_invoice_deser;
 pub mod zbase32;
 
+pub mod fs_store;
 pub mod msg_targets;

fuzz/targets.h

@@ -21,6 +21,7 @@ void base32_run(const unsigned char* data, size_t data_len);
 void fromstr_to_netaddress_run(const unsigned char* data, size_t data_len);
 void feature_flags_run(const unsigned char* data, size_t data_len);
 void lsps_message_run(const unsigned char* data, size_t data_len);
+void fs_store_run(const unsigned char* data, size_t data_len);
 void msg_accept_channel_run(const unsigned char* data, size_t data_len);
 void msg_announcement_signatures_run(const unsigned char* data, size_t data_len);
 void msg_channel_reestablish_run(const unsigned char* data, size_t data_len);

lightning-persister/Cargo.toml

@@ -16,6 +16,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 [dependencies]
 bitcoin = "0.32.2"
 lightning = { version = "0.2.0", path = "../lightning" }
+tokio = { version = "1.35", optional = true, default-features = false, features = ["rt-multi-thread"] }
 
 [target.'cfg(windows)'.dependencies]
 windows-sys = { version = "0.48.0", default-features = false, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] }
@@ -26,6 +27,7 @@ criterion = { version = "0.4", optional = true, default-features = false }
 [dev-dependencies]
 lightning = { version = "0.2.0", path = "../lightning", features = ["_test_utils"] }
 bitcoin = { version = "0.32.2", default-features = false }
+tokio = { version = "1.35", default-features = false, features = ["macros"] }
 
 [lints]
 workspace = true