aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2020-06-10 14:40:36 -0400
committerDeterminant <[email protected]>2020-06-10 14:40:36 -0400
commit415578aa48a12cac01d763b7c08384dbcd46911b (patch)
treec01d7e28e38caac6662215469a172dd99f576654
parent81eb9ce9299b2e31471db3f00457ff3c1a41fb60 (diff)
WIP: random failure test (with emulated storage)
-rw-r--r--examples/demo1.rs13
-rw-r--r--src/wal.rs22
-rw-r--r--tests/common/mod.rs117
-rw-r--r--tests/rand_fail.rs25
4 files changed, 160 insertions, 17 deletions
diff --git a/examples/demo1.rs b/examples/demo1.rs
index e0c7d58..e395adf 100644
--- a/examples/demo1.rs
+++ b/examples/demo1.rs
@@ -49,11 +49,12 @@ impl WALFile for WALFileTest {
self.filename, offset, offset + data.len() as u64, hex::encode(&data));
pwrite(self.fd, &*data, offset as off_t).unwrap();
}
- fn read(&self, offset: WALPos, length: usize) -> WALBytes {
+ fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes> {
let mut buff = Vec::new();
buff.resize(length, 0);
- pread(self.fd, &mut buff[..], offset as off_t).unwrap();
- buff.into_boxed_slice()
+ if pread(self.fd, &mut buff[..], offset as off_t).unwrap() == length {
+ Some(buff.into_boxed_slice())
+ } else { None }
}
}
@@ -89,13 +90,13 @@ impl Drop for WALStoreTest {
impl WALStore for WALStoreTest {
type FileNameIter = std::vec::IntoIter<String>;
- fn open_file(&self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> {
+ fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> {
println!("open_file(filename={}, touch={})", filename, touch);
let filename = filename.to_string();
Some(Box::new(WALFileTest::new(self.rootfd, &filename)))
}
- fn remove_file(&self, filename: &str) -> Result<(), ()> {
+ fn remove_file(&mut self, filename: &str) -> Result<(), ()> {
println!("remove_file(filename={})", filename);
unlinkat(Some(self.rootfd), filename, UnlinkatFlags::NoRemoveDir).or_else(|_| Err(()))
}
@@ -109,7 +110,7 @@ impl WALStore for WALStoreTest {
logfiles.into_iter()
}
- fn apply_payload(&self, payload: WALBytes) {
+ fn apply_payload(&mut self, payload: WALBytes) {
println!("apply_payload(payload={})", std::str::from_utf8(&payload).unwrap())
}
}
diff --git a/src/wal.rs b/src/wal.rs
index 0ba35fb..b5b3e91 100644
--- a/src/wal.rs
+++ b/src/wal.rs
@@ -62,23 +62,23 @@ pub trait WALFile {
/// should be _atomic_ (the entire single write should be all or nothing).
fn write(&self, offset: WALPos, data: WALBytes);
/// Read data with offset.
- fn read(&self, offset: WALPos, length: usize) -> WALBytes;
+ fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes>;
}
pub trait WALStore {
type FileNameIter: Iterator<Item = String>;
/// Open a file given the filename, create the file if not exists when `touch` is `true`.
- fn open_file(&self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>>;
+ fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>>;
/// Unlink a file given the filename.
- fn remove_file(&self, filename: &str) -> Result<(), ()>;
+ fn remove_file(&mut self, filename: &str) -> Result<(), ()>;
/// Enumerate all WAL filenames. It should include all WAL files that are previously opened
/// (created) but not removed. The list could be unordered.
fn enumerate_files(&self) -> Self::FileNameIter;
/// Apply the payload during recovery. This notifies the application should redo the given
/// operation to ensure its state is consistent (the major goal of having a WAL). We assume
/// the application applies the payload by the _order_ of this callback invocation.
- fn apply_payload(&self, payload: WALBytes);
+ fn apply_payload(&mut self, payload: WALBytes);
}
/// The middle layer that manages WAL file handles and invokes public trait functions to actually
@@ -148,7 +148,7 @@ impl<F: WALStore> WALFilePool<F> {
}
}
- fn remove_file(&self, fid: u64) -> Result<(), ()> {
+ fn remove_file(&mut self, fid: u64) -> Result<(), ()> {
self.store.remove_file(&Self::get_fname(fid))
}
@@ -314,8 +314,8 @@ impl<F: WALStore> WALLoader<F> {
let fid = self.file_pool.get_fid(fname);
let f = self.file_pool.get_file(fid, false);
let mut off = 0;
- while block_size - (off & (block_size - 1)) > msize as u64 {
- let header_raw = f.read(off, msize as usize);
+ while let Some(header_raw) = f.read(off, msize as usize) {
+ if block_size - (off & (block_size - 1)) <= msize as u64 { break }
off += msize as u64;
let header = unsafe {
std::mem::transmute::<*const u8, &WALRingBlob>(header_raw.as_ptr())};
@@ -323,21 +323,21 @@ impl<F: WALStore> WALLoader<F> {
match header.rtype {
WALRingType::Full => {
assert!(chunks.is_none());
- let payload = f.read(off, rsize as usize);
+ let payload = f.read(off, rsize as usize).unwrap();
off += rsize as u64;
self.file_pool.store.apply_payload(payload);
},
WALRingType::First => {
assert!(chunks.is_none());
- chunks = Some(vec![f.read(off, rsize as usize)]);
+ chunks = Some(vec![f.read(off, rsize as usize).unwrap()]);
off += rsize as u64;
},
WALRingType::Middle => {
- chunks.as_mut().unwrap().push(f.read(off, rsize as usize));
+ chunks.as_mut().unwrap().push(f.read(off, rsize as usize).unwrap());
off += rsize as u64;
},
WALRingType::Last => {
- chunks.as_mut().unwrap().push(f.read(off, rsize as usize));
+ chunks.as_mut().unwrap().push(f.read(off, rsize as usize).unwrap());
off += rsize as u64;
let _chunks = chunks.take().unwrap();
diff --git a/tests/common/mod.rs b/tests/common/mod.rs
new file mode 100644
index 0000000..fd98539
--- /dev/null
+++ b/tests/common/mod.rs
@@ -0,0 +1,117 @@
+#[cfg(test)]
+#[allow(dead_code)]
+
+extern crate growthring;
+use growthring::wal::{WALFile, WALStore, WALPos, WALBytes};
+use std::collections::{HashMap, hash_map::Entry};
+use std::cell::RefCell;
+use std::rc::Rc;
+
+/*
+thread_local! {
+ //pub static RNG: RefCell<rand::rngs::StdRng> = RefCell::new(<rand::rngs::StdRng as rand::SeedableRng>::from_seed([0; 32]));
+ pub static RNG: RefCell<rand::rngs::ThreadRng> = RefCell::new(rand::thread_rng());
+}
+
+pub fn gen_rand_letters(i: usize) -> String {
+ //let mut rng = rand::thread_rng();
+ RNG.with(|rng| {
+ (0..i).map(|_| (rng.borrow_mut().gen::<u8>() % 26 + 'a' as u8) as char).collect()
+ })
+}
+*/
+
+struct FileContentEmul(RefCell<Vec<u8>>);
+
+impl FileContentEmul {
+ pub fn new() -> Self { FileContentEmul(RefCell::new(Vec::new())) }
+}
+
+impl std::ops::Deref for FileContentEmul {
+ type Target = RefCell<Vec<u8>>;
+ fn deref(&self) -> &Self::Target {&self.0}
+}
+
+/// Emulate the a virtual file handle.
+pub struct WALFileEmul {
+ file: Rc<FileContentEmul>
+}
+
+impl WALFile for WALFileEmul {
+ fn allocate(&self, offset: WALPos, length: usize) -> Result<(), ()> {
+ let offset = offset as usize;
+ if offset + length > self.file.borrow().len() {
+ self.file.borrow_mut().resize(offset + length, 0)
+ }
+ for v in &mut self.file.borrow_mut()[offset..offset + length] { *v = 0 }
+ Ok(())
+ }
+
+ fn truncate(&self, length: usize) -> Result<(), ()> {
+ self.file.borrow_mut().resize(length, 0);
+ Ok(())
+ }
+
+ fn write(&self, offset: WALPos, data: WALBytes) {
+ let offset = offset as usize;
+ &self.file.borrow_mut()[offset..offset + data.len()].copy_from_slice(&data);
+ }
+
+ fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes> {
+ let offset = offset as usize;
+ let file = self.file.borrow();
+ if offset + length > file.len() { None }
+ else {
+ Some((&file[offset..offset + length]).to_vec().into_boxed_slice())
+ }
+ }
+}
+
+pub struct WALStoreEmulState{
+ files: HashMap<String, Rc<FileContentEmul>>,
+}
+
+impl WALStoreEmulState {
+ pub fn new() -> Self { WALStoreEmulState { files: HashMap::new() } }
+}
+
+/// Emulate the persistent storage state.
+pub struct WALStoreEmul<'a>(&'a mut WALStoreEmulState);
+
+impl<'a> WALStoreEmul<'a> {
+ pub fn new(state: &'a mut WALStoreEmulState) -> Self {
+ WALStoreEmul(state)
+ }
+}
+
+impl<'a> WALStore for WALStoreEmul<'a> {
+ type FileNameIter = std::vec::IntoIter<String>;
+
+ fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> {
+ match self.0.files.entry(filename.to_string()) {
+ Entry::Occupied(e) => Some(Box::new(WALFileEmul { file: e.get().clone() })),
+ Entry::Vacant(e) => if touch {
+ Some(Box::new(
+ WALFileEmul { file: e.insert(Rc::new(FileContentEmul::new())).clone() }))
+ } else {
+ None
+ }
+ }
+ }
+
+ fn remove_file(&mut self, filename: &str) -> Result<(), ()> {
+ self.0.files.remove(filename).ok_or(()).and_then(|_| Ok(()))
+ }
+
+ fn enumerate_files(&self) -> Self::FileNameIter {
+ let mut logfiles = Vec::new();
+ for (fname, _) in self.0.files.iter() {
+ logfiles.push(fname.clone())
+ }
+ logfiles.into_iter()
+ }
+
+ fn apply_payload(&mut self, payload: WALBytes) {
+ println!("apply_payload(payload={})", std::str::from_utf8(&payload).unwrap())
+ }
+}
diff --git a/tests/rand_fail.rs b/tests/rand_fail.rs
new file mode 100644
index 0000000..988da05
--- /dev/null
+++ b/tests/rand_fail.rs
@@ -0,0 +1,25 @@
+#[cfg(test)]
+
+extern crate growthring;
+use growthring::wal::{WALLoader, WALWriter, WALRingId, WALBytes};
+
+mod common;
+
+fn test(records: Vec<String>, wal: &mut WALWriter<common::WALStoreEmul>) -> Box<[WALRingId]> {
+ let records: Vec<WALBytes> = records.into_iter().map(|s| s.into_bytes().into_boxed_slice()).collect();
+ let ret = wal.grow(&records);
+ for ring_id in ret.iter() {
+ println!("got ring id: {:?}", ring_id);
+ }
+ ret
+}
+
+#[test]
+fn test_rand_fail() {
+ let mut state = common::WALStoreEmulState::new();
+ let mut wal = WALLoader::new(common::WALStoreEmul::new(&mut state), 9, 8, 1000).recover();
+ for _ in 0..3 {
+ test(["hi", "hello", "lol"].iter().map(|s| s.to_string()).collect::<Vec<String>>(), &mut wal);
+ }
+ let mut wal = WALLoader::new(common::WALStoreEmul::new(&mut state), 9, 8, 1000).recover();
+}