diff options
| -rw-r--r-- | examples/demo1.rs | 13 | ||||
| -rw-r--r-- | src/wal.rs | 22 | ||||
| -rw-r--r-- | tests/common/mod.rs | 117 | ||||
| -rw-r--r-- | tests/rand_fail.rs | 25 |
4 files changed, 160 insertions, 17 deletions
diff --git a/examples/demo1.rs b/examples/demo1.rs index e0c7d58..e395adf 100644 --- a/examples/demo1.rs +++ b/examples/demo1.rs @@ -49,11 +49,12 @@ impl WALFile for WALFileTest { self.filename, offset, offset + data.len() as u64, hex::encode(&data)); pwrite(self.fd, &*data, offset as off_t).unwrap(); } - fn read(&self, offset: WALPos, length: usize) -> WALBytes { + fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes> { let mut buff = Vec::new(); buff.resize(length, 0); - pread(self.fd, &mut buff[..], offset as off_t).unwrap(); - buff.into_boxed_slice() + if pread(self.fd, &mut buff[..], offset as off_t).unwrap() == length { + Some(buff.into_boxed_slice()) + } else { None } } } @@ -89,13 +90,13 @@ impl Drop for WALStoreTest { impl WALStore for WALStoreTest { type FileNameIter = std::vec::IntoIter<String>; - fn open_file(&self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> { + fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> { println!("open_file(filename={}, touch={})", filename, touch); let filename = filename.to_string(); Some(Box::new(WALFileTest::new(self.rootfd, &filename))) } - fn remove_file(&self, filename: &str) -> Result<(), ()> { + fn remove_file(&mut self, filename: &str) -> Result<(), ()> { println!("remove_file(filename={})", filename); unlinkat(Some(self.rootfd), filename, UnlinkatFlags::NoRemoveDir).or_else(|_| Err(())) } @@ -109,7 +110,7 @@ impl WALStore for WALStoreTest { logfiles.into_iter() } - fn apply_payload(&self, payload: WALBytes) { + fn apply_payload(&mut self, payload: WALBytes) { println!("apply_payload(payload={})", std::str::from_utf8(&payload).unwrap()) } } @@ -62,23 +62,23 @@ pub trait WALFile { /// should be _atomic_ (the entire single write should be all or nothing). fn write(&self, offset: WALPos, data: WALBytes); /// Read data with offset. - fn read(&self, offset: WALPos, length: usize) -> WALBytes; + fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes>; } pub trait WALStore { type FileNameIter: Iterator<Item = String>; /// Open a file given the filename, create the file if not exists when `touch` is `true`. - fn open_file(&self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>>; + fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>>; /// Unlink a file given the filename. - fn remove_file(&self, filename: &str) -> Result<(), ()>; + fn remove_file(&mut self, filename: &str) -> Result<(), ()>; /// Enumerate all WAL filenames. It should include all WAL files that are previously opened /// (created) but not removed. The list could be unordered. fn enumerate_files(&self) -> Self::FileNameIter; /// Apply the payload during recovery. This notifies the application should redo the given /// operation to ensure its state is consistent (the major goal of having a WAL). We assume /// the application applies the payload by the _order_ of this callback invocation. - fn apply_payload(&self, payload: WALBytes); + fn apply_payload(&mut self, payload: WALBytes); } /// The middle layer that manages WAL file handles and invokes public trait functions to actually @@ -148,7 +148,7 @@ impl<F: WALStore> WALFilePool<F> { } } - fn remove_file(&self, fid: u64) -> Result<(), ()> { + fn remove_file(&mut self, fid: u64) -> Result<(), ()> { self.store.remove_file(&Self::get_fname(fid)) } @@ -314,8 +314,8 @@ impl<F: WALStore> WALLoader<F> { let fid = self.file_pool.get_fid(fname); let f = self.file_pool.get_file(fid, false); let mut off = 0; - while block_size - (off & (block_size - 1)) > msize as u64 { - let header_raw = f.read(off, msize as usize); + while let Some(header_raw) = f.read(off, msize as usize) { + if block_size - (off & (block_size - 1)) <= msize as u64 { break } off += msize as u64; let header = unsafe { std::mem::transmute::<*const u8, &WALRingBlob>(header_raw.as_ptr())}; @@ -323,21 +323,21 @@ impl<F: WALStore> WALLoader<F> { match header.rtype { WALRingType::Full => { assert!(chunks.is_none()); - let payload = f.read(off, rsize as usize); + let payload = f.read(off, rsize as usize).unwrap(); off += rsize as u64; self.file_pool.store.apply_payload(payload); }, WALRingType::First => { assert!(chunks.is_none()); - chunks = Some(vec![f.read(off, rsize as usize)]); + chunks = Some(vec![f.read(off, rsize as usize).unwrap()]); off += rsize as u64; }, WALRingType::Middle => { - chunks.as_mut().unwrap().push(f.read(off, rsize as usize)); + chunks.as_mut().unwrap().push(f.read(off, rsize as usize).unwrap()); off += rsize as u64; }, WALRingType::Last => { - chunks.as_mut().unwrap().push(f.read(off, rsize as usize)); + chunks.as_mut().unwrap().push(f.read(off, rsize as usize).unwrap()); off += rsize as u64; let _chunks = chunks.take().unwrap(); diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..fd98539 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,117 @@ +#[cfg(test)] +#[allow(dead_code)] + +extern crate growthring; +use growthring::wal::{WALFile, WALStore, WALPos, WALBytes}; +use std::collections::{HashMap, hash_map::Entry}; +use std::cell::RefCell; +use std::rc::Rc; + +/* +thread_local! { + //pub static RNG: RefCell<rand::rngs::StdRng> = RefCell::new(<rand::rngs::StdRng as rand::SeedableRng>::from_seed([0; 32])); + pub static RNG: RefCell<rand::rngs::ThreadRng> = RefCell::new(rand::thread_rng()); +} + +pub fn gen_rand_letters(i: usize) -> String { + //let mut rng = rand::thread_rng(); + RNG.with(|rng| { + (0..i).map(|_| (rng.borrow_mut().gen::<u8>() % 26 + 'a' as u8) as char).collect() + }) +} +*/ + +struct FileContentEmul(RefCell<Vec<u8>>); + +impl FileContentEmul { + pub fn new() -> Self { FileContentEmul(RefCell::new(Vec::new())) } +} + +impl std::ops::Deref for FileContentEmul { + type Target = RefCell<Vec<u8>>; + fn deref(&self) -> &Self::Target {&self.0} +} + +/// Emulate the a virtual file handle. +pub struct WALFileEmul { + file: Rc<FileContentEmul> +} + +impl WALFile for WALFileEmul { + fn allocate(&self, offset: WALPos, length: usize) -> Result<(), ()> { + let offset = offset as usize; + if offset + length > self.file.borrow().len() { + self.file.borrow_mut().resize(offset + length, 0) + } + for v in &mut self.file.borrow_mut()[offset..offset + length] { *v = 0 } + Ok(()) + } + + fn truncate(&self, length: usize) -> Result<(), ()> { + self.file.borrow_mut().resize(length, 0); + Ok(()) + } + + fn write(&self, offset: WALPos, data: WALBytes) { + let offset = offset as usize; + &self.file.borrow_mut()[offset..offset + data.len()].copy_from_slice(&data); + } + + fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes> { + let offset = offset as usize; + let file = self.file.borrow(); + if offset + length > file.len() { None } + else { + Some((&file[offset..offset + length]).to_vec().into_boxed_slice()) + } + } +} + +pub struct WALStoreEmulState{ + files: HashMap<String, Rc<FileContentEmul>>, +} + +impl WALStoreEmulState { + pub fn new() -> Self { WALStoreEmulState { files: HashMap::new() } } +} + +/// Emulate the persistent storage state. +pub struct WALStoreEmul<'a>(&'a mut WALStoreEmulState); + +impl<'a> WALStoreEmul<'a> { + pub fn new(state: &'a mut WALStoreEmulState) -> Self { + WALStoreEmul(state) + } +} + +impl<'a> WALStore for WALStoreEmul<'a> { + type FileNameIter = std::vec::IntoIter<String>; + + fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> { + match self.0.files.entry(filename.to_string()) { + Entry::Occupied(e) => Some(Box::new(WALFileEmul { file: e.get().clone() })), + Entry::Vacant(e) => if touch { + Some(Box::new( + WALFileEmul { file: e.insert(Rc::new(FileContentEmul::new())).clone() })) + } else { + None + } + } + } + + fn remove_file(&mut self, filename: &str) -> Result<(), ()> { + self.0.files.remove(filename).ok_or(()).and_then(|_| Ok(())) + } + + fn enumerate_files(&self) -> Self::FileNameIter { + let mut logfiles = Vec::new(); + for (fname, _) in self.0.files.iter() { + logfiles.push(fname.clone()) + } + logfiles.into_iter() + } + + fn apply_payload(&mut self, payload: WALBytes) { + println!("apply_payload(payload={})", std::str::from_utf8(&payload).unwrap()) + } +} diff --git a/tests/rand_fail.rs b/tests/rand_fail.rs new file mode 100644 index 0000000..988da05 --- /dev/null +++ b/tests/rand_fail.rs @@ -0,0 +1,25 @@ +#[cfg(test)] + +extern crate growthring; +use growthring::wal::{WALLoader, WALWriter, WALRingId, WALBytes}; + +mod common; + +fn test(records: Vec<String>, wal: &mut WALWriter<common::WALStoreEmul>) -> Box<[WALRingId]> { + let records: Vec<WALBytes> = records.into_iter().map(|s| s.into_bytes().into_boxed_slice()).collect(); + let ret = wal.grow(&records); + for ring_id in ret.iter() { + println!("got ring id: {:?}", ring_id); + } + ret +} + +#[test] +fn test_rand_fail() { + let mut state = common::WALStoreEmulState::new(); + let mut wal = WALLoader::new(common::WALStoreEmul::new(&mut state), 9, 8, 1000).recover(); + for _ in 0..3 { + test(["hi", "hello", "lol"].iter().map(|s| s.to_string()).collect::<Vec<String>>(), &mut wal); + } + let mut wal = WALLoader::new(common::WALStoreEmul::new(&mut state), 9, 8, 1000).recover(); +} |