WIP: random failure test (with emulated storage)

4 files changed, 160 insertions, 17 deletions

diff --git a/examples/demo1.rs b/examples/demo1.rs
index e0c7d58..e395adf 100644
--- a/examples/demo1.rs
+++ b/examples/demo1.rs
@@ -49,11 +49,12 @@ impl WALFile for WALFileTest {
                 self.filename, offset, offset + data.len() as u64, hex::encode(&data));
         pwrite(self.fd, &*data, offset as off_t).unwrap();
     }
-    fn read(&self, offset: WALPos, length: usize) -> WALBytes {
+    fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes> {
         let mut buff = Vec::new();
         buff.resize(length, 0);
-        pread(self.fd, &mut buff[..], offset as off_t).unwrap();
-        buff.into_boxed_slice()
+        if pread(self.fd, &mut buff[..], offset as off_t).unwrap() == length {
+            Some(buff.into_boxed_slice())
+        } else { None }
     }
 }
 
@@ -89,13 +90,13 @@ impl Drop for WALStoreTest {
 impl WALStore for WALStoreTest {
     type FileNameIter = std::vec::IntoIter<String>;
 
-    fn open_file(&self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> {
+    fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> {
         println!("open_file(filename={}, touch={})", filename, touch);
         let filename = filename.to_string();
         Some(Box::new(WALFileTest::new(self.rootfd, &filename)))
     }
 
-    fn remove_file(&self, filename: &str) -> Result<(), ()> {
+    fn remove_file(&mut self, filename: &str) -> Result<(), ()> {
         println!("remove_file(filename={})", filename);
         unlinkat(Some(self.rootfd), filename, UnlinkatFlags::NoRemoveDir).or_else(|_| Err(()))
     }
@@ -109,7 +110,7 @@ impl WALStore for WALStoreTest {
         logfiles.into_iter()
     }
 
-    fn apply_payload(&self, payload: WALBytes) {
+    fn apply_payload(&mut self, payload: WALBytes) {
         println!("apply_payload(payload={})", std::str::from_utf8(&payload).unwrap())
     }
 }
diff --git a/src/wal.rs b/src/wal.rs
index 0ba35fb..b5b3e91 100644
--- a/src/wal.rs
+++ b/src/wal.rs
@@ -62,23 +62,23 @@ pub trait WALFile {
     /// should be _atomic_ (the entire single write should be all or nothing).
     fn write(&self, offset: WALPos, data: WALBytes);
     /// Read data with offset.
-    fn read(&self, offset: WALPos, length: usize) -> WALBytes;
+    fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes>;
 }
 
 pub trait WALStore {
     type FileNameIter: Iterator<Item = String>;
 
     /// Open a file given the filename, create the file if not exists when `touch` is `true`.
-    fn open_file(&self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>>;
+    fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>>;
     /// Unlink a file given the filename.
-    fn remove_file(&self, filename: &str) -> Result<(), ()>;
+    fn remove_file(&mut self, filename: &str) -> Result<(), ()>;
     /// Enumerate all WAL filenames. It should include all WAL files that are previously opened
     /// (created) but not removed. The list could be unordered.
     fn enumerate_files(&self) -> Self::FileNameIter;
     /// Apply the payload during recovery. This notifies the application should redo the given
     /// operation to ensure its state is consistent (the major goal of having a WAL). We assume
     /// the application applies the payload by the _order_ of this callback invocation.
-    fn apply_payload(&self, payload: WALBytes);
+    fn apply_payload(&mut self, payload: WALBytes);
 }
 
 /// The middle layer that manages WAL file handles and invokes public trait functions to actually
@@ -148,7 +148,7 @@ impl<F: WALStore> WALFilePool<F> {
         }
     }
 
-    fn remove_file(&self, fid: u64) -> Result<(), ()> {
+    fn remove_file(&mut self, fid: u64) -> Result<(), ()> {
         self.store.remove_file(&Self::get_fname(fid))
     }
 
@@ -314,8 +314,8 @@ impl<F: WALStore> WALLoader<F> {
             let fid = self.file_pool.get_fid(fname);
             let f = self.file_pool.get_file(fid, false);
             let mut off = 0;
-            while block_size - (off & (block_size - 1)) > msize as u64 {
-                let header_raw = f.read(off, msize as usize);
+            while let Some(header_raw) = f.read(off, msize as usize) {
+                if block_size - (off & (block_size - 1)) <= msize as u64 { break }
                 off += msize as u64;
                 let header = unsafe {
                     std::mem::transmute::<*const u8, &WALRingBlob>(header_raw.as_ptr())};
@@ -323,21 +323,21 @@ impl<F: WALStore> WALLoader<F> {
                 match header.rtype {
                     WALRingType::Full => {
                         assert!(chunks.is_none());
-                        let payload = f.read(off, rsize as usize);
+                        let payload = f.read(off, rsize as usize).unwrap();
                         off += rsize as u64;
                         self.file_pool.store.apply_payload(payload);
                     },
                     WALRingType::First => {
                         assert!(chunks.is_none());
-                        chunks = Some(vec![f.read(off, rsize as usize)]);
+                        chunks = Some(vec![f.read(off, rsize as usize).unwrap()]);
                         off += rsize as u64;
                     },
                     WALRingType::Middle => {
-                        chunks.as_mut().unwrap().push(f.read(off, rsize as usize));
+                        chunks.as_mut().unwrap().push(f.read(off, rsize as usize).unwrap());
                         off += rsize as u64;
                     },
                     WALRingType::Last => {
-                        chunks.as_mut().unwrap().push(f.read(off, rsize as usize));
+                        chunks.as_mut().unwrap().push(f.read(off, rsize as usize).unwrap());
                         off += rsize as u64;
 
                         let _chunks = chunks.take().unwrap();
diff --git a/tests/common/mod.rs b/tests/common/mod.rs
new file mode 100644
index 0000000..fd98539
--- /dev/null
+++ b/tests/common/mod.rs
@@ -0,0 +1,117 @@
+#[cfg(test)]
+#[allow(dead_code)]
+
+extern crate growthring;
+use growthring::wal::{WALFile, WALStore, WALPos, WALBytes};
+use std::collections::{HashMap, hash_map::Entry};
+use std::cell::RefCell;
+use std::rc::Rc;
+
+/*
+thread_local! {
+    //pub static RNG: RefCell<rand::rngs::StdRng> = RefCell::new(<rand::rngs::StdRng as rand::SeedableRng>::from_seed([0; 32]));
+    pub static RNG: RefCell<rand::rngs::ThreadRng> = RefCell::new(rand::thread_rng());
+}
+
+pub fn gen_rand_letters(i: usize) -> String {
+    //let mut rng = rand::thread_rng();
+    RNG.with(|rng| {
+        (0..i).map(|_| (rng.borrow_mut().gen::<u8>() % 26 + 'a' as u8) as char).collect()
+    })
+}
+*/
+
+struct FileContentEmul(RefCell<Vec<u8>>);
+
+impl FileContentEmul {
+    pub fn new() -> Self { FileContentEmul(RefCell::new(Vec::new())) }
+}
+
+impl std::ops::Deref for FileContentEmul {
+    type Target = RefCell<Vec<u8>>;
+    fn deref(&self) -> &Self::Target {&self.0}
+}
+
+/// Emulate the a virtual file handle.
+pub struct WALFileEmul {
+    file: Rc<FileContentEmul>
+}
+
+impl WALFile for WALFileEmul {
+    fn allocate(&self, offset: WALPos, length: usize) -> Result<(), ()> {
+        let offset = offset as usize;
+        if offset + length > self.file.borrow().len() {
+            self.file.borrow_mut().resize(offset + length, 0)
+        }
+        for v in &mut self.file.borrow_mut()[offset..offset + length] { *v = 0 }
+        Ok(())
+    }
+
+    fn truncate(&self, length: usize) -> Result<(), ()> {
+        self.file.borrow_mut().resize(length, 0);
+        Ok(())
+    }
+
+    fn write(&self, offset: WALPos, data: WALBytes) {
+        let offset = offset as usize;
+        &self.file.borrow_mut()[offset..offset + data.len()].copy_from_slice(&data);
+    }
+
+    fn read(&self, offset: WALPos, length: usize) -> Option<WALBytes> {
+        let offset = offset as usize;
+        let file = self.file.borrow();
+        if offset + length > file.len() { None }
+        else {
+            Some((&file[offset..offset + length]).to_vec().into_boxed_slice())
+        }
+    }
+}
+
+pub struct WALStoreEmulState{
+    files: HashMap<String, Rc<FileContentEmul>>,
+}
+
+impl WALStoreEmulState {
+    pub fn new() -> Self { WALStoreEmulState { files: HashMap::new() } }
+}
+
+/// Emulate the persistent storage state.
+pub struct WALStoreEmul<'a>(&'a mut WALStoreEmulState);
+
+impl<'a> WALStoreEmul<'a> {
+    pub fn new(state: &'a mut WALStoreEmulState) -> Self {
+        WALStoreEmul(state)
+    }
+}
+
+impl<'a> WALStore for WALStoreEmul<'a> {
+    type FileNameIter = std::vec::IntoIter<String>;
+
+    fn open_file(&mut self, filename: &str, touch: bool) -> Option<Box<dyn WALFile>> {
+        match self.0.files.entry(filename.to_string()) {
+            Entry::Occupied(e) => Some(Box::new(WALFileEmul { file: e.get().clone() })),
+            Entry::Vacant(e) => if touch {
+                Some(Box::new(
+                    WALFileEmul { file: e.insert(Rc::new(FileContentEmul::new())).clone() }))
+            } else {
+                None
+            }
+        }
+    }
+
+    fn remove_file(&mut self, filename: &str) -> Result<(), ()> {
+        self.0.files.remove(filename).ok_or(()).and_then(|_| Ok(()))
+    }
+
+    fn enumerate_files(&self) -> Self::FileNameIter {
+        let mut logfiles = Vec::new();
+        for (fname, _) in self.0.files.iter() {
+            logfiles.push(fname.clone())
+        }
+        logfiles.into_iter()
+    }
+
+    fn apply_payload(&mut self, payload: WALBytes) {
+        println!("apply_payload(payload={})", std::str::from_utf8(&payload).unwrap())
+    }
+}
diff --git a/tests/rand_fail.rs b/tests/rand_fail.rs
new file mode 100644
index 0000000..988da05
--- /dev/null
+++ b/tests/rand_fail.rs
@@ -0,0 +1,25 @@
+#[cfg(test)]
+
+extern crate growthring;
+use growthring::wal::{WALLoader, WALWriter, WALRingId, WALBytes};
+
+mod common;
+
+fn test(records: Vec<String>, wal: &mut WALWriter<common::WALStoreEmul>) -> Box<[WALRingId]> {
+    let records: Vec<WALBytes> = records.into_iter().map(|s| s.into_bytes().into_boxed_slice()).collect();
+    let ret = wal.grow(&records);
+    for ring_id in ret.iter() {
+        println!("got ring id: {:?}", ring_id);
+    }
+    ret
+}
+
+#[test]
+fn test_rand_fail() {
+    let mut state = common::WALStoreEmulState::new();
+    let mut wal = WALLoader::new(common::WALStoreEmul::new(&mut state), 9, 8, 1000).recover();
+    for _ in 0..3 {
+        test(["hi", "hello", "lol"].iter().map(|s| s.to_string()).collect::<Vec<String>>(), &mut wal);
+    }
+    let mut wal = WALLoader::new(common::WALStoreEmul::new(&mut state), 9, 8, 1000).recover();
+}