scan through the single-point failure space

3 files changed, 71 insertions, 33 deletions

diff --git a/src/wal.rs b/src/wal.rs
index d675426..46baeb8 100644
--- a/src/wal.rs
+++ b/src/wal.rs
@@ -192,7 +192,7 @@ impl<F: WALStore> WALWriter<F> {
     /// Submit a sequence of records to WAL; WALStore/WALFile callbacks are invoked before the
     /// function returns.  The caller then has the knowledge of WAL writes so it should defer
     /// actual data writes after WAL writes.
-    pub fn grow<T: AsRef<[WALBytes]>>(&mut self, records: T) -> Result<Box<[WALRingId]>, ()> {
+    pub fn grow<T: AsRef<[WALBytes]>>(&mut self, records: T) -> (Box<[WALRingId]>, Result<(), ()>) {
         let mut res = Vec::new();
         let mut writes = Vec::new();
         let msize = std::mem::size_of::<WALRingBlob>() as u32;
@@ -268,8 +268,8 @@ impl<F: WALStore> WALWriter<F> {
                         .to_vec().into_boxed_slice()));
             self.state.next += (bbuff_cur - bbuff_start) as u64;
         }
-        self.file_pool.write(writes)?;
-        Ok(res.into_boxed_slice())
+
+        (res.into_boxed_slice(), self.file_pool.write(writes))
     }
 
     /// Inform the WALWriter that data writes (specified by a slice of (offset, length) tuples) are
diff --git a/tests/common/mod.rs b/tests/common/mod.rs
index 43b717a..a5a41d5 100644
--- a/tests/common/mod.rs
+++ b/tests/common/mod.rs
@@ -99,11 +99,11 @@ where
 }
 
 impl<'a, G: FailGen, F: FnMut(WALBytes, WALRingId)> WALStoreEmul<'a, G, F> {
-    pub fn new(state: &'a mut WALStoreEmulState, fail_gen: G,
+    pub fn new(state: &'a mut WALStoreEmulState, fgen: Rc<G>,
                 recover: F) -> Self {
         WALStoreEmul {
             state,
-            fgen: Rc::new(fail_gen),
+            fgen,
             recover
         }
     }
@@ -133,7 +133,7 @@ where
     }
 
     fn remove_file(&mut self, filename: &str) -> Result<(), ()> {
-        println!("remove_file(filename={})", filename);
+        //println!("remove_file(filename={})", filename);
         if self.fgen.next_fail() { return Err(()) }
         self.state.files.remove(filename).ok_or(()).and_then(|_| Ok(()))
     }
@@ -149,9 +149,9 @@ where
 
     fn apply_payload(&mut self, payload: WALBytes, ringid: WALRingId) -> Result<(), ()> {
         if self.fgen.next_fail() { return Err(()) }
-        println!("apply_payload(payload=0x{}, ringid={:?})",
-                hex::encode(&payload),
-                ringid);
+        //println!("apply_payload(payload=0x{}, ringid={:?})",
+        //        hex::encode(&payload),
+        //        ringid);
         (self.recover)(payload, ringid);
         Ok(())
     }
@@ -185,11 +185,26 @@ impl FailGen for ZeroFailGen {
     fn next_fail(&self) -> bool { false }
 }
 
+pub struct CountFailGen(std::cell::Cell<usize>);
+
+impl CountFailGen {
+    pub fn new() -> Self { CountFailGen(std::cell::Cell::new(0)) }
+    pub fn get_count(&self) -> usize { self.0.get() }
+}
+
+impl FailGen for CountFailGen {
+    fn next_fail(&self) -> bool {
+        self.0.set(self.0.get() + 1);
+        false
+    }
+}
+
 /// An ordered list of intervals: `(begin, end, color)*`.
 pub struct PaintStrokes(Vec<(u32, u32, u32)>);
 
 impl PaintStrokes {
     pub fn new() -> Self { PaintStrokes(Vec::new()) }
+    pub fn clone(&self) -> Self { PaintStrokes(self.0.clone()) }
     pub fn to_bytes(&self) -> WALBytes {
         let mut res: Vec<u8> = Vec::new();
         let is = std::mem::size_of::<u32>();
diff --git a/tests/rand_fail.rs b/tests/rand_fail.rs
index c630c07..e01fe58 100644
--- a/tests/rand_fail.rs
+++ b/tests/rand_fail.rs
@@ -1,30 +1,34 @@
 #[cfg(test)]
 
 mod common;
-use growthring::wal::{WALLoader, WALWriter, WALStore, WALRingId, WALBytes, WALPos};
-use common::{FailGen, SingleFailGen, Canvas, WALStoreEmulState, WALStoreEmul, PaintStrokes};
 use std::collections::HashMap;
+use std::rc::Rc;
+use growthring::wal::{WALLoader, WALWriter, WALStore, WALRingId, WALBytes, WALPos};
+use common::{FailGen, SingleFailGen, CountFailGen, Canvas, WALStoreEmulState, WALStoreEmul, PaintStrokes};
 
 fn run<G: 'static + FailGen, R: rand::Rng>(n: usize, m: usize, k: usize,
                     state: &mut WALStoreEmulState, canvas: &mut Canvas, wal: WALLoader,
                     ops: &mut Vec<PaintStrokes>, ringid_map: &mut HashMap<WALRingId, usize>,
-                    fgen: G, rng: &mut R) -> Result<(), ()> {
-    let mut wal = wal.recover(WALStoreEmul::new(state, fgen, |_, _|{})).unwrap();
+                    fgen: Rc<G>, rng: &mut R) -> Result<(), ()> {
+    let mut wal = wal.recover(WALStoreEmul::new(state, fgen, |_, _|{}))?;
     for _ in 0..n {
         let s = (0..m).map(|_|
             PaintStrokes::gen_rand(1000, 10, 256, 5, rng)).collect::<Vec<PaintStrokes>>();
         let recs = s.iter().map(|e| e.to_bytes()).collect::<Vec<WALBytes>>();
         // write ahead
-        let rids = wal.grow(recs)?;
-        for rid in rids.iter() {
-            println!("got ring id: {:?}", rid);
+        let (rids, ok) = wal.grow(recs);
+        for (e, rid) in s.iter().zip(rids.iter()) {
+            ops.push(e.clone());
+            ringid_map.insert(*rid, ops.len() - 1);
         }
+        ok?;
+        //for rid in rids.iter() {
+        //    println!("got ring id: {:?}", rid);
+        //}
         // WAL append done
         // prepare data writes
         for (e, rid) in s.into_iter().zip(rids.iter()) {
             canvas.prepaint(&e, &*rid);
-            ops.push(e);
-            ringid_map.insert(*rid, ops.len() - 1);
         }
         // run the scheduler for a bit
         for _ in 0..k {
@@ -49,17 +53,18 @@ fn run<G: 'static + FailGen, R: rand::Rng>(n: usize, m: usize, k: usize,
 fn check(state: &mut WALStoreEmulState, canvas: &mut Canvas,
          wal: WALLoader,
          ops: &Vec<PaintStrokes>, ringid_map: &HashMap<WALRingId, usize>) -> bool {
-    let mut last_idx = ops.len() - 1;
+    if ops.is_empty() { return true }
+    let mut last_idx = 0;
     canvas.clear_queued();
-    wal.recover(WALStoreEmul::new(state, common::ZeroFailGen, |payload, ringid| {
+    wal.recover(WALStoreEmul::new(state, Rc::new(common::ZeroFailGen), |payload, ringid| {
         let s = PaintStrokes::from_bytes(&payload);
         canvas.prepaint(&s, &ringid);
-        last_idx = *ringid_map.get(&ringid).unwrap();
+        last_idx = *ringid_map.get(&ringid).unwrap() + 1;
     })).unwrap();
     println!("last = {}/{}", last_idx, ops.len() - 1);
     canvas.paint_all();
     // recover complete
-    let canvas0 = canvas.new_reference(&ops[..last_idx + 1]);
+    let canvas0 = canvas.new_reference(&ops[..last_idx]);
     let res = canvas.is_same(&canvas0);
     if !res {
         canvas.print(40);
@@ -68,19 +73,37 @@ fn check(state: &mut WALStoreEmulState, canvas: &mut Canvas,
     res
 }
 
-#[test]
-fn test_rand_fail() {
-    let fgen = SingleFailGen::new(105);
-    let n = 100;
-    let m = 10;
-    let k = 100;
-    let mut rng = <rand::rngs::StdRng as rand::SeedableRng>::from_seed([0; 32]); //rand::thread_rng();
+fn get_nticks(n: usize, m: usize, k: usize, csize: usize) -> usize {
+    let mut rng = <rand::rngs::StdRng as rand::SeedableRng>::from_seed([0; 32]);
     let mut state = WALStoreEmulState::new();
     let wal = WALLoader::new(9, 8, 1000);
     let mut ops: Vec<PaintStrokes> = Vec::new();
     let mut ringid_map = HashMap::new();
-    let mut canvas = Canvas::new(1000);
-    run(n, m, k, &mut state, &mut canvas, wal, &mut ops, &mut ringid_map, fgen, &mut rng);
-    let wal = WALLoader::new(9, 8, 1000);
-    assert!(check(&mut state, &mut canvas, wal, &ops, &ringid_map));
+    let mut canvas = Canvas::new(csize);
+    let fgen = Rc::new(CountFailGen::new());
+    run(n, m, k, &mut state, &mut canvas, wal, &mut ops, &mut ringid_map, fgen.clone(), &mut rng).unwrap();
+    fgen.get_count()
+}
+
+fn run_(n: usize, m: usize, k: usize, csize: usize) {
+    let nticks = get_nticks(n, m, k, csize);
+    println!("nticks = {}", nticks);
+    for i in 0..nticks {
+        let mut rng = <rand::rngs::StdRng as rand::SeedableRng>::from_seed([0; 32]);
+        let mut state = WALStoreEmulState::new();
+        let wal = WALLoader::new(9, 8, 1000);
+        let mut ops: Vec<PaintStrokes> = Vec::new();
+        let mut ringid_map = HashMap::new();
+        let mut canvas = Canvas::new(csize);
+        let fgen = Rc::new(SingleFailGen::new(i));
+        if run(n, m, k, &mut state, &mut canvas, wal, &mut ops, &mut ringid_map, fgen, &mut rng).is_err() {
+            let wal = WALLoader::new(9, 8, 1000);
+            assert!(check(&mut state, &mut canvas, wal, &ops, &ringid_map));
+        }
+    }
+}
+
+#[test]
+fn test_rand_fail() {
+    run_(100, 10, 100, 1000)
 }