From 37ac8fdadb79e041dfdd3038f30f48e828b280f8 Mon Sep 17 00:00:00 2001 From: Determinant Date: Fri, 12 Jun 2020 16:11:18 -0400 Subject: finish all async interface --- examples/demo1.rs | 4 +-- src/wal.rs | 79 +++++++++++++++++++++++++++++++++++------------------ tests/common/mod.rs | 6 ++-- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/examples/demo1.rs b/examples/demo1.rs index a360885..625f40d 100644 --- a/examples/demo1.rs +++ b/examples/demo1.rs @@ -101,9 +101,9 @@ impl WALStore for WALStoreTest { WALFileTest::new(self.rootfd, &filename).and_then(|f| Ok(Box::new(f) as Box)) } - fn remove_file(&self, filename: &str) -> Result<(), ()> { + async fn remove_file(&self, filename: String) -> Result<(), ()> { println!("remove_file(filename={})", filename); - unlinkat(Some(self.rootfd), filename, UnlinkatFlags::NoRemoveDir).or_else(|_| Err(())) + unlinkat(Some(self.rootfd), &filename, UnlinkatFlags::NoRemoveDir).or_else(|_| Err(())) } fn enumerate_files(&self) -> Result { diff --git a/src/wal.rs b/src/wal.rs index 454fbd0..8d92d01 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -69,6 +69,8 @@ struct WALState { next: WALPos, /// number of bits for a file file_nbit: u64, + next_complete: WALPos, + io_complete: BinaryHeap, } #[async_trait(?Send)] @@ -101,7 +103,7 @@ pub trait WALStore { touch: bool, ) -> Result, ()>; /// Unlink a file given the filename. - fn remove_file(&self, filename: &str) -> Result<(), ()>; + async fn remove_file(&self, filename: String) -> Result<(), ()>; /// Enumerate all WAL filenames. It should include all WAL files that are previously opened /// (created) but not removed. The list could be unordered. fn enumerate_files(&self) -> Result; @@ -122,6 +124,8 @@ struct WALFilePool { handles: RefCell>>, last_write: UnsafeCell>>>>>, + last_peel: + UnsafeCell>>>>>, file_nbit: u64, file_size: u64, block_nbit: u64, @@ -137,6 +141,9 @@ impl WALFilePool { last_write: UnsafeCell::new(MaybeUninit::new(Box::pin( future::ready(Ok(())), ))), + last_peel: UnsafeCell::new(MaybeUninit::new(Box::pin( + future::ready(Ok(())), + ))), file_nbit, file_size: 1 << (file_nbit as u64), block_nbit, @@ -172,7 +179,6 @@ impl WALFilePool { scan_fmt!(fname, "{x}.log", [hex WALFileId]).unwrap() } - // TODO: evict stale handles fn write<'a>( &'a mut self, writes: Vec<(WALPos, WALBytes)>, @@ -253,8 +259,33 @@ impl WALFilePool { res } - fn remove_file(&mut self, fid: u64) -> Result<(), ()> { - self.store.remove_file(&Self::get_fname(fid)) + fn remove_files<'a>(&'a mut self, fid_s: u64, fid_e: u64) -> impl Future> + 'a { + let last_peel = unsafe { + std::mem::replace( + &mut *self.last_peel.get(), + std::mem::MaybeUninit::uninit(), + ) + .assume_init() + }; + + let mut removes = Vec::new(); + for fid in fid_s..fid_e { + removes.push(self.store.remove_file(Self::get_fname(fid)) + as Pin + 'a>>) + } + let p = async move { + last_peel.await?; + for r in removes.into_iter() { + r.await? + } + Ok(()) + }.shared(); + unsafe { + (*self.last_peel.get()) = MaybeUninit::new( + std::mem::transmute( + Box::pin(p.clone()) as Pin + 'a>>)) + } + p } fn reset(&mut self) { @@ -267,8 +298,6 @@ pub struct WALWriter { file_pool: WALFilePool, block_buffer: WALBytes, block_size: u32, - next_complete: WALPos, - io_complete: BinaryHeap, msize: usize, } @@ -283,8 +312,6 @@ impl WALWriter { file_pool, block_buffer: b.into_boxed_slice(), block_size, - next_complete: 0, - io_complete: BinaryHeap::new(), msize, } } @@ -430,34 +457,32 @@ impl WALWriter { /// Inform the WALWriter that data writes (specified by a slice of (offset, length) tuples) are /// complete so that it could automatically remove obsolete WAL files. - pub fn peel>( - &mut self, + pub fn peel<'a, T: AsRef<[WALRingId]>>( + &'a mut self, records: T, - ) -> Result<(), ()> { + ) -> impl Future> + 'a { let msize = self.msize as u64; let block_size = self.block_size as u64; + let state = &mut self.state; for rec in records.as_ref() { - self.io_complete.push(*rec); + state.io_complete.push(*rec); } - let orig_fid = self.state.first_fid; - while let Some(s) = self.io_complete.peek().and_then(|&e| Some(e.start)) + let orig_fid = state.first_fid; + while let Some(s) = state.io_complete.peek().and_then(|&e| Some(e.start)) { - if s != self.next_complete { + if s != state.next_complete { break; } - let mut m = self.io_complete.pop().unwrap(); + let mut m = state.io_complete.pop().unwrap(); let block_remain = block_size - (m.end & (block_size - 1)); if block_remain <= msize as u64 { m.end += block_remain } - self.next_complete = m.end - } - let next_fid = self.next_complete >> self.state.file_nbit; - for fid in orig_fid..next_fid { - self.file_pool.remove_file(fid)?; + state.next_complete = m.end } - self.state.first_fid = next_fid; - Ok(()) + let next_fid = state.next_complete >> state.file_nbit; + state.first_fid = next_fid; + self.file_pool.remove_files(orig_fid, next_fid) } } @@ -502,8 +527,8 @@ impl WALLoader { // TODO: check for missing logfiles logfiles.sort(); let mut chunks = None; - for fname in logfiles.iter() { - let fid = file_pool.get_fid(fname); + for fname in logfiles.into_iter() { + let fid = file_pool.get_fid(&fname); let f = futures::executor::block_on(file_pool.get_file(fid, false))?; let mut off = 0; @@ -581,7 +606,7 @@ impl WALLoader { } } f.truncate(0)?; - file_pool.remove_file(fid)?; + futures::executor::block_on(file_pool.store.remove_file(fname))?; } file_pool.reset(); Ok(WALWriter::new( @@ -589,6 +614,8 @@ impl WALLoader { first_fid: 0, next: 0, file_nbit: file_pool.file_nbit, + next_complete: 0, + io_complete: BinaryHeap::new(), }, file_pool, )) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 5fc422f..47b04f1 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -171,7 +171,7 @@ where } } - fn remove_file(&self, filename: &str) -> Result<(), ()> { + async fn remove_file(&self, filename: String) -> Result<(), ()> { //println!("remove_file(filename={})", filename); if self.fgen.next_fail() { return Err(()); @@ -179,7 +179,7 @@ where self.state .borrow_mut() .files - .remove(filename) + .remove(&filename) .ok_or(()) .and_then(|_| Ok(())) } @@ -581,7 +581,7 @@ impl PaintingSim { } if let Some((fin_rid, _)) = canvas.rand_paint(&mut rng) { if let Some(rid) = fin_rid { - wal.peel(&[rid])? + futures::executor::block_on(wal.peel(&[rid]))? } } else { break; -- cgit v1.2.3