From c6aa51693d60356b237c3bb682aee7946a45dc12 Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Tue, 26 Oct 2021 00:06:18 -0400 Subject: [PATCH 01/11] Start experimenting with a pool allocator --- src/cursor.rs | 72 ++++++++++++++++++++++++++- src/lib.rs | 4 ++ src/pool.rs | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 src/pool.rs diff --git a/src/cursor.rs b/src/cursor.rs index 39b0c96..22e7c68 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -83,11 +83,13 @@ use std::{ borrow::Cow, - cell::Cell, + cell::{Cell, RefCell}, fmt, hash::{Hash, Hasher}, iter, + marker::PhantomData, mem::{self, ManuallyDrop}, + rc::Rc, ops::Range, ptr, slice, }; @@ -96,6 +98,7 @@ use countme::Count; use crate::{ green::{GreenChild, GreenElementRef, GreenNodeData, GreenTokenData, SyntaxKind}, + pool::{Pool, Chunk}, sll, utility_types::Delta, Direction, GreenNode, GreenToken, NodeOrToken, SyntaxText, TextRange, TextSize, TokenAtOffset, @@ -143,6 +146,73 @@ unsafe impl sll::Elem for NodeData { pub type SyntaxElement = NodeOrToken; +#[derive(Clone)] +enum BoxedOrPooled { + Boxed(ptr::NonNull), + Pooled {chunk: ptr::NonNull>, pool: Rc>>}, +} + +impl BoxedOrPooled { + unsafe fn free(&mut self) { + match self { + BoxedOrPooled::Boxed(ptr) => { + let _ = Box::from_raw(ptr.as_ptr()); + } + BoxedOrPooled::Pooled { chunk, pool } => { + pool.borrow_mut().deallocate(*chunk); + } + } + } + + unsafe fn as_ref(&self) -> &T { + match self { + BoxedOrPooled::Boxed(ptr) => ptr.as_ref(), + BoxedOrPooled::Pooled { chunk, pool: _ } => chunk.as_ref().as_ref(), + } + } + + unsafe fn as_mut(&mut self) -> &mut T { + match self { + BoxedOrPooled::Boxed(ptr) => ptr.as_mut(), + BoxedOrPooled::Pooled { chunk, pool: _ } => chunk.as_mut().as_mut(), + } + } +} + +trait AllocatorStrategy { + fn allocate(&self, val: T) -> BoxedOrPooled; +} + +struct BoxedAllocator { + _p: PhantomData +} + +impl std::default::Default for BoxedAllocator { + fn default() -> Self { + BoxedAllocator { _p: PhantomData } + } +} + +impl AllocatorStrategy for BoxedAllocator { + fn allocate(&self, val: T) -> BoxedOrPooled { + unsafe { BoxedOrPooled::Boxed(ptr::NonNull::new_unchecked(Box::into_raw(Box::new(val)))) } + } +} + +struct TryToPoolAllocator { + pool: Rc>>, +} + +impl AllocatorStrategy for TryToPoolAllocator { + fn allocate(&self, val: T) -> BoxedOrPooled { + if self.pool.borrow().can_allocate() { + BoxedOrPooled::Pooled { chunk: self.pool.borrow_mut().allocate(val).unwrap(), pool: self.pool.clone() } + } else { + BoxedAllocator::default().allocate(val) + } + } +} + pub struct SyntaxNode { ptr: ptr::NonNull, } diff --git a/src/lib.rs b/src/lib.rs index bab6cd9..4368adf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,10 @@ #[allow(unsafe_code)] mod green; + +#[allow(unsafe_code)] +mod pool; + #[allow(unsafe_code)] pub mod cursor; diff --git a/src/pool.rs b/src/pool.rs new file mode 100644 index 0000000..3c6d5e1 --- /dev/null +++ b/src/pool.rs @@ -0,0 +1,132 @@ +use std::{ptr, mem::MaybeUninit, default::Default}; + +pub struct Chunk { + data: MaybeUninit, + next: Option>>, +} + +impl Default for Chunk { + fn default() -> Self { + Self { data: MaybeUninit::uninit(), next: None } + } +} + +impl AsRef for Chunk { + fn as_ref(&self) -> &T { + unsafe { self.data.assume_init_ref() } + } +} + +impl AsMut for Chunk { + fn as_mut(&mut self) -> &mut T { + unsafe { self.data.assume_init_mut() } + } +} + +pub struct Pool { + data: Box<[Chunk]>, + free_chunk: Option>>, + capacity: usize, + num_chunks: usize, +} + +impl Default for Pool { + fn default() -> Self { + Self::new_with_capacity(256) + } +} + +impl Pool { + pub fn new_with_capacity(capacity: usize) -> Self { + debug_assert!(capacity > 0); + let mut vec = Vec::with_capacity(capacity); + for idx in 0..capacity { + vec.push(Chunk::default()); + if idx > 0 { + unsafe { + vec[idx - 1].next = Some(ptr::NonNull::new_unchecked(&mut vec[idx])); + } + } + } + + Self { + free_chunk: unsafe { Some(ptr::NonNull::new_unchecked(&mut vec[0])) }, + data: vec.into_boxed_slice(), + capacity, + num_chunks: 0, + } + } + + pub fn allocate(&mut self, item: T) -> Result>, &'static str> { + if !self.can_allocate() { + return Err("Pool is empty"); + } + + unsafe { + self.free_chunk.unwrap().as_mut().data.write(item); + } + + self.num_chunks += 1; + let result = self.free_chunk.unwrap(); + + unsafe { + self.free_chunk = self.free_chunk.unwrap().as_ref().next; + } + + Ok(result) + } + + pub fn deallocate(&mut self, mut item: ptr::NonNull>) { + unsafe { + ptr::drop_in_place(item.as_mut().data.as_mut_ptr()); + item.as_mut().next = self.free_chunk; + } + self.free_chunk = Some(item); + self.num_chunks -= 1; + } + + pub fn is_empty(&self) -> bool { + self.num_chunks == 0 + } + + pub fn can_allocate(&self) -> bool { + self.num_chunks < self.capacity + } +} + +mod tests { + use super::*; + + struct TestStruct { + id: usize, + } + + impl Drop for TestStruct { + fn drop(&mut self) { + println!("{} dropped", self.id); + } + } + + #[test] + fn test_pool() { + let mut pool = Pool::default(); + let mut allocated = Vec::new(); + for id in 0..10 { + allocated.push(pool.allocate(TestStruct { id }).unwrap()); + } + + for id in 0..10 { + unsafe { + assert_eq!(allocated[id].as_ref().as_ref().id, id); + } + } + + assert!(!pool.is_empty()); + + for it in allocated.iter() { + pool.deallocate(*it); + } + + assert!(pool.is_empty()); + } +} From e90bbc0a41de62518b2dab4bcdee35503f0d051d Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sat, 30 Oct 2021 20:40:35 -0400 Subject: [PATCH 02/11] Create a descendants_pooled interface --- src/api.rs | 4 + src/cursor.rs | 205 +++++++++++++++++++++++++++++++++----------------- src/pool.rs | 58 +++++++++----- 3 files changed, 179 insertions(+), 88 deletions(-) diff --git a/src/api.rs b/src/api.rs index 6e2cd61..0ffef05 100644 --- a/src/api.rs +++ b/src/api.rs @@ -193,6 +193,10 @@ impl SyntaxNode { self.raw.descendants().map(SyntaxNode::from) } + pub fn descendants_pooled(&self, capacity: usize) -> impl Iterator> { + self.raw.descendants_pooled(capacity).map(SyntaxNode::from) + } + pub fn descendants_with_tokens(&self) -> impl Iterator> { self.raw.descendants_with_tokens().map(NodeOrToken::from) } diff --git a/src/cursor.rs b/src/cursor.rs index 22e7c68..d83a17e 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -98,7 +98,7 @@ use countme::Count; use crate::{ green::{GreenChild, GreenElementRef, GreenNodeData, GreenTokenData, SyntaxKind}, - pool::{Pool, Chunk}, + pool::Pool, sll, utility_types::Delta, Direction, GreenNode, GreenToken, NodeOrToken, SyntaxText, TextRange, TextSize, TokenAtOffset, @@ -120,6 +120,8 @@ struct NodeData { index: Cell, green: Green, + allocator_strategy: AllocatorStrategy, + /// Invariant: never changes after NodeData is created. mutable: bool, /// Absolute offset for immutable nodes, unused for mutable nodes. @@ -146,69 +148,53 @@ unsafe impl sll::Elem for NodeData { pub type SyntaxElement = NodeOrToken; -#[derive(Clone)] -enum BoxedOrPooled { - Boxed(ptr::NonNull), - Pooled {chunk: ptr::NonNull>, pool: Rc>>}, +enum AllocatorStrategy { + Boxed, + Pooled(Rc>>), } -impl BoxedOrPooled { - unsafe fn free(&mut self) { - match self { - BoxedOrPooled::Boxed(ptr) => { - let _ = Box::from_raw(ptr.as_ptr()); - } - BoxedOrPooled::Pooled { chunk, pool } => { - pool.borrow_mut().deallocate(*chunk); - } - } +impl Default for AllocatorStrategy { + fn default() -> Self { + AllocatorStrategy::Boxed } +} - unsafe fn as_ref(&self) -> &T { - match self { - BoxedOrPooled::Boxed(ptr) => ptr.as_ref(), - BoxedOrPooled::Pooled { chunk, pool: _ } => chunk.as_ref().as_ref(), - } +impl AllocatorStrategy { + pub fn new_pooled(capacity: usize) -> Self { + AllocatorStrategy::Pooled(Rc::new(RefCell::new(Pool::new_with_capacity(capacity)))) } - unsafe fn as_mut(&mut self) -> &mut T { + fn allocate(&self, val: NodeData) -> Result, &'static str> { match self { - BoxedOrPooled::Boxed(ptr) => ptr.as_mut(), - BoxedOrPooled::Pooled { chunk, pool: _ } => chunk.as_mut().as_mut(), + AllocatorStrategy::Boxed => unsafe { Ok(ptr::NonNull::new_unchecked(Box::into_raw(Box::new(val)))) } + AllocatorStrategy::Pooled(pool) => pool.borrow_mut().allocate(val), } } -} -trait AllocatorStrategy { - fn allocate(&self, val: T) -> BoxedOrPooled; -} - -struct BoxedAllocator { - _p: PhantomData -} - -impl std::default::Default for BoxedAllocator { - fn default() -> Self { - BoxedAllocator { _p: PhantomData } + fn deallocate(&self, val: ptr::NonNull) { + match self { + AllocatorStrategy::Boxed => unsafe { + let _ = Box::from_raw(val.as_ptr()); + } + AllocatorStrategy::Pooled(pool) => { + pool.borrow_mut().deallocate(val); + } + } } -} -impl AllocatorStrategy for BoxedAllocator { - fn allocate(&self, val: T) -> BoxedOrPooled { - unsafe { BoxedOrPooled::Boxed(ptr::NonNull::new_unchecked(Box::into_raw(Box::new(val)))) } + fn can_allocate(&self) -> bool { + match self { + AllocatorStrategy::Boxed => true, + AllocatorStrategy::Pooled(pool) => pool.borrow().can_allocate() + } } } -struct TryToPoolAllocator { - pool: Rc>>, -} - -impl AllocatorStrategy for TryToPoolAllocator { - fn allocate(&self, val: T) -> BoxedOrPooled { - if self.pool.borrow().can_allocate() { - BoxedOrPooled::Pooled { chunk: self.pool.borrow_mut().allocate(val).unwrap(), pool: self.pool.clone() } - } else { - BoxedAllocator::default().allocate(val) +impl Clone for AllocatorStrategy { + fn clone(&self) -> Self { + match self { + AllocatorStrategy::Boxed => AllocatorStrategy::Boxed, + AllocatorStrategy::Pooled(pool) => AllocatorStrategy::Pooled(pool.clone()) } } } @@ -256,12 +242,25 @@ impl Drop for SyntaxToken { } } +struct NodeDataDeallocator { + data: ptr::NonNull +} + +impl Drop for NodeDataDeallocator { + fn drop(&mut self) { + unsafe { + self.data.as_ref().allocator_strategy.deallocate(self.data); + } + } +} + #[inline(never)] unsafe fn free(mut data: ptr::NonNull) { loop { debug_assert_eq!(data.as_ref().rc.get(), 0); debug_assert!(data.as_ref().first.get().is_null()); - let node = Box::from_raw(data.as_ptr()); + let node = data.as_ref(); + let _ = NodeDataDeallocator { data }; match node.parent.take() { Some(parent) => { debug_assert!(parent.as_ref().rc.get() > 0); @@ -297,7 +296,14 @@ impl NodeData { offset: TextSize, green: Green, mutable: bool, + allocator_strategy: AllocatorStrategy, ) -> ptr::NonNull { + let allocator_strategy = if allocator_strategy.can_allocate() { + allocator_strategy + } else { + AllocatorStrategy::Boxed + }; + let parent = ManuallyDrop::new(parent); let res = NodeData { _c: Count::new(), @@ -305,6 +311,7 @@ impl NodeData { parent: Cell::new(parent.as_ref().map(|it| it.ptr)), index: Cell::new(index), green, + allocator_strategy: allocator_strategy.clone(), mutable, offset, @@ -339,13 +346,13 @@ impl NodeData { return ptr::NonNull::new_unchecked(res); } it => { - let res = Box::into_raw(Box::new(res)); - it.add_to_sll(res); - return ptr::NonNull::new_unchecked(res); + let res = allocator_strategy.allocate(res).unwrap(); + it.add_to_sll(res.as_ptr()); + return res; } } } - ptr::NonNull::new_unchecked(Box::into_raw(Box::new(res))) + allocator_strategy.allocate(res).unwrap() } } @@ -446,6 +453,10 @@ impl NodeData { } fn next_sibling(&self) -> Option { + self.next_sibling_in_allocator(AllocatorStrategy::default()) + } + + fn next_sibling_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Option { let mut siblings = self.green_siblings().enumerate(); let index = self.index() as usize; @@ -454,10 +465,11 @@ impl NodeData { child.as_ref().into_node().and_then(|green| { let parent = self.parent_node()?; let offset = parent.offset() + child.rel_offset(); - Some(SyntaxNode::new_child(green, parent, index as u32, offset)) + Some(SyntaxNode::new_child_in_allocator(green, parent, index as u32, offset, allocator_strategy.clone())) }) }) } + fn prev_sibling(&self) -> Option { let mut rev_siblings = self.green_siblings().enumerate().rev(); let index = rev_siblings.len() - (self.index() as usize); @@ -594,13 +606,13 @@ impl SyntaxNode { pub fn new_root(green: GreenNode) -> SyntaxNode { let green = GreenNode::into_raw(green); let green = Green::Node { ptr: Cell::new(green) }; - SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, false) } + SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, false, AllocatorStrategy::default()) } } pub fn new_root_mut(green: GreenNode) -> SyntaxNode { let green = GreenNode::into_raw(green); let green = Green::Node { ptr: Cell::new(green) }; - SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, true) } + SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, true, AllocatorStrategy::default()) } } fn new_child( @@ -608,10 +620,20 @@ impl SyntaxNode { parent: SyntaxNode, index: u32, offset: TextSize, + ) -> SyntaxNode { + Self::new_child_in_allocator(green, parent, index, offset, AllocatorStrategy::default()) + } + + fn new_child_in_allocator( + green: &GreenNodeData, + parent: SyntaxNode, + index: u32, + offset: TextSize, + allocator_strategy: AllocatorStrategy, ) -> SyntaxNode { let mutable = parent.data().mutable; let green = Green::Node { ptr: Cell::new(green.into()) }; - SyntaxNode { ptr: NodeData::new(Some(parent), index, offset, green, mutable) } + SyntaxNode { ptr: NodeData::new(Some(parent), index, offset, green, mutable, allocator_strategy) } } pub fn clone_for_update(&self) -> SyntaxNode { @@ -706,17 +728,23 @@ impl SyntaxNode { } pub fn first_child(&self) -> Option { + self.first_child_in_allocator(AllocatorStrategy::default()) + } + + fn first_child_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Option { self.green_ref().children().raw.enumerate().find_map(|(index, child)| { child.as_ref().into_node().map(|green| { - SyntaxNode::new_child( + SyntaxNode::new_child_in_allocator( green, self.clone(), index as u32, self.offset() + child.rel_offset(), + allocator_strategy.clone() ) }) }) } + pub fn last_child(&self) -> Option { self.green_ref().children().raw.enumerate().rev().find_map(|(index, child)| { child.as_ref().into_node().map(|green| { @@ -749,6 +777,11 @@ impl SyntaxNode { pub fn next_sibling(&self) -> Option { self.data().next_sibling() } + + fn next_sibling_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Option { + self.data().next_sibling_in_allocator(allocator_strategy) + } + pub fn prev_sibling(&self) -> Option { self.data().prev_sibling() } @@ -789,7 +822,17 @@ impl SyntaxNode { #[inline] pub fn descendants(&self) -> impl Iterator { - self.preorder().filter_map(|event| match event { + self.descendants_in_allocator(AllocatorStrategy::default()) + } + + #[inline] + pub fn descendants_pooled(&self, capacity: usize) -> impl Iterator { + self.descendants_in_allocator(AllocatorStrategy::new_pooled(capacity)) + } + + #[inline] + fn descendants_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> impl Iterator { + self.preorder_in_allocator(allocator_strategy).filter_map(|event| match event { WalkEvent::Enter(node) => Some(node), WalkEvent::Leave(_) => None, }) @@ -805,7 +848,12 @@ impl SyntaxNode { #[inline] pub fn preorder(&self) -> Preorder { - Preorder::new(self.clone()) + self.preorder_in_allocator(AllocatorStrategy::default()) + } + + #[inline] + fn preorder_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Preorder { + Preorder::new(self.clone(), allocator_strategy) } #[inline] @@ -912,10 +960,20 @@ impl SyntaxToken { parent: SyntaxNode, index: u32, offset: TextSize, + ) -> SyntaxToken { + Self::new_in_allocator(green, parent, index, offset, AllocatorStrategy::default()) + } + + fn new_in_allocator( + green: &GreenTokenData, + parent: SyntaxNode, + index: u32, + offset: TextSize, + allocator_strategy: AllocatorStrategy ) -> SyntaxToken { let mutable = parent.data().mutable; let green = Green::Token { ptr: green.into() }; - SyntaxToken { ptr: NodeData::new(Some(parent), index, offset, green, mutable) } + SyntaxToken { ptr: NodeData::new(Some(parent), index, offset, green, mutable, allocator_strategy) } } #[inline] @@ -1027,13 +1085,23 @@ impl SyntaxElement { parent: SyntaxNode, index: u32, offset: TextSize, + ) -> SyntaxElement { + Self::new_in_allocator(element, parent, index, offset, AllocatorStrategy::default()) + } + + fn new_in_allocator( + element: GreenElementRef<'_>, + parent: SyntaxNode, + index: u32, + offset: TextSize, + allocator_strategy: AllocatorStrategy, ) -> SyntaxElement { match element { NodeOrToken::Node(node) => { - SyntaxNode::new_child(node, parent, index as u32, offset).into() + SyntaxNode::new_child_in_allocator(node, parent, index as u32, offset, allocator_strategy).into() } NodeOrToken::Token(token) => { - SyntaxToken::new(token, parent, index as u32, offset).into() + SyntaxToken::new_in_allocator(token, parent, index as u32, offset, allocator_strategy).into() } } } @@ -1247,12 +1315,13 @@ pub struct Preorder { start: SyntaxNode, next: Option>, skip_subtree: bool, + allocator_strategy: AllocatorStrategy } impl Preorder { - fn new(start: SyntaxNode) -> Preorder { + fn new(start: SyntaxNode, allocator_strategy: AllocatorStrategy) -> Preorder { let next = Some(WalkEvent::Enter(start.clone())); - Preorder { start, next, skip_subtree: false } + Preorder { start, next, skip_subtree: false, allocator_strategy } } pub fn skip_subtree(&mut self) { @@ -1279,7 +1348,7 @@ impl Iterator for Preorder { let next = self.next.take(); self.next = next.as_ref().and_then(|next| { Some(match next { - WalkEvent::Enter(node) => match node.first_child() { + WalkEvent::Enter(node) => match node.first_child_in_allocator(self.allocator_strategy.clone()) { Some(child) => WalkEvent::Enter(child), None => WalkEvent::Leave(node.clone()), }, @@ -1287,7 +1356,7 @@ impl Iterator for Preorder { if node == &self.start { return None; } - match node.next_sibling() { + match node.next_sibling_in_allocator(self.allocator_strategy.clone()) { Some(sibling) => WalkEvent::Enter(sibling), None => WalkEvent::Leave(node.parent()?), } diff --git a/src/pool.rs b/src/pool.rs index 3c6d5e1..f1b8dd1 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,6 +1,6 @@ use std::{ptr, mem::MaybeUninit, default::Default}; -pub struct Chunk { +struct Chunk { data: MaybeUninit, next: Option>>, } @@ -11,18 +11,6 @@ impl Default for Chunk { } } -impl AsRef for Chunk { - fn as_ref(&self) -> &T { - unsafe { self.data.assume_init_ref() } - } -} - -impl AsMut for Chunk { - fn as_mut(&mut self) -> &mut T { - unsafe { self.data.assume_init_mut() } - } -} - pub struct Pool { data: Box<[Chunk]>, free_chunk: Option>>, @@ -36,6 +24,22 @@ impl Default for Pool { } } +fn ptr_to_chunk(item: ptr::NonNull) -> ptr::NonNull> { + unsafe { + let base = MaybeUninit::>::uninit(); + let base_ptr = base.as_ptr(); + let data_addr = ptr::addr_of!((*base_ptr).data); + let data_offset = (data_addr as usize) - (base_ptr as usize); + ptr::NonNull::new_unchecked(((item.as_ptr() as usize) - data_offset) as *mut Chunk) + } +} + +fn chunk_to_ptr(mut chunk: ptr::NonNull>) -> ptr::NonNull { + unsafe { + ptr::NonNull::new_unchecked(chunk.as_mut().data.as_mut_ptr()) + } +} + impl Pool { pub fn new_with_capacity(capacity: usize) -> Self { debug_assert!(capacity > 0); @@ -57,7 +61,7 @@ impl Pool { } } - pub fn allocate(&mut self, item: T) -> Result>, &'static str> { + pub fn allocate(&mut self, item: T) -> Result, &'static str> { if !self.can_allocate() { return Err("Pool is empty"); } @@ -73,15 +77,17 @@ impl Pool { self.free_chunk = self.free_chunk.unwrap().as_ref().next; } - Ok(result) + Ok(chunk_to_ptr(result)) } - pub fn deallocate(&mut self, mut item: ptr::NonNull>) { + pub fn deallocate(&mut self, mut item: ptr::NonNull) { + let mut chunk_ptr = ptr_to_chunk(item); + unsafe { - ptr::drop_in_place(item.as_mut().data.as_mut_ptr()); - item.as_mut().next = self.free_chunk; + ptr::drop_in_place(item.as_mut()); + chunk_ptr.as_mut().next = self.free_chunk; } - self.free_chunk = Some(item); + self.free_chunk = Some(chunk_ptr); self.num_chunks -= 1; } @@ -117,7 +123,7 @@ mod tests { for id in 0..10 { unsafe { - assert_eq!(allocated[id].as_ref().as_ref().id, id); + assert_eq!(allocated[id].as_ref().id, id); } } @@ -129,4 +135,16 @@ mod tests { assert!(pool.is_empty()); } + + #[test] + fn test_chunk_ptr_conversion() { + let mut chunk: Chunk = Chunk::default(); + chunk.data.write(TestStruct { id: 0 }); + let chunk_ptr = unsafe { ptr::NonNull::new_unchecked(&mut chunk) }; + let test_struct = chunk_to_ptr(chunk_ptr); + unsafe { + assert_eq!(test_struct.as_ref().id, 0); + assert_eq!(ptr_to_chunk(test_struct), chunk_ptr); + } + } } From 764c67e0e447e26e38e867e3eb6dec5d9d449bcb Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sat, 30 Oct 2021 22:58:56 -0400 Subject: [PATCH 03/11] Fix segfault caused by accidentally dropping too early --- src/cursor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cursor.rs b/src/cursor.rs index d83a17e..a78c996 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -259,8 +259,8 @@ unsafe fn free(mut data: ptr::NonNull) { loop { debug_assert_eq!(data.as_ref().rc.get(), 0); debug_assert!(data.as_ref().first.get().is_null()); + let _to_drop = NodeDataDeallocator { data }; let node = data.as_ref(); - let _ = NodeDataDeallocator { data }; match node.parent.take() { Some(parent) => { debug_assert!(parent.as_ref().rc.get() > 0); From bee949e270e2c20dbc8e631617701f21b7fd2014 Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 00:31:48 -0400 Subject: [PATCH 04/11] Try to allocate less, see if that helps --- src/cursor.rs | 40 ++++++++++++++++++++-------------------- src/pool.rs | 11 +++++------ 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/cursor.rs b/src/cursor.rs index a78c996..43dbc4f 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -296,14 +296,14 @@ impl NodeData { offset: TextSize, green: Green, mutable: bool, - allocator_strategy: AllocatorStrategy, + allocator_strategy: &AllocatorStrategy, ) -> ptr::NonNull { let allocator_strategy = if allocator_strategy.can_allocate() { - allocator_strategy + allocator_strategy.clone() } else { AllocatorStrategy::Boxed }; - + let parent = ManuallyDrop::new(parent); let res = NodeData { _c: Count::new(), @@ -453,10 +453,10 @@ impl NodeData { } fn next_sibling(&self) -> Option { - self.next_sibling_in_allocator(AllocatorStrategy::default()) + self.next_sibling_in_allocator(&AllocatorStrategy::default()) } - fn next_sibling_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Option { + fn next_sibling_in_allocator(&self, allocator_strategy: &AllocatorStrategy) -> Option { let mut siblings = self.green_siblings().enumerate(); let index = self.index() as usize; @@ -465,7 +465,7 @@ impl NodeData { child.as_ref().into_node().and_then(|green| { let parent = self.parent_node()?; let offset = parent.offset() + child.rel_offset(); - Some(SyntaxNode::new_child_in_allocator(green, parent, index as u32, offset, allocator_strategy.clone())) + Some(SyntaxNode::new_child_in_allocator(green, parent, index as u32, offset, allocator_strategy)) }) }) } @@ -606,13 +606,13 @@ impl SyntaxNode { pub fn new_root(green: GreenNode) -> SyntaxNode { let green = GreenNode::into_raw(green); let green = Green::Node { ptr: Cell::new(green) }; - SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, false, AllocatorStrategy::default()) } + SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, false, &AllocatorStrategy::default()) } } pub fn new_root_mut(green: GreenNode) -> SyntaxNode { let green = GreenNode::into_raw(green); let green = Green::Node { ptr: Cell::new(green) }; - SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, true, AllocatorStrategy::default()) } + SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, true, &AllocatorStrategy::default()) } } fn new_child( @@ -621,7 +621,7 @@ impl SyntaxNode { index: u32, offset: TextSize, ) -> SyntaxNode { - Self::new_child_in_allocator(green, parent, index, offset, AllocatorStrategy::default()) + Self::new_child_in_allocator(green, parent, index, offset, &AllocatorStrategy::default()) } fn new_child_in_allocator( @@ -629,7 +629,7 @@ impl SyntaxNode { parent: SyntaxNode, index: u32, offset: TextSize, - allocator_strategy: AllocatorStrategy, + allocator_strategy: &AllocatorStrategy, ) -> SyntaxNode { let mutable = parent.data().mutable; let green = Green::Node { ptr: Cell::new(green.into()) }; @@ -728,10 +728,10 @@ impl SyntaxNode { } pub fn first_child(&self) -> Option { - self.first_child_in_allocator(AllocatorStrategy::default()) + self.first_child_in_allocator(&AllocatorStrategy::default()) } - fn first_child_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Option { + fn first_child_in_allocator(&self, allocator_strategy: &AllocatorStrategy) -> Option { self.green_ref().children().raw.enumerate().find_map(|(index, child)| { child.as_ref().into_node().map(|green| { SyntaxNode::new_child_in_allocator( @@ -739,7 +739,7 @@ impl SyntaxNode { self.clone(), index as u32, self.offset() + child.rel_offset(), - allocator_strategy.clone() + allocator_strategy ) }) }) @@ -778,7 +778,7 @@ impl SyntaxNode { self.data().next_sibling() } - fn next_sibling_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Option { + fn next_sibling_in_allocator(&self, allocator_strategy: &AllocatorStrategy) -> Option { self.data().next_sibling_in_allocator(allocator_strategy) } @@ -961,7 +961,7 @@ impl SyntaxToken { index: u32, offset: TextSize, ) -> SyntaxToken { - Self::new_in_allocator(green, parent, index, offset, AllocatorStrategy::default()) + Self::new_in_allocator(green, parent, index, offset, &AllocatorStrategy::default()) } fn new_in_allocator( @@ -969,7 +969,7 @@ impl SyntaxToken { parent: SyntaxNode, index: u32, offset: TextSize, - allocator_strategy: AllocatorStrategy + allocator_strategy: &AllocatorStrategy ) -> SyntaxToken { let mutable = parent.data().mutable; let green = Green::Token { ptr: green.into() }; @@ -1086,7 +1086,7 @@ impl SyntaxElement { index: u32, offset: TextSize, ) -> SyntaxElement { - Self::new_in_allocator(element, parent, index, offset, AllocatorStrategy::default()) + Self::new_in_allocator(element, parent, index, offset, &AllocatorStrategy::default()) } fn new_in_allocator( @@ -1094,7 +1094,7 @@ impl SyntaxElement { parent: SyntaxNode, index: u32, offset: TextSize, - allocator_strategy: AllocatorStrategy, + allocator_strategy: &AllocatorStrategy, ) -> SyntaxElement { match element { NodeOrToken::Node(node) => { @@ -1348,7 +1348,7 @@ impl Iterator for Preorder { let next = self.next.take(); self.next = next.as_ref().and_then(|next| { Some(match next { - WalkEvent::Enter(node) => match node.first_child_in_allocator(self.allocator_strategy.clone()) { + WalkEvent::Enter(node) => match node.first_child_in_allocator(&self.allocator_strategy) { Some(child) => WalkEvent::Enter(child), None => WalkEvent::Leave(node.clone()), }, @@ -1356,7 +1356,7 @@ impl Iterator for Preorder { if node == &self.start { return None; } - match node.next_sibling_in_allocator(self.allocator_strategy.clone()) { + match node.next_sibling_in_allocator(&self.allocator_strategy) { Some(sibling) => WalkEvent::Enter(sibling), None => WalkEvent::Leave(node.parent()?), } diff --git a/src/pool.rs b/src/pool.rs index f1b8dd1..0213eeb 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -24,16 +24,18 @@ impl Default for Pool { } } +#[inline] fn ptr_to_chunk(item: ptr::NonNull) -> ptr::NonNull> { + let base = MaybeUninit::>::uninit(); + let base_ptr = base.as_ptr(); unsafe { - let base = MaybeUninit::>::uninit(); - let base_ptr = base.as_ptr(); let data_addr = ptr::addr_of!((*base_ptr).data); let data_offset = (data_addr as usize) - (base_ptr as usize); ptr::NonNull::new_unchecked(((item.as_ptr() as usize) - data_offset) as *mut Chunk) } } +#[inline] fn chunk_to_ptr(mut chunk: ptr::NonNull>) -> ptr::NonNull { unsafe { ptr::NonNull::new_unchecked(chunk.as_mut().data.as_mut_ptr()) @@ -72,10 +74,7 @@ impl Pool { self.num_chunks += 1; let result = self.free_chunk.unwrap(); - - unsafe { - self.free_chunk = self.free_chunk.unwrap().as_ref().next; - } + self.free_chunk = unsafe { self.free_chunk.unwrap().as_ref().next }; Ok(chunk_to_ptr(result)) } From 4ae6c09465a8f6e6f7a366ba9a046bc11e431ddc Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 00:46:10 -0400 Subject: [PATCH 05/11] Create preorder_pooled --- src/api.rs | 4 ++++ src/cursor.rs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/api.rs b/src/api.rs index 0ffef05..2560798 100644 --- a/src/api.rs +++ b/src/api.rs @@ -207,6 +207,10 @@ impl SyntaxNode { Preorder { raw: self.raw.preorder(), _p: PhantomData } } + pub fn preorder_pooled(&self, capacity: usize) -> Preorder { + Preorder { raw: self.raw.preorder_pooled(capacity), _p: PhantomData } + } + /// Traverse the subtree rooted at the current node (including the current /// node) in preorder, including tokens. pub fn preorder_with_tokens(&self) -> PreorderWithTokens { diff --git a/src/cursor.rs b/src/cursor.rs index 43dbc4f..6460dd2 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -851,6 +851,11 @@ impl SyntaxNode { self.preorder_in_allocator(AllocatorStrategy::default()) } + #[inline] + pub fn preorder_pooled(&self, capacity: usize) -> Preorder { + self.preorder_in_allocator(AllocatorStrategy::new_pooled(capacity)) + } + #[inline] fn preorder_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Preorder { Preorder::new(self.clone(), allocator_strategy) From de5ea28a14c3e86a1997c6af7bf6630a4cbf7991 Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 00:51:05 -0400 Subject: [PATCH 06/11] Inline more --- src/cursor.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cursor.rs b/src/cursor.rs index 6460dd2..3a21342 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -160,10 +160,12 @@ impl Default for AllocatorStrategy { } impl AllocatorStrategy { + #[inline] pub fn new_pooled(capacity: usize) -> Self { AllocatorStrategy::Pooled(Rc::new(RefCell::new(Pool::new_with_capacity(capacity)))) } + #[inline] fn allocate(&self, val: NodeData) -> Result, &'static str> { match self { AllocatorStrategy::Boxed => unsafe { Ok(ptr::NonNull::new_unchecked(Box::into_raw(Box::new(val)))) } @@ -171,6 +173,7 @@ impl AllocatorStrategy { } } + #[inline] fn deallocate(&self, val: ptr::NonNull) { match self { AllocatorStrategy::Boxed => unsafe { @@ -182,6 +185,7 @@ impl AllocatorStrategy { } } + #[inline] fn can_allocate(&self) -> bool { match self { AllocatorStrategy::Boxed => true, From 5c9e475929e52bc9a2b4fb3aec7f46d4c8d52cf3 Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 00:55:01 -0400 Subject: [PATCH 07/11] Inline even more --- src/cursor.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cursor.rs b/src/cursor.rs index 3a21342..ff2db98 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -195,6 +195,7 @@ impl AllocatorStrategy { } impl Clone for AllocatorStrategy { + #[inline] fn clone(&self) -> Self { match self { AllocatorStrategy::Boxed => AllocatorStrategy::Boxed, @@ -251,6 +252,7 @@ struct NodeDataDeallocator { } impl Drop for NodeDataDeallocator { + #[inline] fn drop(&mut self) { unsafe { self.data.as_ref().allocator_strategy.deallocate(self.data); From 5f6029737a6142ba1d5db9ff30e09728dd7415cf Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 00:57:37 -0400 Subject: [PATCH 08/11] Get rid of inline on drop --- src/cursor.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cursor.rs b/src/cursor.rs index ff2db98..569ab96 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -252,7 +252,6 @@ struct NodeDataDeallocator { } impl Drop for NodeDataDeallocator { - #[inline] fn drop(&mut self) { unsafe { self.data.as_ref().allocator_strategy.deallocate(self.data); From c0005e4076f2e7dc84a498c64bf2f867fd297e6c Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 18:18:12 -0400 Subject: [PATCH 09/11] Try to use a thread-local pool instead --- src/api.rs | 8 -- src/arc.rs | 2 +- src/cursor.rs | 174 +++++------------------------ src/lib.rs | 2 +- src/pool.rs | 295 +++++++++++++++++++++++++++++++++++++++----------- 5 files changed, 262 insertions(+), 219 deletions(-) diff --git a/src/api.rs b/src/api.rs index 2560798..6e2cd61 100644 --- a/src/api.rs +++ b/src/api.rs @@ -193,10 +193,6 @@ impl SyntaxNode { self.raw.descendants().map(SyntaxNode::from) } - pub fn descendants_pooled(&self, capacity: usize) -> impl Iterator> { - self.raw.descendants_pooled(capacity).map(SyntaxNode::from) - } - pub fn descendants_with_tokens(&self) -> impl Iterator> { self.raw.descendants_with_tokens().map(NodeOrToken::from) } @@ -207,10 +203,6 @@ impl SyntaxNode { Preorder { raw: self.raw.preorder(), _p: PhantomData } } - pub fn preorder_pooled(&self, capacity: usize) -> Preorder { - Preorder { raw: self.raw.preorder_pooled(capacity), _p: PhantomData } - } - /// Traverse the subtree rooted at the current node (including the current /// node) in preorder, including tokens. pub fn preorder_with_tokens(&self) -> PreorderWithTokens { diff --git a/src/arc.rs b/src/arc.rs index 3488312..6e8fd92 100644 --- a/src/arc.rs +++ b/src/arc.rs @@ -244,7 +244,7 @@ impl Hash for Arc { pub(crate) struct HeaderSlice { pub(crate) header: H, length: usize, - slice: T, + pub(crate) slice: T, } impl HeaderSlice { diff --git a/src/cursor.rs b/src/cursor.rs index 569ab96..c4fe041 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -120,8 +120,6 @@ struct NodeData { index: Cell, green: Green, - allocator_strategy: AllocatorStrategy, - /// Invariant: never changes after NodeData is created. mutable: bool, /// Absolute offset for immutable nodes, unused for mutable nodes. @@ -148,62 +146,6 @@ unsafe impl sll::Elem for NodeData { pub type SyntaxElement = NodeOrToken; -enum AllocatorStrategy { - Boxed, - Pooled(Rc>>), -} - -impl Default for AllocatorStrategy { - fn default() -> Self { - AllocatorStrategy::Boxed - } -} - -impl AllocatorStrategy { - #[inline] - pub fn new_pooled(capacity: usize) -> Self { - AllocatorStrategy::Pooled(Rc::new(RefCell::new(Pool::new_with_capacity(capacity)))) - } - - #[inline] - fn allocate(&self, val: NodeData) -> Result, &'static str> { - match self { - AllocatorStrategy::Boxed => unsafe { Ok(ptr::NonNull::new_unchecked(Box::into_raw(Box::new(val)))) } - AllocatorStrategy::Pooled(pool) => pool.borrow_mut().allocate(val), - } - } - - #[inline] - fn deallocate(&self, val: ptr::NonNull) { - match self { - AllocatorStrategy::Boxed => unsafe { - let _ = Box::from_raw(val.as_ptr()); - } - AllocatorStrategy::Pooled(pool) => { - pool.borrow_mut().deallocate(val); - } - } - } - - #[inline] - fn can_allocate(&self) -> bool { - match self { - AllocatorStrategy::Boxed => true, - AllocatorStrategy::Pooled(pool) => pool.borrow().can_allocate() - } - } -} - -impl Clone for AllocatorStrategy { - #[inline] - fn clone(&self) -> Self { - match self { - AllocatorStrategy::Boxed => AllocatorStrategy::Boxed, - AllocatorStrategy::Pooled(pool) => AllocatorStrategy::Pooled(pool.clone()) - } - } -} - pub struct SyntaxNode { ptr: ptr::NonNull, } @@ -254,7 +196,9 @@ struct NodeDataDeallocator { impl Drop for NodeDataDeallocator { fn drop(&mut self) { unsafe { - self.data.as_ref().allocator_strategy.deallocate(self.data); + NodeData::POOL.with(|pool| { + pool.borrow_mut().deallocate(self.data); + }); } } } @@ -294,6 +238,10 @@ unsafe fn free(mut data: ptr::NonNull) { } impl NodeData { + thread_local! { + static POOL: RefCell> = RefCell::new(Pool::default()); + } + #[inline] fn new( parent: Option, @@ -301,14 +249,7 @@ impl NodeData { offset: TextSize, green: Green, mutable: bool, - allocator_strategy: &AllocatorStrategy, ) -> ptr::NonNull { - let allocator_strategy = if allocator_strategy.can_allocate() { - allocator_strategy.clone() - } else { - AllocatorStrategy::Boxed - }; - let parent = ManuallyDrop::new(parent); let res = NodeData { _c: Count::new(), @@ -316,7 +257,6 @@ impl NodeData { parent: Cell::new(parent.as_ref().map(|it| it.ptr)), index: Cell::new(index), green, - allocator_strategy: allocator_strategy.clone(), mutable, offset, @@ -351,13 +291,17 @@ impl NodeData { return ptr::NonNull::new_unchecked(res); } it => { - let res = allocator_strategy.allocate(res).unwrap(); + let res = Self::POOL.with(move |pool| { + pool.borrow_mut().allocate(res) + }); it.add_to_sll(res.as_ptr()); return res; } } } - allocator_strategy.allocate(res).unwrap() + Self::POOL.with(move |pool| { + pool.borrow_mut().allocate(res) + }) } } @@ -458,10 +402,6 @@ impl NodeData { } fn next_sibling(&self) -> Option { - self.next_sibling_in_allocator(&AllocatorStrategy::default()) - } - - fn next_sibling_in_allocator(&self, allocator_strategy: &AllocatorStrategy) -> Option { let mut siblings = self.green_siblings().enumerate(); let index = self.index() as usize; @@ -470,7 +410,7 @@ impl NodeData { child.as_ref().into_node().and_then(|green| { let parent = self.parent_node()?; let offset = parent.offset() + child.rel_offset(); - Some(SyntaxNode::new_child_in_allocator(green, parent, index as u32, offset, allocator_strategy)) + Some(SyntaxNode::new_child(green, parent, index as u32, offset)) }) }) } @@ -611,13 +551,13 @@ impl SyntaxNode { pub fn new_root(green: GreenNode) -> SyntaxNode { let green = GreenNode::into_raw(green); let green = Green::Node { ptr: Cell::new(green) }; - SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, false, &AllocatorStrategy::default()) } + SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, false) } } pub fn new_root_mut(green: GreenNode) -> SyntaxNode { let green = GreenNode::into_raw(green); let green = Green::Node { ptr: Cell::new(green) }; - SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, true, &AllocatorStrategy::default()) } + SyntaxNode { ptr: NodeData::new(None, 0, 0.into(), green, true) } } fn new_child( @@ -625,20 +565,10 @@ impl SyntaxNode { parent: SyntaxNode, index: u32, offset: TextSize, - ) -> SyntaxNode { - Self::new_child_in_allocator(green, parent, index, offset, &AllocatorStrategy::default()) - } - - fn new_child_in_allocator( - green: &GreenNodeData, - parent: SyntaxNode, - index: u32, - offset: TextSize, - allocator_strategy: &AllocatorStrategy, ) -> SyntaxNode { let mutable = parent.data().mutable; let green = Green::Node { ptr: Cell::new(green.into()) }; - SyntaxNode { ptr: NodeData::new(Some(parent), index, offset, green, mutable, allocator_strategy) } + SyntaxNode { ptr: NodeData::new(Some(parent), index, offset, green, mutable) } } pub fn clone_for_update(&self) -> SyntaxNode { @@ -733,18 +663,13 @@ impl SyntaxNode { } pub fn first_child(&self) -> Option { - self.first_child_in_allocator(&AllocatorStrategy::default()) - } - - fn first_child_in_allocator(&self, allocator_strategy: &AllocatorStrategy) -> Option { self.green_ref().children().raw.enumerate().find_map(|(index, child)| { child.as_ref().into_node().map(|green| { - SyntaxNode::new_child_in_allocator( + SyntaxNode::new_child( green, self.clone(), index as u32, self.offset() + child.rel_offset(), - allocator_strategy ) }) }) @@ -783,10 +708,6 @@ impl SyntaxNode { self.data().next_sibling() } - fn next_sibling_in_allocator(&self, allocator_strategy: &AllocatorStrategy) -> Option { - self.data().next_sibling_in_allocator(allocator_strategy) - } - pub fn prev_sibling(&self) -> Option { self.data().prev_sibling() } @@ -827,17 +748,7 @@ impl SyntaxNode { #[inline] pub fn descendants(&self) -> impl Iterator { - self.descendants_in_allocator(AllocatorStrategy::default()) - } - - #[inline] - pub fn descendants_pooled(&self, capacity: usize) -> impl Iterator { - self.descendants_in_allocator(AllocatorStrategy::new_pooled(capacity)) - } - - #[inline] - fn descendants_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> impl Iterator { - self.preorder_in_allocator(allocator_strategy).filter_map(|event| match event { + self.preorder().filter_map(|event| match event { WalkEvent::Enter(node) => Some(node), WalkEvent::Leave(_) => None, }) @@ -853,17 +764,7 @@ impl SyntaxNode { #[inline] pub fn preorder(&self) -> Preorder { - self.preorder_in_allocator(AllocatorStrategy::default()) - } - - #[inline] - pub fn preorder_pooled(&self, capacity: usize) -> Preorder { - self.preorder_in_allocator(AllocatorStrategy::new_pooled(capacity)) - } - - #[inline] - fn preorder_in_allocator(&self, allocator_strategy: AllocatorStrategy) -> Preorder { - Preorder::new(self.clone(), allocator_strategy) + Preorder::new(self.clone()) } #[inline] @@ -970,20 +871,10 @@ impl SyntaxToken { parent: SyntaxNode, index: u32, offset: TextSize, - ) -> SyntaxToken { - Self::new_in_allocator(green, parent, index, offset, &AllocatorStrategy::default()) - } - - fn new_in_allocator( - green: &GreenTokenData, - parent: SyntaxNode, - index: u32, - offset: TextSize, - allocator_strategy: &AllocatorStrategy ) -> SyntaxToken { let mutable = parent.data().mutable; let green = Green::Token { ptr: green.into() }; - SyntaxToken { ptr: NodeData::new(Some(parent), index, offset, green, mutable, allocator_strategy) } + SyntaxToken { ptr: NodeData::new(Some(parent), index, offset, green, mutable) } } #[inline] @@ -1095,23 +986,13 @@ impl SyntaxElement { parent: SyntaxNode, index: u32, offset: TextSize, - ) -> SyntaxElement { - Self::new_in_allocator(element, parent, index, offset, &AllocatorStrategy::default()) - } - - fn new_in_allocator( - element: GreenElementRef<'_>, - parent: SyntaxNode, - index: u32, - offset: TextSize, - allocator_strategy: &AllocatorStrategy, ) -> SyntaxElement { match element { NodeOrToken::Node(node) => { - SyntaxNode::new_child_in_allocator(node, parent, index as u32, offset, allocator_strategy).into() + SyntaxNode::new_child(node, parent, index as u32, offset).into() } NodeOrToken::Token(token) => { - SyntaxToken::new_in_allocator(token, parent, index as u32, offset, allocator_strategy).into() + SyntaxToken::new(token, parent, index as u32, offset).into() } } } @@ -1325,13 +1206,12 @@ pub struct Preorder { start: SyntaxNode, next: Option>, skip_subtree: bool, - allocator_strategy: AllocatorStrategy } impl Preorder { - fn new(start: SyntaxNode, allocator_strategy: AllocatorStrategy) -> Preorder { + fn new(start: SyntaxNode) -> Preorder { let next = Some(WalkEvent::Enter(start.clone())); - Preorder { start, next, skip_subtree: false, allocator_strategy } + Preorder { start, next, skip_subtree: false } } pub fn skip_subtree(&mut self) { @@ -1358,7 +1238,7 @@ impl Iterator for Preorder { let next = self.next.take(); self.next = next.as_ref().and_then(|next| { Some(match next { - WalkEvent::Enter(node) => match node.first_child_in_allocator(&self.allocator_strategy) { + WalkEvent::Enter(node) => match node.first_child() { Some(child) => WalkEvent::Enter(child), None => WalkEvent::Leave(node.clone()), }, @@ -1366,7 +1246,7 @@ impl Iterator for Preorder { if node == &self.start { return None; } - match node.next_sibling_in_allocator(&self.allocator_strategy) { + match node.next_sibling() { Some(sibling) => WalkEvent::Enter(sibling), None => WalkEvent::Leave(node.parent()?), } diff --git a/src/lib.rs b/src/lib.rs index 4368adf..e806cc9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ mod green; #[allow(unsafe_code)] -mod pool; +pub mod pool; #[allow(unsafe_code)] pub mod cursor; diff --git a/src/pool.rs b/src/pool.rs index 0213eeb..3f138df 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,101 +1,276 @@ -use std::{ptr, mem::MaybeUninit, default::Default}; +use std::{alloc::{self, Layout}, ptr, mem::{self, MaybeUninit}, default::Default, marker::PhantomData}; +use crate::arc::HeaderSlice; + +use memoffset::offset_of; struct Chunk { data: MaybeUninit, next: Option>>, + page: PagePtr, +} + +#[inline] +fn ptr_to_chunk(item: ptr::NonNull) -> ptr::NonNull> { + let data_offset = offset_of!(Chunk, data); + unsafe { + ptr::NonNull::new_unchecked(((item.as_ptr() as usize) - data_offset) as *mut Chunk) + } +} + +#[inline] +fn chunk_to_ptr(mut chunk: ptr::NonNull>) -> ptr::NonNull { + unsafe { + ptr::NonNull::new_unchecked(chunk.as_mut().data.as_mut_ptr()) + } } impl Default for Chunk { fn default() -> Self { - Self { data: MaybeUninit::uninit(), next: None } + Self { data: MaybeUninit::uninit(), next: None, page: PagePtr { ptr: ptr::NonNull::dangling() } } } } -pub struct Pool { - data: Box<[Chunk]>, +struct PageHeader { + prev_page: Option>, + next_page: Option>, free_chunk: Option>>, - capacity: usize, num_chunks: usize, + _p: PhantomData, } -impl Default for Pool { - fn default() -> Self { - Self::new_with_capacity(256) +type Page = HeaderSlice, [Chunk; 0]>; + +struct PagePtr { + ptr: ptr::NonNull> +} + +impl Clone for PagePtr { + fn clone(&self) -> Self { + Self { ptr: self.ptr.clone() } } } -#[inline] -fn ptr_to_chunk(item: ptr::NonNull) -> ptr::NonNull> { - let base = MaybeUninit::>::uninit(); - let base_ptr = base.as_ptr(); - unsafe { - let data_addr = ptr::addr_of!((*base_ptr).data); - let data_offset = (data_addr as usize) - (base_ptr as usize); - ptr::NonNull::new_unchecked(((item.as_ptr() as usize) - data_offset) as *mut Chunk) +impl Copy for PagePtr { +} + +impl PagePtr { + fn next_page(&self) -> Option { + unsafe { self.ptr.as_ref().header.next_page } + } + + fn prev_page(&self) -> Option { + unsafe { self.ptr.as_ref().header.prev_page } + } + + fn set_next_page(&mut self, next: Option) { + unsafe { self.ptr.as_mut().header.next_page = next; } + } + + fn set_prev_page(&mut self, prev: Option) { + unsafe { self.ptr.as_mut().header.prev_page = prev; } + } + + fn link_next_page(&mut self, next: Option) { + self.set_next_page(next); + if let Some(mut ptr) = next { + ptr.set_prev_page(Some(self.clone())); + } + } + + fn link_prev_page(&mut self, prev: Option) { + self.set_prev_page(prev); + if let Some(mut ptr) = prev { + ptr.set_next_page(Some(self.clone())); + } + } + + fn num_chunks(&self) -> usize { + unsafe { self.ptr.as_ref().header.num_chunks } + } + + fn remove_from_list(&self) { + if let Some(mut prev) = self.prev_page() { + prev.set_next_page(self.next_page()); + } + + if let Some(mut next) = self.next_page() { + next.set_prev_page(self.prev_page()); + } + } + + fn free(&self) { + self.remove_from_list(); + unsafe { + let _to_free = Box::from_raw(self.ptr.as_ptr()); + } + } + + fn new(capacity: usize, prev_page: Option>, next_page: Option>) -> Self { + // Implementation mostly based on arc.rs + assert!(capacity > 0); + + // Find size of the HeaderSlice + let slice_offset = offset_of!(Page, slice); + let slice_size = mem::size_of::>().checked_mul(capacity).expect("size overflow"); + let usable_size = slice_offset.checked_add(slice_size).expect("size overflows"); + + // Round size up to alignment + let align = mem::align_of::>(); + let size = usable_size.wrapping_add(align - 1) & !(align -1); + assert!(size >= usable_size, "size overflows"); + + let layout = Layout::from_size_align(size, align).expect("invalid layout"); + + unsafe { + let buffer = alloc::alloc(layout); + if buffer.is_null() { + alloc::handle_alloc_error(layout); + } + + let ptr = buffer as *mut Page; + let result = Self { ptr: ptr::NonNull::new_unchecked(ptr) }; + + let mut current = ptr::addr_of_mut!((*ptr).slice) as *mut Chunk; + + let header = PageHeader { + prev_page, + next_page, + free_chunk: Some(ptr::NonNull::new_unchecked(current)), + num_chunks: 0, + _p: PhantomData, + }; + + ptr::write(ptr::addr_of_mut!((*ptr).header), header); + for idx in 0..capacity { + let chunk = Chunk { + data: MaybeUninit::uninit(), + next: if idx == capacity - 1 { + None + } else { + Some(ptr::NonNull::new_unchecked(current.offset(1))) + }, + page: result, + }; + ptr::write(current, chunk); + current = current.offset(1); + } + + result + } + } + + fn allocate(&mut self, item: T) -> ptr::NonNull { + let header = unsafe { &mut self.ptr.as_mut().header }; + unsafe { + header.free_chunk.unwrap().as_mut().data.write(item); + } + + header.num_chunks += 1; + let result = header.free_chunk.unwrap(); + header.free_chunk = unsafe { header.free_chunk.unwrap().as_ref().next }; + + chunk_to_ptr(result) } } -#[inline] -fn chunk_to_ptr(mut chunk: ptr::NonNull>) -> ptr::NonNull { - unsafe { - ptr::NonNull::new_unchecked(chunk.as_mut().data.as_mut_ptr()) +pub struct Pool { + free_pages: Option>, + full_pages: Option>, + + num_empty_pages: usize, + + max_empty_pages: usize, + + page_capacity: usize, +} + +impl Default for Pool { + fn default() -> Self { + Self::new(256, 4) } } -impl Pool { - pub fn new_with_capacity(capacity: usize) -> Self { - debug_assert!(capacity > 0); - let mut vec = Vec::with_capacity(capacity); - for idx in 0..capacity { - vec.push(Chunk::default()); - if idx > 0 { - unsafe { - vec[idx - 1].next = Some(ptr::NonNull::new_unchecked(&mut vec[idx])); - } - } +impl Drop for Pool { + fn drop(&mut self) { + while let Some(ptr) = self.free_pages { + self.free_pages = ptr.next_page(); + ptr.free(); + } + + while let Some(ptr) = self.full_pages { + self.full_pages = ptr.next_page(); + ptr.free(); } + } +} + +impl Pool { + pub fn new(page_capacity: usize, max_empty_pages: usize) -> Self { + debug_assert!(page_capacity > 0); + debug_assert!(max_empty_pages > 0); Self { - free_chunk: unsafe { Some(ptr::NonNull::new_unchecked(&mut vec[0])) }, - data: vec.into_boxed_slice(), - capacity, - num_chunks: 0, + free_pages: None, + full_pages: None, + num_empty_pages: 0, + max_empty_pages, + page_capacity, } } - pub fn allocate(&mut self, item: T) -> Result, &'static str> { - if !self.can_allocate() { - return Err("Pool is empty"); + pub fn allocate(&mut self, item: T) -> ptr::NonNull { + if self.free_pages.is_none() { + self.free_pages = Some(PagePtr::new(self.page_capacity, None, None)); + self.num_empty_pages += 1; } - unsafe { - self.free_chunk.unwrap().as_mut().data.write(item); + let mut free_page = self.free_pages.unwrap(); + if free_page.num_chunks() == 0 { + self.num_empty_pages -= 1; } - self.num_chunks += 1; - let result = self.free_chunk.unwrap(); - self.free_chunk = unsafe { self.free_chunk.unwrap().as_ref().next }; + let result = free_page.allocate(item); + if self.page_capacity == free_page.num_chunks() { + self.free_pages = free_page.next_page(); + free_page.remove_from_list(); + free_page.link_next_page(self.full_pages); + free_page.set_prev_page(None); + self.full_pages = Some(free_page); + } - Ok(chunk_to_ptr(result)) + result } pub fn deallocate(&mut self, mut item: ptr::NonNull) { let mut chunk_ptr = ptr_to_chunk(item); + let mut page = unsafe { chunk_ptr.as_ref().page }; + let was_full_page = unsafe { page.ptr.as_ref().header.num_chunks == self.page_capacity }; + let empty_page = unsafe { page.ptr.as_ref().header.num_chunks - 1 == 0 }; unsafe { ptr::drop_in_place(item.as_mut()); - chunk_ptr.as_mut().next = self.free_chunk; - } - self.free_chunk = Some(chunk_ptr); - self.num_chunks -= 1; - } + chunk_ptr.as_mut().next = page.ptr.as_ref().header.free_chunk; - pub fn is_empty(&self) -> bool { - self.num_chunks == 0 - } + page.ptr.as_mut().header.free_chunk = Some(chunk_ptr); + page.ptr.as_mut().header.num_chunks -= 1; - pub fn can_allocate(&self) -> bool { - self.num_chunks < self.capacity + if was_full_page { + if page.prev_page().is_none() { // head of list + self.full_pages = page.next_page(); + } + page.remove_from_list(); + page.link_next_page(self.free_pages); + page.set_prev_page(None); + self.free_pages = Some(page); + } else if empty_page && self.max_empty_pages <= self.num_empty_pages { + if page.prev_page().is_none() { // head of list + self.free_pages = page.next_page(); + } + page.free(); + } else if empty_page { + self.num_empty_pages += 1; + } + } } } @@ -114,25 +289,21 @@ mod tests { #[test] fn test_pool() { - let mut pool = Pool::default(); + let mut pool = Pool::new(2, 10); let mut allocated = Vec::new(); - for id in 0..10 { - allocated.push(pool.allocate(TestStruct { id }).unwrap()); + for id in 0..100 { + allocated.push(pool.allocate(TestStruct { id })); } - for id in 0..10 { + for id in 0..100 { unsafe { assert_eq!(allocated[id].as_ref().id, id); } } - assert!(!pool.is_empty()); - for it in allocated.iter() { pool.deallocate(*it); } - - assert!(pool.is_empty()); } #[test] From 71c8c37ebc02580aaa5b12e0fb43a1403906dbfc Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Sun, 31 Oct 2021 18:47:09 -0400 Subject: [PATCH 10/11] Run cargo fmt --- src/cursor.rs | 17 +++++++---------- src/pool.rs | 43 ++++++++++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/cursor.rs b/src/cursor.rs index c4fe041..2575acc 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -89,9 +89,10 @@ use std::{ iter, marker::PhantomData, mem::{self, ManuallyDrop}, - rc::Rc, ops::Range, - ptr, slice, + ptr, + rc::Rc, + slice, }; use countme::Count; @@ -189,8 +190,8 @@ impl Drop for SyntaxToken { } } -struct NodeDataDeallocator { - data: ptr::NonNull +struct NodeDataDeallocator { + data: ptr::NonNull, } impl Drop for NodeDataDeallocator { @@ -291,17 +292,13 @@ impl NodeData { return ptr::NonNull::new_unchecked(res); } it => { - let res = Self::POOL.with(move |pool| { - pool.borrow_mut().allocate(res) - }); + let res = Self::POOL.with(move |pool| pool.borrow_mut().allocate(res)); it.add_to_sll(res.as_ptr()); return res; } } } - Self::POOL.with(move |pool| { - pool.borrow_mut().allocate(res) - }) + Self::POOL.with(move |pool| pool.borrow_mut().allocate(res)) } } diff --git a/src/pool.rs b/src/pool.rs index 3f138df..0340ee5 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,5 +1,11 @@ -use std::{alloc::{self, Layout}, ptr, mem::{self, MaybeUninit}, default::Default, marker::PhantomData}; use crate::arc::HeaderSlice; +use std::{ + alloc::{self, Layout}, + default::Default, + marker::PhantomData, + mem::{self, MaybeUninit}, + ptr, +}; use memoffset::offset_of; @@ -19,14 +25,16 @@ fn ptr_to_chunk(item: ptr::NonNull) -> ptr::NonNull> { #[inline] fn chunk_to_ptr(mut chunk: ptr::NonNull>) -> ptr::NonNull { - unsafe { - ptr::NonNull::new_unchecked(chunk.as_mut().data.as_mut_ptr()) - } + unsafe { ptr::NonNull::new_unchecked(chunk.as_mut().data.as_mut_ptr()) } } impl Default for Chunk { fn default() -> Self { - Self { data: MaybeUninit::uninit(), next: None, page: PagePtr { ptr: ptr::NonNull::dangling() } } + Self { + data: MaybeUninit::uninit(), + next: None, + page: PagePtr { ptr: ptr::NonNull::dangling() }, + } } } @@ -41,7 +49,7 @@ struct PageHeader { type Page = HeaderSlice, [Chunk; 0]>; struct PagePtr { - ptr: ptr::NonNull> + ptr: ptr::NonNull>, } impl Clone for PagePtr { @@ -50,8 +58,7 @@ impl Clone for PagePtr { } } -impl Copy for PagePtr { -} +impl Copy for PagePtr {} impl PagePtr { fn next_page(&self) -> Option { @@ -63,11 +70,15 @@ impl PagePtr { } fn set_next_page(&mut self, next: Option) { - unsafe { self.ptr.as_mut().header.next_page = next; } + unsafe { + self.ptr.as_mut().header.next_page = next; + } } fn set_prev_page(&mut self, prev: Option) { - unsafe { self.ptr.as_mut().header.prev_page = prev; } + unsafe { + self.ptr.as_mut().header.prev_page = prev; + } } fn link_next_page(&mut self, next: Option) { @@ -105,7 +116,7 @@ impl PagePtr { } } - fn new(capacity: usize, prev_page: Option>, next_page: Option>) -> Self { + fn new(capacity: usize, prev_page: Option>, next_page: Option>) -> Self { // Implementation mostly based on arc.rs assert!(capacity > 0); @@ -116,7 +127,7 @@ impl PagePtr { // Round size up to alignment let align = mem::align_of::>(); - let size = usable_size.wrapping_add(align - 1) & !(align -1); + let size = usable_size.wrapping_add(align - 1) & !(align - 1); assert!(size >= usable_size, "size overflows"); let layout = Layout::from_size_align(size, align).expect("invalid layout"); @@ -143,7 +154,7 @@ impl PagePtr { ptr::write(ptr::addr_of_mut!((*ptr).header), header); for idx in 0..capacity { let chunk = Chunk { - data: MaybeUninit::uninit(), + data: MaybeUninit::uninit(), next: if idx == capacity - 1 { None } else { @@ -255,7 +266,8 @@ impl Pool { page.ptr.as_mut().header.num_chunks -= 1; if was_full_page { - if page.prev_page().is_none() { // head of list + if page.prev_page().is_none() { + // head of list self.full_pages = page.next_page(); } page.remove_from_list(); @@ -263,7 +275,8 @@ impl Pool { page.set_prev_page(None); self.free_pages = Some(page); } else if empty_page && self.max_empty_pages <= self.num_empty_pages { - if page.prev_page().is_none() { // head of list + if page.prev_page().is_none() { + // head of list self.free_pages = page.next_page(); } page.free(); From 01de58750fd1211bca60bb0461545340ee4d0b69 Mon Sep 17 00:00:00 2001 From: Theodore Luo Wang Date: Mon, 8 Nov 2021 23:47:30 -0500 Subject: [PATCH 11/11] Inline more functions --- src/pool.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index 0340ee5..8bcb583 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -61,26 +61,31 @@ impl Clone for PagePtr { impl Copy for PagePtr {} impl PagePtr { + #[inline] fn next_page(&self) -> Option { unsafe { self.ptr.as_ref().header.next_page } } + #[inline] fn prev_page(&self) -> Option { unsafe { self.ptr.as_ref().header.prev_page } } + #[inline] fn set_next_page(&mut self, next: Option) { unsafe { self.ptr.as_mut().header.next_page = next; } } + #[inline] fn set_prev_page(&mut self, prev: Option) { unsafe { self.ptr.as_mut().header.prev_page = prev; } } + #[inline] fn link_next_page(&mut self, next: Option) { self.set_next_page(next); if let Some(mut ptr) = next { @@ -88,6 +93,7 @@ impl PagePtr { } } + #[inline] fn link_prev_page(&mut self, prev: Option) { self.set_prev_page(prev); if let Some(mut ptr) = prev { @@ -170,6 +176,7 @@ impl PagePtr { } } + #[inline] fn allocate(&mut self, item: T) -> ptr::NonNull { let header = unsafe { &mut self.ptr.as_mut().header }; unsafe { @@ -197,7 +204,7 @@ pub struct Pool { impl Default for Pool { fn default() -> Self { - Self::new(256, 4) + Self::new(1024, 4) } } @@ -229,6 +236,7 @@ impl Pool { } } + #[inline] pub fn allocate(&mut self, item: T) -> ptr::NonNull { if self.free_pages.is_none() { self.free_pages = Some(PagePtr::new(self.page_capacity, None, None)); @@ -252,6 +260,7 @@ impl Pool { result } + #[inline] pub fn deallocate(&mut self, mut item: ptr::NonNull) { let mut chunk_ptr = ptr_to_chunk(item); let mut page = unsafe { chunk_ptr.as_ref().page };