RecyclableList.EnsureCapacity & .Resize performance improvements. Custom ArrayPool<T> added.

Recyclable.Collections/OneSizeArrayPool.cs

@@ -0,0 +1,179 @@
+using System.Buffers;
+using System.Collections.Concurrent;
+
+namespace Recyclable.Collections
+{
+	public class MemoryBucket<T>
+	{
+		public MemoryBucket(T[] memory, MemoryBucket<T> nextBucket)
+		{
+			_memory = memory;
+			_nextBucket = nextBucket;
+		}
+
+		public MemoryBucket(T[] memory)
+		{
+			_memory = memory;
+		}
+
+		private T[] _memory;
+		public T[] Memory { get => _memory; set => _memory = value; }
+
+		private MemoryBucket<T>? _nextBucket;
+		public MemoryBucket<T>? NextBucket { get => _nextBucket; set => _nextBucket = value; }
+	}
+
+	public static class RecyclableArrayPool<T>
+	{
+		public static readonly OneSizeArrayPool<T> Null = new(0, 0);
+
+		private static readonly ConcurrentDictionary<int, ArrayPool<T>> _pools = new();
+
+		public static ArrayPool<T> Shared(int blockSize)
+		{
+			//lock (_lock)
+			{
+				if (!_pools.TryGetValue(blockSize, out ArrayPool<T>? pool))
+				{
+					pool = new OneSizeArrayPool<T>(blockSize, 100);
+					pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize];
+				}
+
+				return pool;
+			}
+
+			//_lock.Wait();
+			//try
+			//{
+			//}
+			//finally
+			//{
+			//	_lock.Release();
+			//}
+		}
+	}
+
+	public class OneSizeArrayPool<T> : ArrayPool<T>
+	{
+		private readonly int _blockSize;
+		private readonly int _minBlockCount;
+		private int _blockCount;
+		private MemoryBucket<T>? _currentBucket;
+		//SemaphoreSlim _lock = new(1);
+		//ManualResetEventSlim _lock = new(true, 1);
+		private readonly object _updateLock = new();
+		//private Task? _bufferFillUpTask;
+
+		//private Task CreateAdditionalBucket(int minimumLength, int blockCount)
+		//{
+		//	return Task.Factory.StartNew(() =>
+		//	{
+		//		MemoryBucket<T> firstBucket = new(new T[minimumLength]);
+		//		MemoryBucket<T> currentBucket = firstBucket;
+		//		for (var i = 0; i < blockCount; i++)
+		//		{
+		//			currentBucket = new(new T[minimumLength], currentBucket);
+		//		}
+
+		//		lock (_updateLock)
+		//		//_lock.Wait();
+		//		//try
+		//		{
+		//			currentBucket.NextBucket = _currentBucket;
+		//			_currentBucket = firstBucket;
+		//			_blockCount += blockCount;
+		//			//_bufferFillUpTask = null;
+		//		}
+		//		//finally
+		//		//{
+		//		//	_ = _lock.Release();
+		//		//	//_lock.Set();
+		//		//}
+		//	});
+		//}
+
+		public OneSizeArrayPool(int blockSize, int minBlockCount)
+		{
+			_blockSize = blockSize;
+			_minBlockCount = minBlockCount;
+		}
+
+		public override T[] Rent(int minimumLength)
+		{
+			if (minimumLength != _blockSize)
+			{
+				ThrowWrongRentedArraySizeException(minimumLength, _blockSize);
+			}
+
+			MemoryBucket<T>? bucket;
+			lock (_updateLock)
+			//_lock.Wait();
+			//try
+			{
+				if (_currentBucket != null)
+				{
+					_blockCount--;
+					bucket = _currentBucket;
+					_currentBucket = _currentBucket.NextBucket;
+				}
+				else
+				{
+					bucket = null;
+					//if (_bufferFillUpTask == null)
+					//{
+					//	_bufferFillUpTask = CreateAdditionalBucket(minimumLength, _minBlockCount);
+					//}
+				}
+			}
+			//finally
+			//{
+			//	_ = _lock.Release();
+			//	//_lock.Set();
+			//}
+
+			return bucket != null ? bucket.Memory : new T[_blockSize];
+		}
+
+		public override void Return(T[] array, bool clearArray = false)
+		{
+			if (array.Length != _blockSize)
+			{
+				ThrowWrongReturnedArraySizeException(array.Length, _blockSize);
+			}
+
+			MemoryBucket<T> bucket = new(array);
+			if (clearArray)
+			{
+				Array.Clear(array, 0, array.Length);
+			}
+
+			lock (_updateLock)
+			//lock (_lock)
+			//_lock.Wait();
+			//try
+			{
+				if (_blockCount < _minBlockCount)
+				{
+					bucket.NextBucket = _currentBucket;
+					_currentBucket = bucket;
+					_blockCount++;
+				}
+			}
+			//finally
+			//{
+			//	_ = _lock.Release();
+			//	//_lock.Set();
+			//}
+		}
+
+		private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize)
+		{
+			throw new InvalidOperationException($"The array size {returnedLength} doesn't match the size {blockSize} of the pool buffer and cannot be returned to this pool.");
+		}
+
+		private static void ThrowWrongRentedArraySizeException(int requestedSize, int blockSize)
+		{
+			throw new InvalidOperationException($"The requested size {requestedSize} doesn't match the size {blockSize} of the pool buffer and cannot be taken from this pool.");
+		}
+	}
+}

Recyclable.Collections/RecyclableList.cs

@@ -15,7 +15,7 @@ namespace Recyclable.Collections
 		private static readonly T[] _emptyBlockArray = new T[0];
 
 		private int _blockSize;
-		private int _blockSizePow2Shift;
+		private byte _blockSizePow2Shift;
 		private int _nextItemBlockIndex;
 		private int _nextItemIndex;
 
@@ -65,7 +65,7 @@ namespace Recyclable.Collections
 			while (sourceBlockIndex < lastTakenBlockIndex || (sourceBlockIndex == lastTakenBlockIndex && (sourceItemIndex < nextItemIndex || sourceBlockIndex != list._nextItemBlockIndex)))
 			{
 				int toCopy = sourceBlockIndex < lastTakenBlockIndex || nextItemIndex == 0
-					? Math.Min(blockSize - sourceItemIndex, blockSize - targetItemIndex)
+					? blockSize - (sourceItemIndex >= targetItemIndex ? sourceItemIndex : targetItemIndex)
 					: Math.Min(nextItemIndex, blockSize - targetItemIndex);
 
 				Array.Copy(memoryBlocks[sourceBlockIndex], sourceItemIndex, memoryBlocks[targetBlockIndex], targetItemIndex, toCopy);
@@ -100,7 +100,6 @@ namespace Recyclable.Collections
 		protected void AddRangeEnumerated(IEnumerable<T> source, int growByCount)
 		{
 			long i, copied = _longCount;
-
 			int blockSize = _blockSize, targetItemIdx = _nextItemIndex, targetBlockIdx = _nextItemBlockIndex;
 			
 			using IEnumerator<T> enumerator = source.GetEnumerator();
@@ -240,13 +239,11 @@ namespace Recyclable.Collections
 		}
 
 		[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
-		protected static long Resize(RecyclableList<T> list, long newCapacity)
+		protected static long Resize(RecyclableList<T> list, int minBlockSize, byte minBlockSizeBitShift, long newCapacity)
 		{
 			ArrayPool<T> blockArrayPool = list._blockArrayPool;
-			int minBlockSize = list._blockSize;
+			int sourceBlockCount = checked((int)(list._capacity >> minBlockSizeBitShift));
-			int minBlockSizePowerToShift = 31 - BitOperations.LeadingZeroCount((uint)minBlockSize);
+			int requiredBlockCount = checked((int)(newCapacity >> minBlockSizeBitShift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0));
-			int sourceBlockCount = checked((int)(list._capacity >> minBlockSizePowerToShift));
-			int requiredBlockCount = checked((int)(newCapacity >> minBlockSizePowerToShift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0));
 			int blockIndex;
 			Span<T[]> memoryBlocksSpan;
 
@@ -313,7 +310,7 @@ namespace Recyclable.Collections
 				}
 			}
 
-			return (long)newMemoryBlocks.Length << minBlockSizePowerToShift;
+			return (long)newMemoryBlocks.Length << minBlockSizeBitShift;
 		}
 
 		[MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
@@ -323,37 +320,60 @@ namespace Recyclable.Collections
 				? checked((long)BitOperations.RoundUpToPowerOf2((ulong)requestedCapacity))
 				: requestedCapacity;
 
-			newCapacity = Resize(list, newCapacity);
+			int blockSize = list._blockSize;
-			if (newCapacity > 0 && list._capacity == 0)
+			newCapacity = RecyclableList<T>.Resize(list, blockSize, list._blockSizePow2Shift, newCapacity);
+			if (newCapacity > 0 && blockSize != list._memoryBlocks[0].Length)
 			{
-				list._blockSize = list._memoryBlocks[0].Length;
+				blockSize = list._memoryBlocks[0].Length;
-				list._blockSizePow2Shift = MathUtils.GetPow2Shift(list._blockSize);
+				if (blockSize >= RecyclableDefaults.MinPooledArrayLength)
+				{
+					list._blockArrayPool = RecyclableArrayPool<T>.Shared(blockSize);
+				}
+
+				list._blockSizePow2Shift = (byte)MathUtils.GetPow2Shift(blockSize);
+				list._blockSize = blockSize;
 			}
 
 			list._capacity = newCapacity;
 			return newCapacity;
 		}
 
+		[MethodImpl(MethodImplOptions.AggressiveOptimization)]
 		public RecyclableList(int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool<T[]>? memoryBlocksPool = default, ArrayPool<T>? blockArrayPool = default)
 		{
 			_memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool;
-			_blockArrayPool = blockArrayPool ?? _defaultBlockArrayPool;
 
 			if (expectedItemsCount > 0)
 			{
-				_blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize));
+				minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize));
-				_capacity = Resize(this, expectedItemsCount.Value);
+				_blockArrayPool = minBlockSize >= RecyclableDefaults.MinPooledArrayLength
-				if (_memoryBlocks!.Length > 0)
+					? blockArrayPool ?? RecyclableArrayPool<T>.Shared(minBlockSize)
+					: RecyclableArrayPool<T>.Null;
+
+				_blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)minBlockSize));
+				_capacity = Resize(this, minBlockSize, _blockSizePow2Shift, expectedItemsCount.Value);
+				if (_memoryBlocks!.Length > 0 && _blockSize != _memoryBlocks[0].Length)
 				{
 					_blockSize = _memoryBlocks[0].Length;
+					_blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)_blockSize));
+					if (_blockSize >= RecyclableDefaults.MinPooledArrayLength)
+					{
+						_blockArrayPool = RecyclableArrayPool<T>.Shared(_blockSize);
+					}
+				}
+				else
+				{
+					_blockSize = minBlockSize;
 				}
-
-				_blockSizePow2Shift = MathUtils.GetPow2Shift(_blockSize);
 			}
 			else
 			{
 				_blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize));
-				_blockSizePow2Shift = MathUtils.GetPow2Shift(minBlockSize);
+				_blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)_blockSize));
+				_blockArrayPool = _blockSize >= RecyclableDefaults.MinPooledArrayLength
+					? blockArrayPool ?? RecyclableArrayPool<T>.Shared(_blockSize)
+					: RecyclableArrayPool<T>.Null;
+
 				_memoryBlocks = _emptyMemoryBlocksArray;
 			}
 		}
@@ -361,25 +381,39 @@ namespace Recyclable.Collections
 		public RecyclableList(IEnumerable<T> source, int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool<T[]>? memoryBlocksPool = default, ArrayPool<T>? blockArrayPool = default)
 		{
 			memoryBlocksPool ??= _defaultMemoryBlocksPool;
-			blockArrayPool ??= _defaultBlockArrayPool;
 			_memoryBlocksPool = memoryBlocksPool;
-			_blockArrayPool = blockArrayPool;
 
 			if (expectedItemsCount > 0)
 			{
-				_blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize));
+				minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize));
-				_capacity = Resize(this, expectedItemsCount.Value);
+				_blockArrayPool = minBlockSize >= RecyclableDefaults.MinPooledArrayLength
-				if (_memoryBlocks.Length > 0)
+					? blockArrayPool ?? RecyclableArrayPool<T>.Shared(minBlockSize)
+					: RecyclableArrayPool<T>.Null;
+
+				_blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)minBlockSize));
+				_capacity = Resize(this, minBlockSize, _blockSizePow2Shift, expectedItemsCount.Value);
+				if (_memoryBlocks?.Length > 0 && minBlockSize != _memoryBlocks[0].Length)
 				{
 					_blockSize = _memoryBlocks[0].Length;
+					_blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)minBlockSize));
+					if (_blockSize >= RecyclableDefaults.MinPooledArrayLength)
+					{
+						_blockArrayPool = RecyclableArrayPool<T>.Shared(_blockSize);
+					}
+				}
+				else
+				{
+					_blockSize = minBlockSize;
 				}
-
-				_blockSizePow2Shift = MathUtils.GetPow2Shift(_blockSize);
 			}
 			else
 			{
 				_blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize));
-				_blockSizePow2Shift = MathUtils.GetPow2Shift(minBlockSize);
+				_blockArrayPool = _blockSize >= RecyclableDefaults.MinPooledArrayLength
+					? blockArrayPool ?? RecyclableArrayPool<T>.Shared(_blockSize)
+					: RecyclableArrayPool<T>.Null;
+
+				_blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)_blockSize));
 				_memoryBlocks = _emptyMemoryBlocksArray;
 			}
 
@@ -635,7 +669,9 @@ namespace Recyclable.Collections
 				targetBlockIndex = _nextItemBlockIndex,
 				memoryBlockCount = _memoryBlocks.Length;
 
-			T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _blockArrayPool.Rent(items.Count) : new T[items.Count];
+			// We're better off to temporarily copy it to a fixed array,
+			// than copying them item by item. We may run into OOMs, though.
+			T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _defaultBlockArrayPool.Rent(items.Count) : new T[items.Count];
 			try
 			{
 				items.CopyTo(itemsBuffer, 0);
@@ -665,7 +701,7 @@ namespace Recyclable.Collections
 			{
 				if (items.Count >= RecyclableDefaults.MinPooledArrayLength)
 				{
-					_blockArrayPool.Return(itemsBuffer, NeedsClearing);
+					_defaultBlockArrayPool.Return(itemsBuffer, NeedsClearing);
 				}
 			}
 		}