From 4165c9c6d91d7855af207b47604115c68b2eeba3 Mon Sep 17 00:00:00 2001 From: mlemanczyk Date: Mon, 10 Apr 2023 22:35:00 +0200 Subject: [PATCH] RecyclableList.EnsureCapacity & .Resize performance improvements. Custom ArrayPool added. --- Recyclable.Collections/OneSizeArrayPool.cs | 179 +++++++++++++++++++++ Recyclable.Collections/RecyclableList.cs | 98 +++++++---- 2 files changed, 246 insertions(+), 31 deletions(-) create mode 100644 Recyclable.Collections/OneSizeArrayPool.cs diff --git a/Recyclable.Collections/OneSizeArrayPool.cs b/Recyclable.Collections/OneSizeArrayPool.cs new file mode 100644 index 0000000..3f54b31 --- /dev/null +++ b/Recyclable.Collections/OneSizeArrayPool.cs @@ -0,0 +1,179 @@ +using System.Buffers; +using System.Collections.Concurrent; + +namespace Recyclable.Collections +{ + public class MemoryBucket + { + public MemoryBucket(T[] memory, MemoryBucket nextBucket) + { + _memory = memory; + _nextBucket = nextBucket; + } + + public MemoryBucket(T[] memory) + { + _memory = memory; + } + + private T[] _memory; + public T[] Memory { get => _memory; set => _memory = value; } + + private MemoryBucket? _nextBucket; + public MemoryBucket? NextBucket { get => _nextBucket; set => _nextBucket = value; } + } + + public static class RecyclableArrayPool + { + public static readonly OneSizeArrayPool Null = new(0, 0); + + private static readonly ConcurrentDictionary> _pools = new(); + + public static ArrayPool Shared(int blockSize) + { + //lock (_lock) + { + if (!_pools.TryGetValue(blockSize, out ArrayPool? pool)) + { + pool = new OneSizeArrayPool(blockSize, 100); + pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize]; + } + + return pool; + } + + //_lock.Wait(); + //try + //{ + //} + //finally + //{ + // _lock.Release(); + //} + } + } + + public class OneSizeArrayPool : ArrayPool + { + private readonly int _blockSize; + private readonly int _minBlockCount; + private int _blockCount; + private MemoryBucket? _currentBucket; + //SemaphoreSlim _lock = new(1); + //ManualResetEventSlim _lock = new(true, 1); + private readonly object _updateLock = new(); + //private Task? _bufferFillUpTask; + + //private Task CreateAdditionalBucket(int minimumLength, int blockCount) + //{ + // return Task.Factory.StartNew(() => + // { + // MemoryBucket firstBucket = new(new T[minimumLength]); + // MemoryBucket currentBucket = firstBucket; + // for (var i = 0; i < blockCount; i++) + // { + // currentBucket = new(new T[minimumLength], currentBucket); + // } + + // lock (_updateLock) + // //_lock.Wait(); + // //try + // { + // currentBucket.NextBucket = _currentBucket; + // _currentBucket = firstBucket; + // _blockCount += blockCount; + // //_bufferFillUpTask = null; + // } + // //finally + // //{ + // // _ = _lock.Release(); + // // //_lock.Set(); + // //} + // }); + //} + + public OneSizeArrayPool(int blockSize, int minBlockCount) + { + _blockSize = blockSize; + _minBlockCount = minBlockCount; + } + + public override T[] Rent(int minimumLength) + { + if (minimumLength != _blockSize) + { + ThrowWrongRentedArraySizeException(minimumLength, _blockSize); + } + + MemoryBucket? bucket; + lock (_updateLock) + //_lock.Wait(); + //try + { + if (_currentBucket != null) + { + _blockCount--; + bucket = _currentBucket; + _currentBucket = _currentBucket.NextBucket; + } + else + { + bucket = null; + //if (_bufferFillUpTask == null) + //{ + // _bufferFillUpTask = CreateAdditionalBucket(minimumLength, _minBlockCount); + //} + } + } + //finally + //{ + // _ = _lock.Release(); + // //_lock.Set(); + //} + + return bucket != null ? bucket.Memory : new T[_blockSize]; + } + + public override void Return(T[] array, bool clearArray = false) + { + if (array.Length != _blockSize) + { + ThrowWrongReturnedArraySizeException(array.Length, _blockSize); + } + + MemoryBucket bucket = new(array); + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + + lock (_updateLock) + //lock (_lock) + //_lock.Wait(); + //try + { + if (_blockCount < _minBlockCount) + { + bucket.NextBucket = _currentBucket; + _currentBucket = bucket; + _blockCount++; + } + } + //finally + //{ + // _ = _lock.Release(); + // //_lock.Set(); + //} + } + + private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize) + { + throw new InvalidOperationException($"The array size {returnedLength} doesn't match the size {blockSize} of the pool buffer and cannot be returned to this pool."); + } + + private static void ThrowWrongRentedArraySizeException(int requestedSize, int blockSize) + { + throw new InvalidOperationException($"The requested size {requestedSize} doesn't match the size {blockSize} of the pool buffer and cannot be taken from this pool."); + } + } +} diff --git a/Recyclable.Collections/RecyclableList.cs b/Recyclable.Collections/RecyclableList.cs index 15b268a..43b5212 100644 --- a/Recyclable.Collections/RecyclableList.cs +++ b/Recyclable.Collections/RecyclableList.cs @@ -15,7 +15,7 @@ namespace Recyclable.Collections private static readonly T[] _emptyBlockArray = new T[0]; private int _blockSize; - private int _blockSizePow2Shift; + private byte _blockSizePow2Shift; private int _nextItemBlockIndex; private int _nextItemIndex; @@ -65,7 +65,7 @@ namespace Recyclable.Collections while (sourceBlockIndex < lastTakenBlockIndex || (sourceBlockIndex == lastTakenBlockIndex && (sourceItemIndex < nextItemIndex || sourceBlockIndex != list._nextItemBlockIndex))) { int toCopy = sourceBlockIndex < lastTakenBlockIndex || nextItemIndex == 0 - ? Math.Min(blockSize - sourceItemIndex, blockSize - targetItemIndex) + ? blockSize - (sourceItemIndex >= targetItemIndex ? sourceItemIndex : targetItemIndex) : Math.Min(nextItemIndex, blockSize - targetItemIndex); Array.Copy(memoryBlocks[sourceBlockIndex], sourceItemIndex, memoryBlocks[targetBlockIndex], targetItemIndex, toCopy); @@ -100,9 +100,8 @@ namespace Recyclable.Collections protected void AddRangeEnumerated(IEnumerable source, int growByCount) { long i, copied = _longCount; - int blockSize = _blockSize, targetItemIdx = _nextItemIndex, targetBlockIdx = _nextItemBlockIndex; - + using IEnumerator enumerator = source.GetEnumerator(); if (!enumerator.MoveNext()) { @@ -240,13 +239,11 @@ namespace Recyclable.Collections } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] - protected static long Resize(RecyclableList list, long newCapacity) + protected static long Resize(RecyclableList list, int minBlockSize, byte minBlockSizeBitShift, long newCapacity) { ArrayPool blockArrayPool = list._blockArrayPool; - int minBlockSize = list._blockSize; - int minBlockSizePowerToShift = 31 - BitOperations.LeadingZeroCount((uint)minBlockSize); - int sourceBlockCount = checked((int)(list._capacity >> minBlockSizePowerToShift)); - int requiredBlockCount = checked((int)(newCapacity >> minBlockSizePowerToShift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0)); + int sourceBlockCount = checked((int)(list._capacity >> minBlockSizeBitShift)); + int requiredBlockCount = checked((int)(newCapacity >> minBlockSizeBitShift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0)); int blockIndex; Span memoryBlocksSpan; @@ -313,7 +310,7 @@ namespace Recyclable.Collections } } - return (long)newMemoryBlocks.Length << minBlockSizePowerToShift; + return (long)newMemoryBlocks.Length << minBlockSizeBitShift; } [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] @@ -323,37 +320,60 @@ namespace Recyclable.Collections ? checked((long)BitOperations.RoundUpToPowerOf2((ulong)requestedCapacity)) : requestedCapacity; - newCapacity = Resize(list, newCapacity); - if (newCapacity > 0 && list._capacity == 0) + int blockSize = list._blockSize; + newCapacity = RecyclableList.Resize(list, blockSize, list._blockSizePow2Shift, newCapacity); + if (newCapacity > 0 && blockSize != list._memoryBlocks[0].Length) { - list._blockSize = list._memoryBlocks[0].Length; - list._blockSizePow2Shift = MathUtils.GetPow2Shift(list._blockSize); + blockSize = list._memoryBlocks[0].Length; + if (blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + list._blockArrayPool = RecyclableArrayPool.Shared(blockSize); + } + + list._blockSizePow2Shift = (byte)MathUtils.GetPow2Shift(blockSize); + list._blockSize = blockSize; } list._capacity = newCapacity; return newCapacity; } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] public RecyclableList(int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) { _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; - _blockArrayPool = blockArrayPool ?? _defaultBlockArrayPool; if (expectedItemsCount > 0) { - _blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); - _capacity = Resize(this, expectedItemsCount.Value); - if (_memoryBlocks!.Length > 0) + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + _blockArrayPool = minBlockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(minBlockSize) + : RecyclableArrayPool.Null; + + _blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)minBlockSize)); + _capacity = Resize(this, minBlockSize, _blockSizePow2Shift, expectedItemsCount.Value); + if (_memoryBlocks!.Length > 0 && _blockSize != _memoryBlocks[0].Length) { _blockSize = _memoryBlocks[0].Length; + _blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)_blockSize)); + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + _blockArrayPool = RecyclableArrayPool.Shared(_blockSize); + } + } + else + { + _blockSize = minBlockSize; } - - _blockSizePow2Shift = MathUtils.GetPow2Shift(_blockSize); } else { _blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); - _blockSizePow2Shift = MathUtils.GetPow2Shift(minBlockSize); + _blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)_blockSize)); + _blockArrayPool = _blockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(_blockSize) + : RecyclableArrayPool.Null; + _memoryBlocks = _emptyMemoryBlocksArray; } } @@ -361,25 +381,39 @@ namespace Recyclable.Collections public RecyclableList(IEnumerable source, int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) { memoryBlocksPool ??= _defaultMemoryBlocksPool; - blockArrayPool ??= _defaultBlockArrayPool; _memoryBlocksPool = memoryBlocksPool; - _blockArrayPool = blockArrayPool; if (expectedItemsCount > 0) { - _blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); - _capacity = Resize(this, expectedItemsCount.Value); - if (_memoryBlocks.Length > 0) + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + _blockArrayPool = minBlockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(minBlockSize) + : RecyclableArrayPool.Null; + + _blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)minBlockSize)); + _capacity = Resize(this, minBlockSize, _blockSizePow2Shift, expectedItemsCount.Value); + if (_memoryBlocks?.Length > 0 && minBlockSize != _memoryBlocks[0].Length) { _blockSize = _memoryBlocks[0].Length; + _blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)minBlockSize)); + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + _blockArrayPool = RecyclableArrayPool.Shared(_blockSize); + } + } + else + { + _blockSize = minBlockSize; } - - _blockSizePow2Shift = MathUtils.GetPow2Shift(_blockSize); } else { _blockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); - _blockSizePow2Shift = MathUtils.GetPow2Shift(minBlockSize); + _blockArrayPool = _blockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(_blockSize) + : RecyclableArrayPool.Null; + + _blockSizePow2Shift = (byte)(31 - BitOperations.LeadingZeroCount((uint)_blockSize)); _memoryBlocks = _emptyMemoryBlocksArray; } @@ -635,7 +669,9 @@ namespace Recyclable.Collections targetBlockIndex = _nextItemBlockIndex, memoryBlockCount = _memoryBlocks.Length; - T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _blockArrayPool.Rent(items.Count) : new T[items.Count]; + // We're better off to temporarily copy it to a fixed array, + // than copying them item by item. We may run into OOMs, though. + T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _defaultBlockArrayPool.Rent(items.Count) : new T[items.Count]; try { items.CopyTo(itemsBuffer, 0); @@ -665,7 +701,7 @@ namespace Recyclable.Collections { if (items.Count >= RecyclableDefaults.MinPooledArrayLength) { - _blockArrayPool.Return(itemsBuffer, NeedsClearing); + _defaultBlockArrayPool.Return(itemsBuffer, NeedsClearing); } } }