From 4da13cd9c10b7aba3b795adeda2712f23aa5fa94 Mon Sep 17 00:00:00 2001 From: mlemanczyk Date: Sat, 8 Jul 2023 14:45:24 +0200 Subject: [PATCH] Initial OneSizeArrayPool performance improvements. --- .../Pools/OneSizeArrayPool.cs | 124 ++++++++++++++++-- .../Pools/RecyclableArrayPool.cs | 91 ++++++++++--- Recyclable.Collections/RecyclableDefaults.cs | 2 +- .../RecyclableList.Helpers.cs | 11 +- Recyclable.Collections/RecyclableList.cs | 7 +- .../RecyclableLongList.Helpers.cs | 14 -- 6 files changed, 190 insertions(+), 59 deletions(-) diff --git a/Recyclable.Collections/Pools/OneSizeArrayPool.cs b/Recyclable.Collections/Pools/OneSizeArrayPool.cs index 4434d63..04e441f 100644 --- a/Recyclable.Collections/Pools/OneSizeArrayPool.cs +++ b/Recyclable.Collections/Pools/OneSizeArrayPool.cs @@ -2,18 +2,85 @@ namespace Recyclable.Collections.Pools { - public sealed class OneSizeArrayPool + public sealed class OneSizeArrayPool : ArrayPool { + private const int MemoryReleaseTaskSleepMilliseconds = 500; + internal bool _memoryExceeded; + private bool _disposed; + + ~OneSizeArrayPool() + { + _disposed = true; + } + + private async Task ReleaseMemoryOnExcess() + { + while (!_disposed) + { + if (_memoryExceeded) + { + Enter(); + if (_currentBucket != null) + { + _blockCount--; + _currentBucket = _currentBucket.NextBucket; + } + + if (_currentBucket != null) + { + _blockCount--; + _currentBucket = _currentBucket.NextBucket; + } + + if (_currentBucket != null) + { + _blockCount--; + _currentBucket = _currentBucket.NextBucket; + } + + if (_currentBucket != null) + { + _blockCount--; + _currentBucket = _currentBucket.NextBucket; + } + + Exit(); + } + + await Task.Delay(MemoryReleaseTaskSleepMilliseconds); + } + } + private readonly int _blockSize; private readonly int _minBlockCount; - private int _blockCount; - private MemoryBucket? _currentBucket; - private readonly SpinLock _updateLock = new(false); + private volatile int _blockCount; + private volatile MemoryBucket? _currentBucket; + private volatile int _locked; + + private void Enter() + { + if (Interlocked.CompareExchange(ref _locked, 1, 0) == 0) + { + return; + } + + SpinWait waiter = new(); + while (Interlocked.CompareExchange(ref _locked, 1, 0) == 1) + { + waiter.SpinOnce(); + } + } + + private void Exit() + { + _locked = 0; + } public OneSizeArrayPool(int blockSize, int minBlockCount) { _blockSize = blockSize; _minBlockCount = minBlockCount; + _ = Task.Run(ReleaseMemoryOnExcess); } public override T[] Rent(int minimumLength) @@ -24,8 +91,7 @@ namespace Recyclable.Collections.Pools } MemoryBucket? bucket; - bool lockTaken = false; - _updateLock.Enter(ref lockTaken); + Enter(); if (_currentBucket != null) { _blockCount--; @@ -37,14 +103,46 @@ namespace Recyclable.Collections.Pools bucket = null; } - if (lockTaken) + Exit(); + return bucket != null ? bucket.Memory : new T[_blockSize]; + } + + public T[] Rent() + { + MemoryBucket? bucket; + Enter(); + if (_currentBucket != null) { - _updateLock.Exit(false); + _blockCount--; + bucket = _currentBucket; + _currentBucket = _currentBucket.NextBucket; + } + else + { + bucket = null; } + Exit(); return bucket != null ? bucket.Memory : new T[_blockSize]; } + public void ReturnUnchecked(T[] array, bool clearArray) + { + MemoryBucket bucket = new(array); + if (clearArray) + { + // TODO: Measure performance + // new Span(array).Clear(); + Array.Clear(array, 0, array.Length); + } + + Enter(); + bucket.NextBucket = _currentBucket; + _currentBucket = bucket; + _blockCount++; + Exit(); + } + public override void Return(T[] array, bool clearArray = false) { if (array.Length != _blockSize) @@ -55,11 +153,12 @@ namespace Recyclable.Collections.Pools MemoryBucket bucket = new(array); if (clearArray) { + // TODO: Measure performance + // new Span(array).Clear(); Array.Clear(array, 0, array.Length); } - bool lockTaken = false; - _updateLock.Enter(ref lockTaken); + Enter(); if (_blockCount < _minBlockCount) { bucket.NextBucket = _currentBucket; @@ -67,10 +166,7 @@ namespace Recyclable.Collections.Pools _blockCount++; } - if (lockTaken) - { - _updateLock.Exit(false); - } + Exit(); } private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize) diff --git a/Recyclable.Collections/Pools/RecyclableArrayPool.cs b/Recyclable.Collections/Pools/RecyclableArrayPool.cs index 3e44463..c8e3121 100644 --- a/Recyclable.Collections/Pools/RecyclableArrayPool.cs +++ b/Recyclable.Collections/Pools/RecyclableArrayPool.cs @@ -1,35 +1,84 @@ -using System.Buffers; -using System.Collections.Concurrent; +using System.Numerics; namespace Recyclable.Collections.Pools { - public static class RecyclableArrayPool + public static class RecyclableArrayPool { - public static readonly OneSizeArrayPool Null = new(0, 0); - - private static readonly ConcurrentDictionary> _pools = new(); - - public static ArrayPool Shared(int blockSize) + private enum MemoryPressure { - //lock (_lock) + High, + Medium, + Low + } + + private static MemoryPressure GetMemoryPressure() + { + const double HighPressureThreshold = .90; // Percent of GC memory pressure threshold we consider "high" + const double MediumPressureThreshold = .70; // Percent of GC memory pressure threshold we consider "medium" + + GCMemoryInfo memoryInfo = GC.GetGCMemoryInfo(); + + if (memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * HighPressureThreshold) + { + return MemoryPressure.High; + } + + if (memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * MediumPressureThreshold) + { + return MemoryPressure.Medium; + } + + return MemoryPressure.Low; + } + + private static async Task CheckAndReportMemoryPressure() + { + var previousValue = false; + while (true) { - if (!_pools.TryGetValue(blockSize, out ArrayPool? pool)) + bool memoryExceeded = GetMemoryPressure() == MemoryPressure.High; + if (memoryExceeded != previousValue) { - pool = new OneSizeArrayPool(blockSize, 100); - pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize]; + previousValue = memoryExceeded; + for (var i = 0; i < _pools!.Length; i++) + { + _pools[i]._memoryExceeded = memoryExceeded; + } } - return pool; + await Task.Delay(500); + } + } + + static RecyclableArrayPool() + { + var pools = _pools; + for (var arraySize = 0; arraySize < 32; arraySize++) + { + int blockSize = 2 << arraySize; + pools[arraySize] = new OneSizeArrayPool(blockSize, 1000); } - //_lock.Wait(); - //try - //{ - //} - //finally - //{ - // _lock.Release(); - //} + _ = Task.Run(CheckAndReportMemoryPressure); } + + public static readonly OneSizeArrayPool Null = new(0, 0); + + private static readonly OneSizeArrayPool[] _pools = new OneSizeArrayPool[32]; + public static T[] Rent(int blockSize) => _pools[31 - BitOperations.LeadingZeroCount((uint)blockSize) - 1].Rent(); + public static void Return(in T[] memoryBlock, bool needsClearing) => _pools[31 - BitOperations.LeadingZeroCount((uint)memoryBlock.Length) - 1].ReturnUnchecked(memoryBlock, needsClearing); + + // private static readonly ConcurrentDictionary> _pools = new(); + + public static OneSizeArrayPool Shared(int blockSize) => + //lock (_lock) + //{ + + // if (!_pools.TryGetValue(blockSize, out ArrayPool? pool)) + // { + // pool = new OneSizeArrayPool(blockSize, 100); + // pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize]; + // } + _pools[31 - BitOperations.LeadingZeroCount((uint)blockSize) - 1];//}//_lock.Wait();//try//{//}//finally//{// _lock.Release();//} } } diff --git a/Recyclable.Collections/RecyclableDefaults.cs b/Recyclable.Collections/RecyclableDefaults.cs index 5e5898a..ef11b3c 100644 --- a/Recyclable.Collections/RecyclableDefaults.cs +++ b/Recyclable.Collections/RecyclableDefaults.cs @@ -6,7 +6,7 @@ public const int BlockSize = 16_384; public const int Capacity = 1; public const long MinItemsCountForParallelization = 850_000; - public const int MinPooledArrayLength = 128; + public const int MinPooledArrayLength = 37; public const int MaxPooledBlockSize = 2_147_483_591; public const long ItemNotFoundIndexLong = -1L; public const int ItemNotFoundIndex = -1; diff --git a/Recyclable.Collections/RecyclableList.Helpers.cs b/Recyclable.Collections/RecyclableList.Helpers.cs index 10322f3..47d324e 100644 --- a/Recyclable.Collections/RecyclableList.Helpers.cs +++ b/Recyclable.Collections/RecyclableList.Helpers.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Runtime.CompilerServices; +using Recyclable.Collections.Pools; namespace Recyclable.Collections { @@ -223,8 +224,6 @@ namespace Recyclable.Collections return values[middle]; } - internal static readonly ArrayPool _arrayPool = ArrayPool.Create(); - private static readonly bool NeedsClearing = !typeof(T).IsValueType; [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] @@ -235,7 +234,7 @@ namespace Recyclable.Collections T[] oldMemoryBlock = list._memoryBlock; list._memoryBlock = (list._capacity <<= 1) < RecyclableDefaults.MinPooledArrayLength ? new T[list._capacity] - : _arrayPool.Rent(list._capacity); + : RecyclableArrayPool.Rent(list._capacity); // & WAS SLOWER WITHOUT new Span(oldMemoryBlock).CopyTo(list._memoryBlock); @@ -249,7 +248,7 @@ namespace Recyclable.Collections //} // // If anything, it has been already cleared above, so we don't need to repeat it. - _arrayPool.Return(oldMemoryBlock, NeedsClearing); + RecyclableArrayPool.Return(oldMemoryBlock, NeedsClearing); } // list._memoryBlock = newMemoryBlock; @@ -260,7 +259,7 @@ namespace Recyclable.Collections public static void ResizeAndCopy(RecyclableList sourceList, int newSize) { T[] newMemoryBlock = newSize >= RecyclableDefaults.MinPooledArrayLength - ? _arrayPool.Rent(newSize) + ? RecyclableArrayPool.Rent(newSize) : new T[newSize]; // & WAS SLOWER WITHOUT @@ -278,7 +277,7 @@ namespace Recyclable.Collections //} // If anything, it has been already cleared above, so we don't need to repeat it. - _arrayPool.Return(oldMemoryBlock, NeedsClearing); + RecyclableArrayPool.Return(oldMemoryBlock, NeedsClearing); } sourceList._memoryBlock = newMemoryBlock; diff --git a/Recyclable.Collections/RecyclableList.cs b/Recyclable.Collections/RecyclableList.cs index 180bd41..fbd0bd5 100644 --- a/Recyclable.Collections/RecyclableList.cs +++ b/Recyclable.Collections/RecyclableList.cs @@ -1,5 +1,6 @@ using System.Collections; using System.Runtime.CompilerServices; +using Recyclable.Collections.Pools; namespace Recyclable.Collections { @@ -80,7 +81,7 @@ namespace Recyclable.Collections if (initialCapacity >= InitialCapacity) { _memoryBlock = initialCapacity >= RecyclableDefaults.MinPooledArrayLength - ? RecyclableListHelpers._arrayPool.Rent(initialCapacity) + ? RecyclableArrayPool.Rent(initialCapacity) : new T[initialCapacity]; _capacity = _memoryBlock.Length; } @@ -138,7 +139,7 @@ namespace Recyclable.Collections if (initialCapacity >= InitialCapacity) { _memoryBlock = initialCapacity >= RecyclableDefaults.MinPooledArrayLength - ? RecyclableListHelpers._arrayPool.Rent(initialCapacity) + ? RecyclableArrayPool.Rent(initialCapacity) : new T[initialCapacity]; _capacity = _memoryBlock.Length; } @@ -515,7 +516,7 @@ namespace Recyclable.Collections { _capacity = 0; // If anything, it has been already cleared by .Clear(), Remove() or RemoveAt() methods, as the list was modified / disposed. - RecyclableListHelpers._arrayPool.Return(_memoryBlock, false); + RecyclableArrayPool.Return(_memoryBlock, false); } GC.SuppressFinalize(this); diff --git a/Recyclable.Collections/RecyclableLongList.Helpers.cs b/Recyclable.Collections/RecyclableLongList.Helpers.cs index 424f49e..a60cc66 100644 --- a/Recyclable.Collections/RecyclableLongList.Helpers.cs +++ b/Recyclable.Collections/RecyclableLongList.Helpers.cs @@ -156,15 +156,6 @@ namespace Recyclable.Collections } } - [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] - private static int GetEstimatedBlockSize(int minBlockSize, ArrayPool blockArrayPool) - { - var array = blockArrayPool.Rent(minBlockSize); - minBlockSize = array.Length; - blockArrayPool.Return(array); - return minBlockSize; - } - /// /// Creates a set of new memory buffers, if needed, to allow storing at minimum no. of items. /// @@ -220,11 +211,6 @@ namespace Recyclable.Collections { if (minBlockSize >= RecyclableDefaults.MinPooledArrayLength) { - if (sourceBlockCount == 0) - { - minBlockSize = GetEstimatedBlockSize(minBlockSize, blockArrayPool); - } - blockIndex = sourceBlockCount; while (true) {