RecyclableList.IndexOf parallel performance improvements.

This commit is contained in:
mlemanczyk committed 2023-05-03 12:36:37 +02:00
1 parent 99600d4854
commit 3db10d7602
21 files changed
+4239 -468

No files matched your search

@@ -1,4 +1,5 @@
using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Attributes;
using Recyclable.Collections.Pools;
using System.Buffers; using System.Buffers;
using System.Collections; using System.Collections;
using System.Numerics; using System.Numerics;
File diff suppressed because it is too large. Load diff
@@ -1,6 +1,7 @@
using System.Reflection; using System.Reflection;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Attributes;
using Collections.Pooled;
using Recyclable.Collections.Benchmarks.Core; using Recyclable.Collections.Benchmarks.Core;
namespace Recyclable.Collections.Benchmarks.POC namespace Recyclable.Collections.Benchmarks.POC
@@ -10,9 +11,17 @@ namespace Recyclable.Collections.Benchmarks.POC
private long[]? _testArray; private long[]? _testArray;
public long[] TestObjects => _testArray ?? throw new NullReferenceException("Something is wrong and test objects are null"); public long[] TestObjects => _testArray ?? throw new NullReferenceException("Something is wrong and test objects are null");
private RecyclableList<long>? _testRecyclableList; private RecyclableList<long>? _testRecyclableList;
public RecyclableList<long> TestObjectsAsRecyclableList => _testRecyclableList ?? throw new NullReferenceException("Something is wrong and RecyclableList is null"); public RecyclableList<long> TestObjectsAsRecyclableList => _testRecyclableList ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableList<long>)} is null");
private RecyclableListIndexOfV1<long>? _testRecyclableListIndexOfV1; private RecyclableListIndexOfV1<long>? _testRecyclableListIndexOfV1;
internal RecyclableListIndexOfV1<long> TestObjectsAsRecyclableListIndexOfV1 => _testRecyclableListIndexOfV1 ?? throw new NullReferenceException("Something is wrong and RecyclableListIndexOfV1 is null"); internal RecyclableListIndexOfV1<long> TestObjectsAsRecyclableListIndexOfV1 => _testRecyclableListIndexOfV1 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV1<long>)} is null");
private RecyclableListIndexOfV2<long>? _testRecyclableListIndexOfV2;
internal RecyclableListIndexOfV2<long> TestObjectsAsRecyclableListIndexOfV2 => _testRecyclableListIndexOfV2 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV2<long>)} is null");
private RecyclableListIndexOfV3<long>? _testRecyclableListIndexOfV3;
internal RecyclableListIndexOfV3<long> TestObjectsAsRecyclableListIndexOfV3 => _testRecyclableListIndexOfV3 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV3<long>)} is null");
private RecyclableListIndexOfV4<long>? _testRecyclableListIndexOfV4;
internal RecyclableListIndexOfV4<long> TestObjectsAsRecyclableListIndexOfV4 => _testRecyclableListIndexOfV4 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV4<long>)} is null");
private PooledList<long>? _testPooledList;
internal PooledList<long> TestPooledList => _testPooledList ?? throw new NullReferenceException($"Something is wrong and {nameof(PooledList<long>)} is null");
private static readonly MethodInfo _ensureCapacityNewFunc; private static readonly MethodInfo _ensureCapacityNewFunc;
private static readonly MethodInfo _ensureCapacityV1Func; private static readonly MethodInfo _ensureCapacityV1Func;
@@ -47,6 +56,9 @@ namespace Recyclable.Collections.Benchmarks.POC
benchmark.RecyclableList_EnsureCapacityV3_ByBlockSize(); benchmark.RecyclableList_EnsureCapacityV3_ByBlockSize();
benchmark.RecyclableList_IndexOf(); benchmark.RecyclableList_IndexOf();
benchmark.RecyclableList_IndexOfV1(); benchmark.RecyclableList_IndexOfV1();
benchmark.RecyclableList_IndexOfV2();
benchmark.RecyclableList_IndexOfV3();
benchmark.RecyclableList_IndexOfV4();
} }
[GlobalSetup] [GlobalSetup]
@@ -55,6 +67,10 @@ namespace Recyclable.Collections.Benchmarks.POC
_testArray = DataGenerator.EnumerateTestObjects(TestObjectCount); _testArray = DataGenerator.EnumerateTestObjects(TestObjectCount);
_testRecyclableList = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); _testRecyclableList = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount);
_testRecyclableListIndexOfV1 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); _testRecyclableListIndexOfV1 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount);
_testRecyclableListIndexOfV2 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount);
_testRecyclableListIndexOfV3 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount);
_testRecyclableListIndexOfV4 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount);
_testPooledList = new PooledList<long>(TestObjects, ClearMode.Auto);
// Warm up memory pools for fair comparison with 1st run // Warm up memory pools for fair comparison with 1st run
RecyclableList_EnsureCapacity_ByPowOf2(); RecyclableList_EnsureCapacity_ByPowOf2();
@@ -63,6 +79,10 @@ namespace Recyclable.Collections.Benchmarks.POC
RecyclableList_EnsureCapacityV3_ByPowOf2(); RecyclableList_EnsureCapacityV3_ByPowOf2();
RecyclableList_IndexOf(); RecyclableList_IndexOf();
RecyclableList_IndexOfV1(); RecyclableList_IndexOfV1();
RecyclableList_IndexOfV2();
RecyclableList_IndexOfV3();
RecyclableList_IndexOfV4();
PooledList_IndexOf();
} }
[GlobalCleanup] [GlobalCleanup]
@@ -73,6 +93,15 @@ namespace Recyclable.Collections.Benchmarks.POC
_testRecyclableList = null; _testRecyclableList = null;
_testRecyclableListIndexOfV1?.Dispose(); _testRecyclableListIndexOfV1?.Dispose();
_testRecyclableListIndexOfV1 = null; _testRecyclableListIndexOfV1 = null;
_testRecyclableListIndexOfV2?.Dispose();
_testRecyclableListIndexOfV2 = null;
_testRecyclableListIndexOfV3?.Dispose();
_testRecyclableListIndexOfV3 = null;
_testRecyclableListIndexOfV4?.Dispose();
_testRecyclableListIndexOfV4 = null;
_testPooledList?.Dispose();
_testPooledList = null;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
} }
} }
} }
+32
View File
@@ -0,0 +1,32 @@
using Recyclable.Collections.Pools;
namespace Recyclable.Collections
{
public sealed class ItemRange
{
public int BlockIndex;
public int StartingItemIndex;
public long ItemsToSearchCount;
internal bool _returned;
public ItemRange()
{
}
public ItemRange(int blockIndex, int startingItemIndex, long itemsToSearchCount)
{
BlockIndex = blockIndex;
ItemsToSearchCount = itemsToSearchCount;
StartingItemIndex = startingItemIndex;
}
public void Dispose()
{
if (!_returned)
{
_returned = true;
ItemRangePool.Shared.Return(this);
}
}
}
}
+13 -221
View File
@@ -1,237 +1,29 @@
using Microsoft.Extensions.ObjectPool; using System.Runtime.CompilerServices;
using System.Runtime.CompilerServices; using Recyclable.Collections.Parallel;
namespace Recyclable.Collections namespace Recyclable.Collections
{ {
public sealed class ItemRange : IDisposable
{
private static readonly ObjectPool<ItemRange> _defaultPool = ItemRangePool.Shared;
public int BlockIndex;
public int StartingItemIndex;
public long ItemsToSearchCount;
private ObjectPool<ItemRange>? _assignedPool;
public ItemRange()
{
}
public ItemRange(int blockIndex, int startingItemIndex, long itemsToSearchCount)
{
BlockIndex = blockIndex;
ItemsToSearchCount = itemsToSearchCount;
StartingItemIndex = startingItemIndex;
}
public void Dispose()
{
if (_assignedPool != null)
{
var assignedPool = _assignedPool;
_assignedPool = null;
assignedPool.Return(this);
}
GC.SuppressFinalize(this);
}
public static ItemRange Create(int blockIndex, int startingItemIndex, long itemsToSearchCount)
{
var itemRange = _defaultPool.Get();
itemRange.BlockIndex = blockIndex;
itemRange.ItemsToSearchCount = itemsToSearchCount;
itemRange.StartingItemIndex = startingItemIndex;
return itemRange;
}
}
public static class ItemRangePool
{
[ThreadStatic]
private static ObjectPool<ItemRange> _shared;
public static ObjectPool<ItemRange> Shared => _shared;
static ItemRangePool()
{
_shared = new DefaultObjectPool<ItemRange>(new DefaultPooledObjectPolicy<ItemRange>());
}
}
public sealed class BarrierSlimmer : IDisposable
{
private static readonly ObjectPool<BarrierSlimmer> _defaultPool = new DefaultObjectPool<BarrierSlimmer>(new DefaultPooledObjectPolicy<BarrierSlimmer>());
private ObjectPool<BarrierSlimmer>? _assignedPool;
private readonly object _lock = new();
public bool HasParticipants;
public int Participants;
public void AddParticipant()
{
Monitor.Enter(_lock);
if (++Participants == 1)
{
HasParticipants = true;
}
Monitor.Exit(_lock);
}
public void RemoveParticipant()
{
Monitor.Enter(_lock);
if (Participants == 0)
{
Monitor.Exit(_lock);
ThrowNoMoreParticipants();
}
if (--Participants == 0)
{
HasParticipants = false;
}
Monitor.Exit(_lock);
}
private static void ThrowNoMoreParticipants()
{
throw new InvalidOperationException("There are no more participants. The operation is not allowed");
}
public BarrierSlimmer()
{
}
public BarrierSlimmer(int participants)
{
Participants = participants;
HasParticipants = participants > 0;
}
public static BarrierSlimmer Create(int participants)
{
var result = _defaultPool.Get();
result.Participants = participants;
result.HasParticipants = participants > 0;
return result;
}
public void Dispose()
{
var assignedPool = _assignedPool;
_assignedPool = null;
assignedPool?.Return(this);
GC.SuppressFinalize(this);
}
public void SignalAndWait()
{
RemoveParticipant();
if (!HasParticipants)
{
return;
}
if (!HasParticipants)
{
return;
}
if (!HasParticipants)
{
return;
}
var waitSpinner = new SpinWait();
while (HasParticipants)
{
waitSpinner.SpinOnce();
}
}
}
public sealed class ManualResetEventSlimmer : IDisposable
{
private static readonly ObjectPool<ManualResetEventSlimmer> _defaultPool = ManualResetEventSlimmerPool.Shared;
private bool _isSet;
public bool IsSet => _isSet;
private ObjectPool<ManualResetEventSlimmer>? _assignedPool;
public ManualResetEventSlimmer(bool initialState)
{
_isSet = initialState;
}
public ManualResetEventSlimmer()
{
}
public void Set()
{
_isSet = true;
}
public void Reset()
{
_isSet = false;
}
public void Dispose()
{
var assignedPool = _assignedPool;
_assignedPool = null;
assignedPool?.Return(this);
GC.SuppressFinalize(this);
}
public static ManualResetEventSlimmer Create(bool initialState)
{
ManualResetEventSlimmer result = _defaultPool.Get();
result._assignedPool = _defaultPool;
result._isSet = initialState;
return result;
}
}
public static class ManualResetEventSlimmerPool
{
[ThreadStatic]
private static ObjectPool<ManualResetEventSlimmer> _shared;
public static ObjectPool<ManualResetEventSlimmer> Shared => _shared;
static ManualResetEventSlimmerPool()
{
_shared = new DefaultObjectPool<ManualResetEventSlimmer>(new DefaultPooledObjectPolicy<ManualResetEventSlimmer>());
}
}
public static class ItemRangesIterator public static class ItemRangesIterator
{ {
[MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.NoInlining)] [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.NoInlining)]
public static void Iterate(int blockIndex, int blockSize, byte blockSizePow2BitShift, long itemsCount, long step, Func<ItemRange, bool> action) public static void Iterate<TList, T>(in ParallelSynchronizationContext context, in SearchConfiguration<TList, T> search, Func<ParallelSynchronizationContext, SearchConfiguration<TList, T>, ItemRange, bool> action)
{ {
int itemIndex = 0; int itemIndex = 0;
ItemRange itemRange; ItemRange itemRange;
while (itemsCount > step) int blockIndex = search.StartingBlockIndex;
long itemsCount = search.ItemsCount;
while (itemsCount > search.Step)
{ {
itemRange = ItemRange.Create(blockIndex, itemIndex, step); itemRange = new(blockIndex, itemIndex, search.Step);
if (!action(itemRange)) if (!action(context, search, itemRange))
{ {
break; break;
} }
itemsCount -= step; itemsCount -= search.Step;
//blockIndex = (int)Math.DivRem(itemIndex + step, blockSize, out itemIndex); //blockIndex = (int)Math.DivRem(itemIndex + step, blockSize, out itemIndex);
blockIndex += (int)((itemIndex + step) >> blockSizePow2BitShift); blockIndex += (int)((itemIndex + search.Step) >> search.BlockSizePow2BitShift);
itemIndex = (int)((itemIndex + step) & (blockSize - 1)); itemIndex = (int)((itemIndex + search.Step) & search.BlockSizeMinus1);
//itemIndex += step; //itemIndex += step;
//while (itemIndex >= blockSize) //while (itemIndex >= blockSize)
//{ //{
@@ -240,8 +32,8 @@ namespace Recyclable.Collections
//} //}
} }
itemRange = ItemRange.Create(blockIndex, itemIndex, itemsCount); itemRange = new(blockIndex, itemIndex, itemsCount);
_ = action(itemRange); _ = action(context, search, itemRange);
} }
} }
} }
+22
View File
@@ -0,0 +1,22 @@
namespace Recyclable.Collections
{
public class MemoryBucket<T>
{
public MemoryBucket(T[] memory, MemoryBucket<T> nextBucket)
{
_memory = memory;
_nextBucket = nextBucket;
}
public MemoryBucket(T[] memory)
{
_memory = memory;
}
private T[] _memory;
public T[] Memory { get => _memory; set => _memory = value; }
private MemoryBucket<T>? _nextBucket;
public MemoryBucket<T>? NextBucket { get => _nextBucket; set => _nextBucket = value; }
}
}
-179
View File
@@ -1,179 +0,0 @@
using System.Buffers;
using System.Collections.Concurrent;
namespace Recyclable.Collections
{
public class MemoryBucket<T>
{
public MemoryBucket(T[] memory, MemoryBucket<T> nextBucket)
{
_memory = memory;
_nextBucket = nextBucket;
}
public MemoryBucket(T[] memory)
{
_memory = memory;
}
private T[] _memory;
public T[] Memory { get => _memory; set => _memory = value; }
private MemoryBucket<T>? _nextBucket;
public MemoryBucket<T>? NextBucket { get => _nextBucket; set => _nextBucket = value; }
}
public static class RecyclableArrayPool<T>
{
public static readonly OneSizeArrayPool<T> Null = new(0, 0);
private static readonly ConcurrentDictionary<int, ArrayPool<T>> _pools = new();
public static ArrayPool<T> Shared(int blockSize)
{
//lock (_lock)
{
if (!_pools.TryGetValue(blockSize, out ArrayPool<T>? pool))
{
pool = new OneSizeArrayPool<T>(blockSize, 100);
pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize];
}
return pool;
}
//_lock.Wait();
//try
//{
//}
//finally
//{
// _lock.Release();
//}
}
}
public class OneSizeArrayPool<T> : ArrayPool<T>
{
private readonly int _blockSize;
private readonly int _minBlockCount;
private int _blockCount;
private MemoryBucket<T>? _currentBucket;
//SemaphoreSlim _lock = new(1);
//ManualResetEventSlim _lock = new(true, 1);
private readonly object _updateLock = new();
//private Task? _bufferFillUpTask;
//private Task CreateAdditionalBucket(int minimumLength, int blockCount)
//{
// return Task.Factory.StartNew(() =>
// {
// MemoryBucket<T> firstBucket = new(new T[minimumLength]);
// MemoryBucket<T> currentBucket = firstBucket;
// for (var i = 0; i < blockCount; i++)
// {
// currentBucket = new(new T[minimumLength], currentBucket);
// }
// lock (_updateLock)
// //_lock.Wait();
// //try
// {
// currentBucket.NextBucket = _currentBucket;
// _currentBucket = firstBucket;
// _blockCount += blockCount;
// //_bufferFillUpTask = null;
// }
// //finally
// //{
// // _ = _lock.Release();
// // //_lock.Set();
// //}
// });
//}
public OneSizeArrayPool(int blockSize, int minBlockCount)
{
_blockSize = blockSize;
_minBlockCount = minBlockCount;
}
public override T[] Rent(int minimumLength)
{
if (minimumLength != _blockSize)
{
ThrowWrongRentedArraySizeException(minimumLength, _blockSize);
}
MemoryBucket<T>? bucket;
lock (_updateLock)
//_lock.Wait();
//try
{
if (_currentBucket != null)
{
_blockCount--;
bucket = _currentBucket;
_currentBucket = _currentBucket.NextBucket;
}
else
{
bucket = null;
//if (_bufferFillUpTask == null)
//{
// _bufferFillUpTask = CreateAdditionalBucket(minimumLength, _minBlockCount);
//}
}
}
//finally
//{
// _ = _lock.Release();
// //_lock.Set();
//}
return bucket != null ? bucket.Memory : new T[_blockSize];
}
public override void Return(T[] array, bool clearArray = false)
{
if (array.Length != _blockSize)
{
ThrowWrongReturnedArraySizeException(array.Length, _blockSize);
}
MemoryBucket<T> bucket = new(array);
if (clearArray)
{
Array.Clear(array, 0, array.Length);
}
lock (_updateLock)
//lock (_lock)
//_lock.Wait();
//try
{
if (_blockCount < _minBlockCount)
{
bucket.NextBucket = _currentBucket;
_currentBucket = bucket;
_blockCount++;
}
}
//finally
//{
// _ = _lock.Release();
// //_lock.Set();
//}
}
private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize)
{
throw new InvalidOperationException($"The array size {returnedLength} doesn't match the size {blockSize} of the pool buffer and cannot be returned to this pool.");
}
private static void ThrowWrongRentedArraySizeException(int requestedSize, int blockSize)
{
throw new InvalidOperationException($"The requested size {requestedSize} doesn't match the size {blockSize} of the pool buffer and cannot be taken from this pool.");
}
}
}
@@ -0,0 +1,117 @@
using Microsoft.Extensions.ObjectPool;
using System.Runtime.CompilerServices;
namespace Recyclable.Collections.Parallel
{
public sealed class BarrierSlimmer : IDisposable
{
private static readonly ObjectPool<BarrierSlimmer> _defaultPool = new DefaultObjectPool<BarrierSlimmer>(new DefaultPooledObjectPolicy<BarrierSlimmer>());
private ObjectPool<BarrierSlimmer>? _assignedPool;
private readonly SpinLock _lock = new(false);
public bool HasParticipants;
public int Participants;
[MethodImpl(MethodImplOptions.NoInlining)]
public void AddParticipant()
{
bool isLocked = false;
_lock.Enter(ref isLocked);
Participants++;
if (Participants == 1)
{
HasParticipants = true;
}
if (isLocked)
{
_lock.Exit(false);
}
}
[MethodImpl(MethodImplOptions.NoInlining)]
public void RemoveParticipant()
{
if (!HasParticipants)
{
ThrowNoMoreParticipants();
}
bool isLocked = false;
_lock.Enter(ref isLocked);
Participants--;
if (Participants == 0)
{
HasParticipants = false;
}
if (isLocked)
{
_lock.Exit(false);
}
}
private static void ThrowNoMoreParticipants()
{
throw new InvalidOperationException("There are no more participants. The operation is not allowed");
}
public BarrierSlimmer()
{
_lock = new(false);
}
public BarrierSlimmer(int participants)
{
_lock = new(false);
Participants = participants;
HasParticipants = participants > 0;
}
public static BarrierSlimmer Create(int participants)
{
var result = _defaultPool.Get();
result.Participants = participants;
result.HasParticipants = participants > 0;
return result;
}
public void Dispose()
{
var assignedPool = _assignedPool;
_assignedPool = null;
assignedPool?.Return(this);
GC.SuppressFinalize(this);
}
public void SignalAndWait()
{
RemoveParticipant();
//if (!HasParticipants)
//{
// return;
//}
//if (!HasParticipants)
//{
// return;
//}
//if (!HasParticipants)
//{
// return;
//}
// SpinWait.SpinUntil(() => HasParticipants);
var waitSpinner = new SpinWait();
while (HasParticipants)
{
waitSpinner.SpinOnce(0);
}
}
}
}
@@ -0,0 +1,37 @@
namespace Recyclable.Collections.Parallel
{
public sealed class ManualResetEventSlimmer
{
internal bool _isSet;
public bool IsSet => _isSet;
internal bool _returned;
public ManualResetEventSlimmer()
{
}
public ManualResetEventSlimmer(bool initialState)
{
_isSet = initialState;
}
public void Set()
{
_isSet = true;
}
public void Reset()
{
_isSet = false;
}
public void Dispose()
{
if (!_returned)
{
_returned = true;
ManualResetEventSlimmerPool.Shared.Return(this);
}
}
}
}
@@ -0,0 +1,29 @@
using Microsoft.Extensions.ObjectPool;
namespace Recyclable.Collections.Parallel
{
public static class ManualResetEventSlimmerPool
{
public static readonly SpinLock _updateLock = new(false);
public static readonly ObjectPool<ManualResetEventSlimmer> Shared = new DefaultObjectPool<ManualResetEventSlimmer>(new DefaultPooledObjectPolicy<ManualResetEventSlimmer>());
public static ManualResetEventSlimmer Create(bool initialState)
{
bool lockTaken = false;
_updateLock.Enter(ref lockTaken);
ManualResetEventSlimmer result = Shared.Get();
if (lockTaken)
{
_updateLock.Exit(false);
}
result._returned = false;
if (result._isSet != initialState)
{
result._isSet = initialState;
}
return result;
}
}
}
@@ -0,0 +1,38 @@
namespace Recyclable.Collections.Parallel
{
public sealed class ParallelSynchronizationContext : IDisposable
{
public readonly ManualResetEventSlimmer ItemFoundSignal;
public readonly Barrier AllDoneSignal;
public long FoundItemIndex;
public Exception? Exception;
internal bool _returned;
public ParallelSynchronizationContext()
{
ItemFoundSignal = ManualResetEventSlimmerPool.Create(false);
AllDoneSignal = new(0);
}
public ParallelSynchronizationContext(int participantCount)
{
ItemFoundSignal = ManualResetEventSlimmerPool.Create(false);
AllDoneSignal = new(participantCount);
}
public void Dispose()
{
if (!_returned)
{
_returned = true;
ParallelSynchronizationContextPool.Shared.Return(this);
}
else
{
ItemFoundSignal.Dispose();
AllDoneSignal.Dispose();
GC.SuppressFinalize(this);
}
}
}
}
@@ -0,0 +1,50 @@
using Microsoft.Extensions.ObjectPool;
namespace Recyclable.Collections.Parallel
{
public static class ParallelSynchronizationContextPool
{
private static readonly SpinLock _updateLock = new(false);
public static readonly ObjectPool<ParallelSynchronizationContext> Shared = new DefaultObjectPool<ParallelSynchronizationContext>(new DefaultPooledObjectPolicy<ParallelSynchronizationContext>());
public static ParallelSynchronizationContext GetWithOneParticipant()
{
bool lockTaken = false;
_updateLock.Enter(ref lockTaken);
var context = Shared.Get();
if (lockTaken)
{
_updateLock.Exit(false);
}
if (context.ItemFoundSignal._isSet)
{
context.ItemFoundSignal.Reset();
}
switch (context.AllDoneSignal.ParticipantsRemaining)
{
case > 1:
context.AllDoneSignal.RemoveParticipants(context.AllDoneSignal.ParticipantsRemaining - 1);
break;
case 0:
context.AllDoneSignal.AddParticipant();
break;
}
if (context.Exception != null)
{
context.Exception = null;
}
if (context.FoundItemIndex >= 0)
{
context.FoundItemIndex = -1L;
}
context._returned = false;
return context;
}
}
}
@@ -0,0 +1,27 @@
using Microsoft.Extensions.ObjectPool;
namespace Recyclable.Collections.Pools
{
public static class ItemRangePool
{
private static readonly SpinLock _updateLock = new(false);
public static readonly ObjectPool<ItemRange> Shared = new DefaultObjectPool<ItemRange>(new DefaultPooledObjectPolicy<ItemRange>());
public static ItemRange Create(int blockIndex, int startingItemIndex, long itemsToSearchCount)
{
bool lockTaken = false;
_updateLock.Enter(ref lockTaken);
var itemRange = Shared.Get();
if (lockTaken)
{
_updateLock.Exit(false);
}
itemRange._returned = false;
itemRange.BlockIndex = blockIndex;
itemRange.ItemsToSearchCount = itemsToSearchCount;
itemRange.StartingItemIndex = startingItemIndex;
return itemRange;
}
}
}
@@ -0,0 +1,86 @@
using System.Buffers;
namespace Recyclable.Collections.Pools
{
public class OneSizeArrayPool<T> : ArrayPool<T>
{
private readonly int _blockSize;
private readonly int _minBlockCount;
private int _blockCount;
private MemoryBucket<T>? _currentBucket;
private readonly SpinLock _updateLock = new(false);
public OneSizeArrayPool(int blockSize, int minBlockCount)
{
_blockSize = blockSize;
_minBlockCount = minBlockCount;
}
public override T[] Rent(int minimumLength)
{
if (minimumLength != _blockSize)
{
ThrowWrongRentedArraySizeException(minimumLength, _blockSize);
}
MemoryBucket<T>? bucket;
bool lockTaken = false;
_updateLock.Enter(ref lockTaken);
if (_currentBucket != null)
{
_blockCount--;
bucket = _currentBucket;
_currentBucket = _currentBucket.NextBucket;
}
else
{
bucket = null;
}
if (lockTaken)
{
_updateLock.Exit(false);
}
return bucket != null ? bucket.Memory : new T[_blockSize];
}
public override void Return(T[] array, bool clearArray = false)
{
if (array.Length != _blockSize)
{
ThrowWrongReturnedArraySizeException(array.Length, _blockSize);
}
MemoryBucket<T> bucket = new(array);
if (clearArray)
{
Array.Clear(array, 0, array.Length);
}
bool lockTaken = false;
_updateLock.Enter(ref lockTaken);
if (_blockCount < _minBlockCount)
{
bucket.NextBucket = _currentBucket;
_currentBucket = bucket;
_blockCount++;
}
if (lockTaken)
{
_updateLock.Exit(false);
}
}
private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize)
{
throw new InvalidOperationException($"The array size {returnedLength} doesn't match the size {blockSize} of the pool buffer and cannot be returned to this pool.");
}
private static void ThrowWrongRentedArraySizeException(int requestedSize, int blockSize)
{
throw new InvalidOperationException($"The requested size {requestedSize} doesn't match the size {blockSize} of the pool buffer and cannot be taken from this pool.");
}
}
}
@@ -0,0 +1,35 @@
using System.Buffers;
using System.Collections.Concurrent;
namespace Recyclable.Collections.Pools
{
public static class RecyclableArrayPool<T>
{
public static readonly OneSizeArrayPool<T> Null = new(0, 0);
private static readonly ConcurrentDictionary<int, ArrayPool<T>> _pools = new();
public static ArrayPool<T> Shared(int blockSize)
{
//lock (_lock)
{
if (!_pools.TryGetValue(blockSize, out ArrayPool<T>? pool))
{
pool = new OneSizeArrayPool<T>(blockSize, 100);
pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize];
}
return pool;
}
//_lock.Wait();
//try
//{
//}
//finally
//{
// _lock.Release();
//}
}
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
namespace Recyclable.Collections namespace Recyclable.Collections
{ {
public class RecyclableDefaults public static class RecyclableDefaults
{ {
public const int BlockSize = 16_384; public const int BlockSize = 16_384;
public const int Capacity = 1; public const int Capacity = 1;
+113 -55
View File
@@ -2,6 +2,10 @@
using System.Collections; using System.Collections;
using System.Numerics; using System.Numerics;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using Recyclable.Collections.Parallel;
using Recyclable.Collections.Pools;
[assembly:InternalsVisibleTo("Recyclable.Collections.Benchmarks")]
namespace Recyclable.Collections namespace Recyclable.Collections
{ {
@@ -9,16 +13,16 @@ namespace Recyclable.Collections
{ {
private const long ItemNotFoundIndexLong = -1L; private const long ItemNotFoundIndexLong = -1L;
private const int ItemNotFoundIndex = -1; private const int ItemNotFoundIndex = -1;
private const double OptimalParallelSearchStep = 0.329; private const double OptimalParallelSearchStep = 0.1;// 0.329;
private static readonly ArrayPool<T[]> _defaultMemoryBlocksPool = ArrayPool<T[]>.Create(); private static readonly ArrayPool<T[]> _defaultMemoryBlocksPool = ArrayPool<T[]>.Create();
private static readonly ArrayPool<T> _defaultBlockArrayPool = ArrayPool<T>.Create(); private static readonly ArrayPool<T> _defaultBlockArrayPool = ArrayPool<T>.Create();
private static readonly T[][] _emptyMemoryBlocksArray = new T[0][]; private static readonly T[][] _emptyMemoryBlocksArray = new T[0][];
private static readonly T[] _emptyBlockArray = new T[0]; private static readonly T[] _emptyBlockArray = new T[0];
private int _blockSize; internal int _blockSize;
private byte _blockSizePow2BitShift; internal byte _blockSizePow2BitShift;
private int _blockSizeMinus1; internal int _blockSizeMinus1;
private int _nextItemBlockIndex; private int _nextItemBlockIndex;
private int _nextItemIndex; private int _nextItemIndex;
private int _reservedBlockCount; private int _reservedBlockCount;
@@ -27,7 +31,7 @@ namespace Recyclable.Collections
protected ArrayPool<T[]> _memoryBlocksPool; protected ArrayPool<T[]> _memoryBlocksPool;
protected ArrayPool<T> _blockArrayPool; protected ArrayPool<T> _blockArrayPool;
protected T[][] _memoryBlocks; internal T[][] _memoryBlocks;
private long _capacity; private long _capacity;
public long Capacity public long Capacity
@@ -40,8 +44,8 @@ namespace Recyclable.Collections
public bool IsReadOnly { get; } public bool IsReadOnly { get; }
public int LastBlockWithData => _nextItemBlockIndex - (_nextItemIndex > 0 ? 0 : 1); public int LastBlockWithData => _nextItemBlockIndex - (_nextItemIndex > 0 ? 0 : 1);
protected long _longCountIndexOfStep; internal long _longCountIndexOfStep;
protected long _longCount; internal long _longCount;
public long LongCount public long LongCount
{ {
@@ -56,7 +60,6 @@ namespace Recyclable.Collections
public int ReservedBlockCount => _reservedBlockCount; public int ReservedBlockCount => _reservedBlockCount;
public int BlockSize => _blockSize; public int BlockSize => _blockSize;
public byte BlockSizePow2BitShift => _blockSizePow2BitShift; public byte BlockSizePow2BitShift => _blockSizePow2BitShift;
public int NextItemBlockIndex => _nextItemBlockIndex; public int NextItemBlockIndex => _nextItemBlockIndex;
public int NextItemIndex => _nextItemIndex; public int NextItemIndex => _nextItemIndex;
@@ -220,12 +223,13 @@ namespace Recyclable.Collections
_longCountIndexOfStep = Math.Max((long)(requiredCapacity * OptimalParallelSearchStep), 1L); _longCountIndexOfStep = Math.Max((long)(requiredCapacity * OptimalParallelSearchStep), 1L);
_nextItemBlockIndex = targetBlockIdx; _nextItemBlockIndex = targetBlockIdx;
_nextItemIndex = targetItemIdx; _nextItemIndex = targetItemIdx;
return;
} }
[MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] // & WAS SLOWER
private static long DoLongIndexOfParallel(in RecyclableList<T> list, in ItemRange itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) // [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)]
private static long DoLongIndexOfParallel(RecyclableList<T> list, in ItemRange itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal)
{ {
// & WAS SLOWER without
int blockSize = list._blockSize; int blockSize = list._blockSize;
int blockIndex = itemRange.BlockIndex; int blockIndex = itemRange.BlockIndex;
long itemsToSearchCount = itemRange.ItemsToSearchCount; long itemsToSearchCount = itemRange.ItemsToSearchCount;
@@ -267,75 +271,74 @@ namespace Recyclable.Collections
blockIndex++; blockIndex++;
} }
index = itemsToSearchCount <= 0 || blockIndex > list.LastBlockWithData index = itemsToSearchCount <= 0 //|| blockIndex > list.LastBlockWithData
? ItemNotFoundIndex ? ItemNotFoundIndex
: blockIndex == list._nextItemBlockIndex // : blockIndex == list._nextItemBlockIndex
? Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount) // ? Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount)
: Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount); : Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount);
//? Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(itemRange.List._nextItemIndex, itemsToSearchCount))) // ? Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(itemRange.List._nextItemIndex, itemsToSearchCount)))
//: Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(blockSize, itemsToSearchCount))); //: Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(blockSize, itemsToSearchCount)));
return !itemFoundSignal.IsSet && index >= 0 ? ((long)blockIndex << list._blockSizePow2BitShift) + index : ItemNotFoundIndexLong; return !itemFoundSignal.IsSet && index >= 0 ? ((long)blockIndex << list._blockSizePow2BitShift) + index : ItemNotFoundIndexLong;
} }
[MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private long DoIndexOfParallel(T item) private long DoIndexOfParallel(in T item)
{ {
using var itemFoundSignal = ManualResetEventSlimmer.Create(false); if (_longCount == 0)
using var allDoneSignal = new Barrier(1); {
long foundItemIndex = ItemNotFoundIndexLong; return ItemNotFoundIndex;
Exception? ex = null; }
using var context = ParallelSynchronizationContextPool.GetWithOneParticipant();
// (long)(_longCount * 0.329) => most efficient step, based on benchmarks // (long)(_longCount * 0.329) => most efficient step, based on benchmarks
ItemRangesIterator.Iterate(0, _blockSize, _blockSizePow2BitShift, _longCount, _longCountIndexOfStep, itemRange => Iterate(context, item);
context.AllDoneSignal.SignalAndWait();
if (context.Exception == null)
{ {
_ = allDoneSignal.AddParticipant(); return context.FoundItemIndex;
}
context.Exception.CaptureAndRethrow();
return ItemNotFoundIndexLong;
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
// [MethodImpl(MethodImplOptions.AggressiveInlining)]
// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.NoOptimization)]
private static bool ScheduleIndexOfTask(ParallelSynchronizationContext context, RecyclableList<T> list, T itemToFind, ItemRange itemRange)
{
_ = context.AllDoneSignal.AddParticipant();
// & WAS SLOWER
// _ = ThreadPool.QueueUserWorkItem<object?>((_) =>
_ = Task.Factory.StartNew(() => _ = Task.Factory.StartNew(() =>
{ {
try try
{ {
var index = DoLongIndexOfParallel(this, itemRange, item, itemFoundSignal); if (context.ItemFoundSignal.IsSet)
{
return;
}
var index = DoLongIndexOfParallel(list, itemRange, itemToFind, context.ItemFoundSignal);
if (index >= 0) if (index >= 0)
{ {
itemFoundSignal.Set(); context.ItemFoundSignal.Set();
foundItemIndex = index; _ = Interlocked.Exchange(ref context.FoundItemIndex, index);
} }
} }
catch (Exception e) catch (Exception e)
{ {
//_ = Interlocked.Exchange(ref ex, e); _ = Interlocked.Exchange(ref context.Exception, e);
lock (allDoneSignal)
{
ex = e;
}
} }
finally finally
{ {
context.AllDoneSignal.RemoveParticipant();
itemRange.Dispose(); itemRange.Dispose();
allDoneSignal.RemoveParticipant();
} }
}); });
//.ContinueWith(x =>
//{
// if (x.Exception != null)
// {
// lock (allDoneSignal)
// {
// ex = x.Exception;
// }
// }
//});
return !itemFoundSignal.IsSet; return !context.ItemFoundSignal.IsSet;
});
allDoneSignal.SignalAndWait();
if (ex == null)
{
return foundItemIndex;
}
ex.CaptureAndRethrow();
return ItemNotFoundIndexLong;
} }
[MethodImpl(MethodImplOptions.AggressiveOptimization)] [MethodImpl(MethodImplOptions.AggressiveOptimization)]
@@ -471,6 +474,63 @@ namespace Recyclable.Collections
return minBlockSize; return minBlockSize;
} }
// & WAS SLOWER
// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
// & WAS SLOWER
// [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)]
// & WAS SLOWER
// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.NoOptimization)]
// & WAS SLOWER
// [MethodImpl(MethodImplOptions.NoInlining)]
private void Iterate(in ParallelSynchronizationContext context, in T itemToFind)
{
// TODO: Switch to use _blockSize instead of list._longCountIndeexOfStep for simplifications & better performance.
// & WAS SLOWER without
ItemRange itemRange;
int blockIndex = 0;
// & WAS SLOWER
// int blockSize = _blockSize;
long itemIndex = 0;
long itemsCount = _longCount;
// & WAS SLOWER
// int blockSize = _blockSize;
// long step = list._longCountIndexOfStep;
// & WAS SLOWER
// while ((blockIndex << list._blockSizePow2BitShift) + itemIndex > step)
// & WAS SLOWER
// long longCountIndexOfStep = _longCountIndexOfStep;
while (itemsCount > _longCountIndexOfStep)
{
// At this point itemIndex is limited to blockSize range - int.
itemRange = ItemRangePool.Create(blockIndex, (int)itemIndex, _longCountIndexOfStep);
if (!ScheduleIndexOfTask(context, this, itemToFind, itemRange))
{
break;
}
itemsCount -= _longCountIndexOfStep;
// & WAS SLOWER
// blockIndex = (int)Math.DivRem(itemIndex + step, step, out itemIndex);
// & WAS SLOWER
// blockIndex += (int)((itemIndex + list._longCountIndexOfStep) >> list._blockSizePow2BitShift);
// itemIndex = (int)((itemIndex + list._longCountIndexOfStep) & list._blockSizeMinus1);
itemIndex += _longCountIndexOfStep;
while (itemIndex >= _blockSize)
{
blockIndex++;
itemIndex -= _blockSize;
}
}
// & WAS SLOWER
// itemRange = new(blockIndex, (int)itemIndex, list._longCount - (blockIndex << list._blockSizePow2BitShift) - itemIndex);
itemRange = ItemRangePool.Create(blockIndex, (int)itemIndex, itemsCount);
_ = ScheduleIndexOfTask(context, this, itemToFind, itemRange);
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
protected static void SetupBlockArrayPooling(RecyclableList<T> list, int blockSize, ArrayPool<T>? blockArrayPool = null) protected static void SetupBlockArrayPooling(RecyclableList<T> list, int blockSize, ArrayPool<T>? blockArrayPool = null)
{ {
@@ -951,9 +1011,7 @@ namespace Recyclable.Collections
public IEnumerator<T> GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); public IEnumerator<T> GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator();
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public int IndexOf(T item) => _longCount == 0 public int IndexOf(T item) => checked((int)DoIndexOfParallel(item));
? ItemNotFoundIndex
: checked((int)DoIndexOfParallel(item));
public void Insert(int index, T item) => throw new NotSupportedException(); public void Insert(int index, T item) => throw new NotSupportedException();
@@ -6,6 +6,8 @@ namespace Recyclable.Collections
public static class RecyclableListExtensions public static class RecyclableListExtensions
{ {
private const int _minPooledArraySize = RecyclableDefaults.MinPooledArrayLength; private const int _minPooledArraySize = RecyclableDefaults.MinPooledArrayLength;
private const long ItemNotFoundIndexLong = -1L;
private const int ItemNotFoundIndex = -1;
public static void CopyItems<T>(this T[] memory, long startingIndex, long count, ref T[] destArray, long offset = 1) public static void CopyItems<T>(this T[] memory, long startingIndex, long count, ref T[] destArray, long offset = 1)
{ {
@@ -0,0 +1,29 @@
namespace Recyclable.Collections
{
public readonly struct SearchConfiguration<TList, TItem>
{
public TItem ItemToFind { get; init; }
public TList List { get; init; }
public int BlockSizeMinus1 { get; init; }
public byte BlockSizePow2BitShift { get; init; }
public long ItemsCount { get; init; }
public int StartingBlockIndex { get; init; }
public long Step { get; init; }
// public SearchConfiguration(TList list, in TItem itemToFind)
// {
// ItemToFind = itemToFind;
// List = list;
// }
public static SearchConfiguration<RecyclableList<TItem>, TItem> CreateFrom(RecyclableList<TItem> list, in TItem itemToFind) => new()
{
List = list,
ItemToFind = itemToFind,
BlockSizeMinus1 = list._blockSizeMinus1,
BlockSizePow2BitShift = list._blockSizePow2BitShift,
ItemsCount = list._longCount,
Step = list._longCountIndexOfStep
};
}
}
@@ -1,5 +1,6 @@
using FluentAssertions; using FluentAssertions;
using Recyclable.Collections; using Recyclable.Collections;
using Recyclable.Collections.Parallel;
using System.Numerics; using System.Numerics;
namespace Recyclable.CollectionsTests namespace Recyclable.CollectionsTests
@@ -11,18 +12,31 @@ namespace Recyclable.CollectionsTests
public void IterateShouldYieldAllItems(int startingBlockIndex, int blockSize, long itemsCount, int step, IEnumerable<(int BlockIndex, int StartingItemIndex, long ItemsCount)> expected) public void IterateShouldYieldAllItems(int startingBlockIndex, int blockSize, long itemsCount, int step, IEnumerable<(int BlockIndex, int StartingItemIndex, long ItemsCount)> expected)
{ {
// Prepare // Prepare
var mockList = new RecyclableList<long>();
byte blockSizePow2BitShift = (byte)(31 - BitOperations.LeadingZeroCount((uint)blockSize)); byte blockSizePow2BitShift = (byte)(31 - BitOperations.LeadingZeroCount((uint)blockSize));
var actual = new List<(int BlockIndex, int StartingItemIndex, long ItemsCount)>();
var synchronizationContext = new ParallelSynchronizationContext(1);
var search = new SearchConfiguration<RecyclableList<long>, long>()
{
List = mockList,
ItemToFind = 0,
BlockSizeMinus1 = blockSize - 1,
BlockSizePow2BitShift = blockSizePow2BitShift,
ItemsCount = itemsCount,
Step = step,
};
var actualItemRanges = new List<(int BlockIndex, int StartingItemIndex, long ItemsCount)>();
// Act // Act
ItemRangesIterator.Iterate(startingBlockIndex, blockSize, blockSizePow2BitShift, itemsCount, step, (searchInfo) => ItemRangesIterator.Iterate(synchronizationContext, search, (context, search, itemRange) =>
{ {
actual.Add((searchInfo.BlockIndex, searchInfo.StartingItemIndex, searchInfo.ItemsToSearchCount)); actualItemRanges.Add((itemRange.BlockIndex, itemRange.StartingItemIndex, itemRange.ItemsToSearchCount));
return true; return true;
}); });
// Validate // Validate
_ = actual.Should().HaveCount(expected.Count()).And.ContainInConsecutiveOrder(expected); _ = actualItemRanges.Should().HaveCount(expected.Count()).And.ContainInConsecutiveOrder(expected);
} }
public static IEnumerable<object[]> TestCases => new[] public static IEnumerable<object[]> TestCases => new[]
@@ -2,7 +2,7 @@
namespace Recyclable.CollectionsTests namespace Recyclable.CollectionsTests
{ {
public class RecyclableListTestData public static class RecyclableListTestData
{ {
private static IEnumerable<long> DoCreateTestData(long itemsCount) private static IEnumerable<long> DoCreateTestData(long itemsCount)
{ {