From 3db10d760297b4620eae5fbab3d5241f01d20b01 Mon Sep 17 00:00:00 2001 From: mlemanczyk Date: Wed, 3 May 2023 12:36:37 +0200 Subject: [PATCH] RecyclableList.IndexOf parallel performance improvements. --- ...clableListPoc.Benchmarks.EnsureCapacity.cs | 1 + .../RecyclableListPoc.Benchmarks.IndexOf.cs | 3562 ++++++++++++++++- .../POC/RecyclableListPoc.Benchmarks.cs | 33 +- Recyclable.Collections/ItemRange.cs | 32 + Recyclable.Collections/ItemRangesIterator.cs | 236 +- Recyclable.Collections/MemoryBucket.cs | 22 + Recyclable.Collections/OneSizeArrayPool.cs | 179 - .../Parallel/BarrierSlimmer.cs | 117 + .../Parallel/ManualResetEventSlimmer.cs | 37 + .../Parallel/ManualResetEventSlimmerPool.cs | 29 + .../ParallelSynchronizationContext.cs | 38 + .../ParallelSynchronizationContextPool.cs | 50 + Recyclable.Collections/Pools/ItemRangePool.cs | 27 + .../Pools/OneSizeArrayPool.cs | 86 + .../Pools/RecyclableArrayPool.cs | 35 + Recyclable.Collections/RecyclableDefaults.cs | 2 +- Recyclable.Collections/RecyclableList.cs | 252 +- .../RecyclableListExtensions.cs | 2 + Recyclable.Collections/SearchConfiguration.cs | 29 + .../ItemRangesIteratorTests.cs | 22 +- .../RecyclableListTestData.cs | 2 +- 21 files changed, 4282 insertions(+), 511 deletions(-) create mode 100644 Recyclable.Collections/ItemRange.cs create mode 100644 Recyclable.Collections/MemoryBucket.cs delete mode 100644 Recyclable.Collections/OneSizeArrayPool.cs create mode 100644 Recyclable.Collections/Parallel/BarrierSlimmer.cs create mode 100644 Recyclable.Collections/Parallel/ManualResetEventSlimmer.cs create mode 100644 Recyclable.Collections/Parallel/ManualResetEventSlimmerPool.cs create mode 100644 Recyclable.Collections/Parallel/ParallelSynchronizationContext.cs create mode 100644 Recyclable.Collections/Parallel/ParallelSynchronizationContextPool.cs create mode 100644 Recyclable.Collections/Pools/ItemRangePool.cs create mode 100644 Recyclable.Collections/Pools/OneSizeArrayPool.cs create mode 100644 Recyclable.Collections/Pools/RecyclableArrayPool.cs create mode 100644 Recyclable.Collections/SearchConfiguration.cs diff --git a/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.EnsureCapacity.cs b/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.EnsureCapacity.cs index 182aa61..c8ff64a 100644 --- a/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.EnsureCapacity.cs +++ b/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.EnsureCapacity.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Attributes; +using Recyclable.Collections.Pools; using System.Buffers; using System.Collections; using System.Numerics; diff --git a/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.IndexOf.cs b/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.IndexOf.cs index 6741414..4a646f3 100644 --- a/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.IndexOf.cs +++ b/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.IndexOf.cs @@ -1,4 +1,7 @@ using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.ObjectPool; +using Recyclable.Collections.Parallel; +using Recyclable.Collections.Pools; using System.Buffers; using System.Collections; using System.Numerics; @@ -6,6 +9,94 @@ using System.Runtime.CompilerServices; namespace Recyclable.Collections.Benchmarks.POC { + #region ItemRangesIterator V1 + + internal sealed class ItemRangeV1 : IDisposable + { + public int BlockIndex; + public int StartingItemIndex; + public long ItemsToSearchCount; + internal ObjectPool? _assignedPool; + + public ItemRangeV1() + { + + } + + public ItemRangeV1(int blockIndex, int startingItemIndex, long itemsToSearchCount) + { + BlockIndex = blockIndex; + ItemsToSearchCount = itemsToSearchCount; + StartingItemIndex = startingItemIndex; + } + + public void Dispose() + { + if (_assignedPool != null) + { + var pool = _assignedPool; + _assignedPool = null; + pool!.Return(this); + } + + GC.SuppressFinalize(this); + } + } + + internal static class ItemRangePoolV1 + { + [ThreadStatic] + private static readonly ObjectPool _shared; + public static ObjectPool Shared => _shared; + + static ItemRangePoolV1() + { + _shared = new DefaultObjectPool(new DefaultPooledObjectPolicy()); + } + + public static ItemRangeV1 Create(int blockIndex, int startingItemIndex, long itemsToSearchCount) + { + var itemRange = _shared.Get(); + itemRange.BlockIndex = blockIndex; + itemRange.ItemsToSearchCount = itemsToSearchCount; + itemRange.StartingItemIndex = startingItemIndex; + return itemRange; + } + } + + internal static class ItemRangesIteratorV1 + { + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.NoInlining)] + public static void Iterate(int blockIndex, int blockSize, byte blockSizePow2BitShift, long itemsCount, long step, Func action) + { + int itemIndex = 0; + ItemRangeV1 itemRange; + while (itemsCount > step) + { + itemRange = ItemRangePoolV1.Create(blockIndex, itemIndex, step); + if (!action(itemRange)) + { + break; + } + + itemsCount -= step; + blockIndex += (int)((itemIndex + step) >> blockSizePow2BitShift); + itemIndex = (int)((itemIndex + step) & (blockSize - 1)); + //itemIndex += step; + //while (itemIndex >= blockSize) + //{ + // blockIndex++; + // itemIndex -= blockSize; + //} + } + + itemRange = ItemRangePoolV1.Create(blockIndex, itemIndex, itemsCount); + _ = action(itemRange); + } + } + + #endregion + #region RecyclableList IndexOf v1 internal class RecyclableListIndexOfV1 : IDisposable, IList @@ -226,7 +317,7 @@ namespace Recyclable.Collections.Benchmarks.POC } [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] - private static long DoLongIndexOfParallel(in RecyclableListIndexOfV1 list, in ItemRange itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) + private static long DoLongIndexOfParallel(in RecyclableListIndexOfV1 list, in ItemRangeV1 itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) { int blockSize = list._blockSize; int blockIndex = itemRange.BlockIndex; @@ -285,14 +376,15 @@ namespace Recyclable.Collections.Benchmarks.POC [MethodImpl(MethodImplOptions.AggressiveOptimization)] private long DoIndexOfParallel(T item) { - using var itemFoundSignal = ManualResetEventSlimmer.Create(false); + var itemFoundSignal = ManualResetEventSlimmerPool.Create(false); using var allDoneSignal = new Barrier(1); //BarrierSlimmer.Create(1); long foundItemIndex = -1L; Exception? ex = null; // (long)(_longCount * 0.329) => most efficient step, based on benchmarks - ItemRangesIterator.Iterate(0, _blockSize, _blockSizePow2BitShift, _longCount, _longCountIndexOfStep, LocalIndexOf); + ItemRangesIteratorV1.Iterate(0, _blockSize, _blockSizePow2BitShift, _longCount, _longCountIndexOfStep, LocalIndexOf); allDoneSignal.SignalAndWait(); + itemFoundSignal.Dispose(); if (ex == null) { return foundItemIndex; @@ -326,7 +418,7 @@ namespace Recyclable.Collections.Benchmarks.POC //_ = LocalIndexOf(new(blockIndex, itemIndex, itemsCount)); [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] - bool LocalIndexOf(ItemRange itemRange) + bool LocalIndexOf(ItemRangeV1 itemRange) { _ = allDoneSignal.AddParticipant(); _ = Task.Factory.StartNew(() => @@ -1152,9 +1244,3421 @@ namespace Recyclable.Collections.Benchmarks.POC #endregion + #region RecyclableList IndexOf v2 + + internal class RecyclableListIndexOfV2 : IDisposable, IList + { + private const int ItemNotFoundIndex = -1; + private const double OptimalParallelSearchStep = 0.329; + + private static readonly ArrayPool _defaultMemoryBlocksPool = ArrayPool.Create(); + private static readonly ArrayPool _defaultBlockArrayPool = ArrayPool.Create(); + private static readonly T[][] _emptyMemoryBlocksArray = new T[0][]; + private static readonly T[] _emptyBlockArray = new T[0]; + + private int _blockSize; + private byte _blockSizePow2BitShift; + private int _blockSizeMinus1; + private int _nextItemBlockIndex; + private int _nextItemIndex; + private int _reservedBlockCount; + + protected static readonly bool NeedsClearing = !typeof(T).IsValueType; + + protected ArrayPool _memoryBlocksPool; + protected ArrayPool _blockArrayPool; + protected T[][] _memoryBlocks; + + private long _capacity; + public long Capacity + { + get => _capacity; + protected set => _capacity = value; + } + + public int Count => checked((int)_longCount); + public bool IsReadOnly { get; } + public int LastBlockWithData => _nextItemBlockIndex - (_nextItemIndex > 0 ? 0 : 1); + + protected long _longCountIndexOfStep; + protected long _longCount; + + public long LongCount + { + get => _longCount; + set + { + _longCount = value; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + } + } + + public int ReservedBlockCount => _reservedBlockCount; + public int BlockSize => _blockSize; + public byte BlockSizePow2BitShift => _blockSizePow2BitShift; + + public int NextItemBlockIndex => _nextItemBlockIndex; + public int NextItemIndex => _nextItemIndex; + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static void CopyItems(RecyclableListIndexOfV2 list, long index) + { + T[][] memoryBlocks = list._memoryBlocks; + int nextItemIndex = list._nextItemIndex; + int blockSize = list._blockSize; + int sourceBlockIndex = checked((int)((index + 1) >> list._blockSizePow2BitShift)); + int sourceItemIndex = checked((int)((index + 1) & list._blockSizeMinus1)); + + int targetBlockIndex = checked((int)(index >> list._blockSizePow2BitShift)); + int targetItemIndex = checked((int)(index & list._blockSizeMinus1)); + + int lastTakenBlockIndex = list.LastBlockWithData; + while (sourceBlockIndex < lastTakenBlockIndex || (sourceBlockIndex == lastTakenBlockIndex && (sourceItemIndex < nextItemIndex || sourceBlockIndex != list._nextItemBlockIndex))) + { + int toCopy = sourceBlockIndex < lastTakenBlockIndex || nextItemIndex == 0 + ? blockSize - (sourceItemIndex >= targetItemIndex ? sourceItemIndex : targetItemIndex) + : Math.Min(nextItemIndex, blockSize - targetItemIndex); + + Array.Copy(memoryBlocks[sourceBlockIndex], sourceItemIndex, memoryBlocks[targetBlockIndex], targetItemIndex, toCopy); + + // We didn't have enough room in the target array block. There are still items in the source array block to copy. + if (sourceItemIndex + toCopy < blockSize) + { + sourceItemIndex += toCopy; + targetBlockIndex++; + targetItemIndex = 0; + } + // We copied all the source items in the current array block. But have we filled the target? + else + { + sourceItemIndex = 0; + sourceBlockIndex++; + if (targetItemIndex + toCopy < blockSize) + { + targetItemIndex += toCopy; + } + else + { + targetBlockIndex++; + targetItemIndex = 0; + } + } + } + } + + private static void ThrowArgumentOutOfRangeException(in string message) => throw new ArgumentOutOfRangeException("index", message); + + protected void AddRangeEnumerated(IEnumerable source, int growByCount) + { + long i, copied = _longCount; + int blockSize = _blockSize, targetItemIdx = _nextItemIndex, targetBlockIdx = _nextItemBlockIndex; + + using IEnumerator enumerator = source.GetEnumerator(); + if (!enumerator.MoveNext()) + { + return; + } + + long capacity = _capacity; + long available = capacity - copied; + + Span memoryBlocksSpan = new(_memoryBlocks, 0, ReservedBlockCount); + Span blockArraySpan; + var memoryBlocksCount = memoryBlocksSpan.Length; + while (true) + { + if (copied + growByCount > capacity) + { + capacity = EnsureCapacity(this, capacity + growByCount); + memoryBlocksCount = _reservedBlockCount; + if (blockSize != _blockSize) + { + blockSize = _blockSize; + } + + memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlocksCount); + available = capacity - copied; + } + + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + for (i = 1; i <= available; i++) + { + blockArraySpan[targetItemIdx++] = enumerator.Current; + + if (!enumerator.MoveNext()) + { + if (targetItemIdx < blockSize) + { + _nextItemBlockIndex = targetBlockIdx; + _nextItemIndex = targetItemIdx; + } + else + { + _nextItemBlockIndex = targetBlockIdx + 1; + _nextItemIndex = 0; + } + + _longCount += copied + i - _longCount; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + return; + } + + if (targetItemIdx == blockSize) + { + targetBlockIdx++; + targetItemIdx = 0; + + if (targetBlockIdx == memoryBlocksCount) + { + break; + } + + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + } + } + + copied += available; + } + } + + protected void AddRangeWithKnownCount(IEnumerable source, int requiredAdditionalCapacity) + { + long copied = _longCount; + int blockSize = _blockSize; + int targetItemIdx = _nextItemIndex; + int targetBlockIdx = _nextItemBlockIndex; + + Span memoryBlocksSpan; + Span blockArraySpan; + long requiredCapacity = copied + requiredAdditionalCapacity; + if (_capacity < requiredCapacity) + { + _ = EnsureCapacity(this, requiredCapacity); + blockSize = _blockSize; + } + + var memoryBlocksCount = ReservedBlockCount; + memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlocksCount); + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + foreach (var item in source) + { + blockArraySpan[targetItemIdx++] = item; + if (targetItemIdx == blockSize) + { + targetBlockIdx++; + targetItemIdx = 0; + if (targetBlockIdx == memoryBlocksCount) + { + break; + } + + blockArraySpan = new Span(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + } + } + + _longCount = requiredCapacity; + _longCountIndexOfStep = Math.Max((long)(requiredCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIdx; + _nextItemIndex = targetItemIdx; + return; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] + private static long DoLongIndexOfParallel(in RecyclableListIndexOfV2 list, in ItemRangeV1 itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) + { + int blockSize = list._blockSize; + int blockIndex = itemRange.BlockIndex; + long itemsToSearchCount = itemRange.ItemsToSearchCount; + int index = Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, itemRange.StartingItemIndex, (int)Math.Min(blockSize - itemRange.StartingItemIndex, itemsToSearchCount)); + if (itemFoundSignal.IsSet) + { + return ItemNotFoundIndex; + } + else if (index >= 0) + { + return (blockIndex << list._blockSizePow2BitShift) + index; + } + + // We don't need Math.Min here, because if itemsToSearchCount < blockSize - itemRange.StartingItemIndex, then it would mean that + // there are no more items and we already scanned all. We want to break the loop. Subtracting a higher value from itemsToSearchCount + // will always result in a negative result. + itemsToSearchCount -= blockSize - itemRange.StartingItemIndex; + if (itemsToSearchCount <= 0) + { + return ItemNotFoundIndex; + } + + //T itemToFind = itemRange.ItemToFind; + //int lastBlockWithData = itemRange.LastBlockWithData; + blockIndex++; + while (itemsToSearchCount > blockSize) + { + index = Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, blockSize); + if (itemFoundSignal.IsSet) + { + return ItemNotFoundIndex; + } + else if (index >= 0) + { + return ((long)blockIndex << list._blockSizePow2BitShift) + index; + } + + itemsToSearchCount -= blockSize; + blockIndex++; + } + + index = itemsToSearchCount <= 0 || blockIndex > list.LastBlockWithData + ? -1 + : blockIndex == list._nextItemBlockIndex + ? Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount) + : Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount); + //? Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(itemRange.List._nextItemIndex, itemsToSearchCount))) + //: Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(blockSize, itemsToSearchCount))); + + return index >= 0 ? ((long)blockIndex << list._blockSizePow2BitShift) + index : ItemNotFoundIndex; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private long DoIndexOfParallel(T item) + { + var itemFoundSignal = ManualResetEventSlimmerPool.Create(false); + using var allDoneSignal = new Barrier(1); + long foundItemIndex = -1L; + Exception? ex = null; + + // (long)(_longCount * 0.329) => most efficient step, based on benchmarks + ItemRangesIteratorV1.Iterate(0, _blockSize, _blockSizePow2BitShift, _longCount, _longCountIndexOfStep, LocalIndexOf); + allDoneSignal.SignalAndWait(); + itemFoundSignal.Dispose(); + if (ex == null) + { + return foundItemIndex; + } + + ex.CaptureAndRethrow(); + return ItemNotFoundIndex; + + //int blockIndex = 0; + //int itemIndex = 0; + //long itemsCount = _longCount; + //while (itemsCount > step) + //{ + // if (!LocalIndexOf(new(blockIndex, itemIndex, step))) + // { + // break; + // } + + // itemsCount -= step; + // //blockIndex = (int)Math.DivRem(itemIndex + step, blockSize, out itemIndex); + // blockIndex += (int)((itemIndex + step) >> _blockSizePow2BitShift); + // itemIndex = (int)((itemIndex + step) & _blockSizeMinus1); + // //itemIndex += step; + // //while (itemIndex >= blockSize) + // //{ + // // blockIndex++; + // // itemIndex -= blockSize; + // //} + //} + + //_ = LocalIndexOf(new(blockIndex, itemIndex, itemsCount)); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + bool LocalIndexOf(ItemRangeV1 itemRange) + { + _ = allDoneSignal.AddParticipant(); + _ = Task.Factory.StartNew(() => + { + try + { + var index = DoLongIndexOfParallel(this, itemRange, item, itemFoundSignal); + if (index >= 0) + { + itemFoundSignal.Set(); + foundItemIndex = index; + } + } + catch (Exception e) + { + //_ = Interlocked.Exchange(ref ex, e); + lock (allDoneSignal) + { + ex = e; + } + } + finally + { + itemRange.Dispose(); + allDoneSignal.RemoveParticipant(); + } + }); + //.ContinueWith(x => + //{ + // if (x.Exception != null) + // { + // lock (allDoneSignal) + // { + // ex = x.Exception; + // } + // } + //}); + + return !itemFoundSignal.IsSet; + } + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + protected static long DoIndexOf(T item, in T[][] memoryBlocks, int lastBlockIndex, int blockSize, int nextItemIndex) + { + Span memoryBlocksSpan = new(memoryBlocks, 0, lastBlockIndex + 1); + int itemIndex, blockIndex; + for (blockIndex = 0; blockIndex < lastBlockIndex; blockIndex++) + { + itemIndex = Array.IndexOf(memoryBlocksSpan[blockIndex], item, 0, blockSize); + if (itemIndex >= 0) + { + return itemIndex + ((long)blockIndex << (31 - BitOperations.LeadingZeroCount((uint)blockSize))); + } + } + + if (blockIndex == lastBlockIndex) + { + itemIndex = nextItemIndex != 0 + ? Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, nextItemIndex) + : Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, blockSize); + } + else + { + return ItemNotFoundIndex; + } + + return itemIndex >= 0 ? itemIndex + ((long)lastBlockIndex << (31 - BitOperations.LeadingZeroCount((uint)blockSize))) : ItemNotFoundIndex; + } + + /// + /// Creates a set of new memory buffers, if needed, to allow storing at minimum no. of items. + /// + /// that needs to be resized. + /// Minimal requested block size. It MUST be rounded to the power of 2, see remarks. + /// Pre-calculated bit shifting value for left & right shift operations against. + /// The minimum no. of items MUST be able to store after . + /// + /// For performance reasons, MUST a power of 2. This simplifies a lot block & item + /// index calculations, i.e. makes them logical operations on bits. + /// + /// This method checks for integral overflow. + /// + /// + /// This method doesn't support downsizing the memory block. As such it doesn't release excessive blocks. This is to for additional performance + /// gain - 1+ operation less for each call. If you need this functionality, you need to implement it on a higher level. + /// + /// The maximum no. of items can store. + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + protected static long Resize(RecyclableListIndexOfV2 list, int minBlockSize, byte minBlockSizePow2Shift, long newCapacity) + { + ArrayPool blockArrayPool = list._blockArrayPool; + int sourceBlockCount = list._reservedBlockCount; + int requiredBlockCount = checked((int)(newCapacity >> minBlockSizePow2Shift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0)); + int blockIndex; + + T[][] newMemoryBlocks; + if (requiredBlockCount > (list._memoryBlocks?.Length ?? 0)) + { + // Allocate new memory block for all arrays + newMemoryBlocks = requiredBlockCount >= RecyclableDefaults.MinPooledArrayLength ? list._memoryBlocksPool.Rent(requiredBlockCount) : new T[requiredBlockCount][]; + + // Copy arrays from the old memory block for all arrays + if (sourceBlockCount > 0) + { + Array.Copy(list._memoryBlocks!, newMemoryBlocks, sourceBlockCount); + // We can now return the old memory block for all arrays itself + if (sourceBlockCount >= RecyclableDefaults.MinPooledArrayLength) + { + list._memoryBlocksPool.Return(list._memoryBlocks!, true); + } + } + + list._memoryBlocks = newMemoryBlocks; + } + else + { + newMemoryBlocks = list._memoryBlocks!; + } + + // Allocate arrays for any new blocks + if (requiredBlockCount > sourceBlockCount) + { + if (minBlockSize >= RecyclableDefaults.MinPooledArrayLength) + { + if (sourceBlockCount == 0) + { + minBlockSize = GetEstimatedBlockSize(minBlockSize, blockArrayPool); + } + + blockIndex = sourceBlockCount; + while (true) + { + newMemoryBlocks[blockIndex] = blockArrayPool.Rent(minBlockSize); + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } + else + { + blockIndex = sourceBlockCount; + while (true) + { + newMemoryBlocks[blockIndex] = new T[minBlockSize]; + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } + } + + list._reservedBlockCount = requiredBlockCount; + return (long)requiredBlockCount << minBlockSizePow2Shift; + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static int GetEstimatedBlockSize(int minBlockSize, ArrayPool blockArrayPool) + { + var array = blockArrayPool.Rent(minBlockSize); + minBlockSize = array.Length; + blockArrayPool.Return(array); + return minBlockSize; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + protected static void SetupBlockArrayPooling(RecyclableListIndexOfV2 list, int blockSize, ArrayPool? blockArrayPool = null) + { + list._blockSize = blockSize; + list._blockSizePow2BitShift = (byte)(31 - BitOperations.LeadingZeroCount((uint)blockSize)); + list._blockArrayPool = blockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(blockSize) + : RecyclableArrayPool.Null; + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + protected static long EnsureCapacity(RecyclableListIndexOfV2 list, long requestedCapacity) + { + long newCapacity = list._capacity > 0 + ? checked((long)BitOperations.RoundUpToPowerOf2((ulong)requestedCapacity)) + : requestedCapacity; + + int blockSize = list._blockSize; + newCapacity = Resize(list, blockSize, list._blockSizePow2BitShift, newCapacity); + if (blockSize != list._memoryBlocks[0].Length && newCapacity > 0) + { + SetupBlockArrayPooling(list, list._memoryBlocks[0].Length); + list._blockSizeMinus1 = list._blockSize - 1; + } + + list._capacity = newCapacity; + return newCapacity; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] +#pragma warning disable CS8618 // It's set by SetupBlockArrayPooling + public RecyclableListIndexOfV2(int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) +#pragma warning restore CS8618 + { + _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; + + if (expectedItemsCount > 0) + { + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + SetupBlockArrayPooling(this, minBlockSize, blockArrayPool); + _capacity = Resize(this, minBlockSize, _blockSizePow2BitShift, expectedItemsCount.Value); + if (_blockSize != _memoryBlocks![0].Length && _capacity > 0) + { + SetupBlockArrayPooling(this, _memoryBlocks[0].Length, blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + } + else + { + _blockSizeMinus1 = minBlockSize - 1; + } + } + else + { + SetupBlockArrayPooling(this, checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)), blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + _memoryBlocks = _emptyMemoryBlocksArray; + } + } + +#pragma warning disable CS8618 // It's set by SetupBlockArrayPooling + public RecyclableListIndexOfV2(IEnumerable source, int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) +#pragma warning restore CS8618 + { + _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; + if (expectedItemsCount > 0) + { + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + SetupBlockArrayPooling(this, minBlockSize, blockArrayPool); + _capacity = Resize(this, minBlockSize, _blockSizePow2BitShift, expectedItemsCount.Value); + if (minBlockSize != _memoryBlocks![0].Length && _capacity > 0) + { + SetupBlockArrayPooling(this, _memoryBlocks[0].Length, blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + } + else + { + _blockSizeMinus1 = minBlockSize - 1; + } + } + else + { + SetupBlockArrayPooling(this, checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)), blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + _memoryBlocks = _emptyMemoryBlocksArray; + } + + AddRange(source); + } + + public T this[long index] + { + get => _memoryBlocks[index >> _blockSizePow2BitShift][index & _blockSizeMinus1]; + set => new Span(_memoryBlocks[checked((int)(index >> _blockSizePow2BitShift))])[checked((int)(index & _blockSizeMinus1))] = value; + } + + public T this[int index] + { + get => _memoryBlocks[index >> _blockSizePow2BitShift][index & _blockSizeMinus1]; + set => new Span(_memoryBlocks[index >> _blockSizePow2BitShift])[index & _blockSizeMinus1] = value; + } + + public T[][] AsArray { get => _memoryBlocks; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Add(T item) + { + if (_capacity < _longCount + 1) + { + _ = EnsureCapacity(this, _longCount + 1); + } + + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex++] = item; + if (_nextItemIndex == _blockSize) + { + _nextItemBlockIndex++; + _nextItemIndex = 0; + } + + _longCount++; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + } + + public void AddRange(in T[] items) + { + if (items.LongLength == 0) + { + return; + } + + long targetCapacity = _longCount + items.LongLength; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + Span itemsSpan = new(items); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + if (itemsSpan.Length > 0) + { + itemsSpan.CopyTo(targetBlockArraySpan); + } + + _nextItemIndex = itemsSpan.Length; + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + } + + public void AddRange(RecyclableArrayList items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + Span itemsSpan = new(items.AsArray, 0, items.Count); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + _nextItemIndex = itemsSpan.Length; + itemsSpan.CopyTo(targetBlockArraySpan); + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + } + + public void AddRange(RecyclableListIndexOfV2 items) + { + if (items.LongCount == 0) + { + return; + } + + long itemsCount = items.LongCount, + targetCapacity = _longCount + itemsCount; + + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + sourceBlockIndex = 0, + targetBlockIndex = _nextItemBlockIndex, + toCopy; + + Span sourceMemoryBlocksSpan = new(items._memoryBlocks, 0, items._reservedBlockCount); + Span itemsSpan = new(sourceMemoryBlocksSpan[sourceBlockIndex], 0, items._blockSize); + Span targetMemoryBlocksSpan = new(_memoryBlocks, 0, _reservedBlockCount); + Span targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + long copiedCount = 0; + while (copiedCount < itemsCount) + { + toCopy = checked((int)Math.Min(itemsCount - copiedCount, itemsSpan.Length)); + if (targetBlockArraySpan.Length < toCopy) + { + toCopy = targetBlockArraySpan.Length; + itemsSpan[..toCopy].CopyTo(targetBlockArraySpan); + targetBlockIndex++; + targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], 0, blockSize); + itemsSpan = itemsSpan[toCopy..]; + copiedCount += toCopy; + } + else + { + itemsSpan[..toCopy].CopyTo(targetBlockArraySpan); + targetBlockArraySpan = targetBlockArraySpan[toCopy..]; + itemsSpan = itemsSpan[toCopy..]; + if (itemsSpan.IsEmpty && sourceBlockIndex + 1 < sourceMemoryBlocksSpan.Length) + { + sourceBlockIndex++; + itemsSpan = new(sourceMemoryBlocksSpan[sourceBlockIndex], 0, items._blockSize); + } + + copiedCount += toCopy; + if (targetBlockArraySpan.IsEmpty && copiedCount < itemsCount) + { + targetBlockIndex++; + targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + } + } + + if ((targetCapacity & _blockSizeMinus1) == 0) + { + _nextItemIndex = 0; + _nextItemBlockIndex = targetBlockIndex + 1; + } + else + { + _nextItemIndex = checked((int)(targetCapacity & _blockSizeMinus1)); + _nextItemBlockIndex = targetBlockIndex; + } + + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + } + + public void AddRange(List items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + itemsCount = items.Count, + copied = Math.Min(blockSize - _nextItemIndex, itemsCount); + + Span memoryBlocksSpan = new(_memoryBlocks, 0, _reservedBlockCount); + items.CopyTo(0, memoryBlocksSpan[targetBlockIndex], _nextItemIndex, copied); + if (_nextItemIndex + copied == blockSize) + { + targetBlockIndex++; + _nextItemIndex = 0; + } + else + { + _nextItemIndex += copied; + } + + while (blockSize <= itemsCount - copied) + { + items.CopyTo(copied, memoryBlocksSpan[targetBlockIndex++], 0, blockSize); + copied += blockSize; + } + + if ((itemsCount - copied < blockSize) && (itemsCount - copied != 0)) + { + _nextItemIndex = itemsCount - copied; + items.CopyTo(copied, memoryBlocksSpan[targetBlockIndex], 0, _nextItemIndex); + } + + _nextItemBlockIndex = targetBlockIndex; + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + } + + public void AddRange(IList items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + // We're better off to temporarily copy it to a fixed array, + // than copying them item by item. We may run into OOMs, though. + T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _defaultBlockArrayPool.Rent(items.Count) : new T[items.Count]; + try + { + items.CopyTo(itemsBuffer, 0); + + Span itemsSpan = new(itemsBuffer, 0, items.Count); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + itemsSpan.CopyTo(targetBlockArraySpan); + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + _nextItemIndex = itemsSpan.Length; + } + finally + { + if (items.Count >= RecyclableDefaults.MinPooledArrayLength) + { + _defaultBlockArrayPool.Return(itemsBuffer, NeedsClearing); + } + } + } + + public void AddRange(IEnumerable source, int growByCount = RecyclableDefaults.BlockSize) + { + if (source is RecyclableListIndexOfV2 sourceRecyclableList) + { + AddRange(sourceRecyclableList); + return; + } + + if (source is RecyclableArrayList sourceRecyclableArrayList) + { + AddRange(sourceRecyclableArrayList); + return; + } + + if (source is T[] sourceArray) + { + AddRange(sourceArray); + return; + } + + if (source is List sourceList) + { + AddRange(sourceList); + return; + } + + if (source is IList sourceIList) + { + AddRange(sourceIList); + return; + } + + if (source.TryGetNonEnumeratedCount(out var requiredAdditionalCapacity)) + { + AddRangeWithKnownCount(source, requiredAdditionalCapacity); + return; + } + + AddRangeEnumerated(source, growByCount); + } + + public void Clear() + { + if (_longCount == 0) + { + return; + } + + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + T[][] memoryBlocks = _memoryBlocks; + int memoryBlocksCount = _reservedBlockCount; + ArrayPool blockArrayPool = _blockArrayPool; + int toRemoveIdx = 0; + while (toRemoveIdx < memoryBlocksCount) + { + blockArrayPool.Return(memoryBlocks[toRemoveIdx++], NeedsClearing); + } + } + + Array.Fill(_memoryBlocks, _emptyBlockArray, 0, _reservedBlockCount); + _capacity = 0; + _reservedBlockCount = 0; + _nextItemBlockIndex = 0; + _nextItemIndex = 0; + _longCount = 0; + _longCountIndexOfStep = 1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Contains(T item) + { + if (_longCount == 0) + { + return false; + } + + if (_nextItemBlockIndex == 0 || (_nextItemBlockIndex == 1 && _nextItemIndex == 0)) + { + return Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) >= 0; + } + + Span memoryBlocksSpan = new(_memoryBlocks); + int lastBlockIndex = _nextItemBlockIndex; + for (int blockIndex = 0; blockIndex < lastBlockIndex; blockIndex++) + { + if (Array.IndexOf(memoryBlocksSpan[blockIndex], item, 0, _blockSize) >= 0) + { + return true; + } + } + + return lastBlockIndex < memoryBlocksSpan.Length && Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, _nextItemIndex) >= 0; + } + + public void CopyTo(T[] array, int arrayIndex) => _memoryBlocks.CopyTo(0, _blockSize, _longCount, array, arrayIndex); + + public IEnumerator GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int IndexOf(T item) => _longCount == 0 + ? ItemNotFoundIndex + //// Optimization for tiny collections to avoid any vars etc. + : _longCount <= _blockSize + ? Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) + : checked((int)DoIndexOfParallel(item)); + + public void Insert(int index, T item) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public long LongIndexOf(T item) => _longCount == 0 + ? ItemNotFoundIndex + : _longCount <= _blockSize + ? Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) + : DoIndexOfParallel(item); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public bool Remove(T item) + { + var index = LongIndexOf(item); + if (index >= 0) + { + _longCount--; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + + return true; + } + + return false; + } + + public void RemoveBlock(int index) + { + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + _blockArrayPool.Return(_memoryBlocks[index], NeedsClearing); + _memoryBlocks[index] = _emptyBlockArray; + } + else + { + _memoryBlocks[index] = _emptyBlockArray; + } + + _capacity -= _blockSize; + if (_nextItemBlockIndex == index) + { + _nextItemIndex = 0; + } + else if (_nextItemBlockIndex > index) + { + _nextItemBlockIndex--; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public void RemoveAt(int index) + { + if (index >= _longCount || index < 0) + { + ThrowArgumentOutOfRangeException($"Argument \"{nameof(index)}\" = {index} is out of range. Expected value between 0 and {_longCount - 1}"); + } + + _longCount--; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public void RemoveAt(long index) + { + if (index >= _longCount || index < 0) + { + ThrowArgumentOutOfRangeException($"Argument \"{nameof(index)}\" = {index} is out of range. Expected value between 0 and {_longCount - 1}"); + } + + _longCount--; + _longCountIndexOfStep = Math.Max((long)Math.Floor(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + } + + IEnumerator IEnumerable.GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); + + ~RecyclableListIndexOfV2() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + if (_capacity > 0) + { + Clear(); + if (_memoryBlocks.Length >= RecyclableDefaults.MinPooledArrayLength) + { + _memoryBlocksPool.Return(_memoryBlocks, NeedsClearing); + } + + GC.SuppressFinalize(this); + } + } + } + + #endregion + + #region RecyclableList IndexOf v3 + + internal class RecyclableListIndexOfV3 : IDisposable, IList + { + private const long ItemNotFoundIndexLong = -1L; + private const int ItemNotFoundIndex = -1; + private const double OptimalParallelSearchStep = 0.329; + + private static readonly ArrayPool _defaultMemoryBlocksPool = ArrayPool.Create(); + private static readonly ArrayPool _defaultBlockArrayPool = ArrayPool.Create(); + private static readonly T[][] _emptyMemoryBlocksArray = new T[0][]; + private static readonly T[] _emptyBlockArray = new T[0]; + + private int _blockSize; + private byte _blockSizePow2BitShift; + private int _blockSizeMinus1; + private int _nextItemBlockIndex; + private int _nextItemIndex; + private int _reservedBlockCount; + + protected static readonly bool NeedsClearing = !typeof(T).IsValueType; + + protected ArrayPool _memoryBlocksPool; + protected ArrayPool _blockArrayPool; + protected T[][] _memoryBlocks; + + private long _capacity; + public long Capacity + { + get => _capacity; + protected set => _capacity = value; + } + + public int Count => checked((int)_longCount); + public bool IsReadOnly { get; } + public int LastBlockWithData => _nextItemBlockIndex - (_nextItemIndex > 0 ? 0 : 1); + + protected long _longCountIndexOfStep; + protected long _longCount; + + public long LongCount + { + get => _longCount; + set + { + _longCount = value; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + } + } + + public int ReservedBlockCount => _reservedBlockCount; + public int BlockSize => _blockSize; + public byte BlockSizePow2BitShift => _blockSizePow2BitShift; + + public int NextItemBlockIndex => _nextItemBlockIndex; + public int NextItemIndex => _nextItemIndex; + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static void CopyItems(RecyclableListIndexOfV3 list, long index) + { + T[][] memoryBlocks = list._memoryBlocks; + int nextItemIndex = list._nextItemIndex; + int blockSize = list._blockSize; + int sourceBlockIndex = (int)((index + 1) >> list._blockSizePow2BitShift); + int sourceItemIndex = (int)((index + 1) & list._blockSizeMinus1); + + int targetBlockIndex = (int)(index >> list._blockSizePow2BitShift); + int targetItemIndex = (int)(index & list._blockSizeMinus1); + + int lastTakenBlockIndex = list.LastBlockWithData; + while (sourceBlockIndex < lastTakenBlockIndex || (sourceBlockIndex == lastTakenBlockIndex && (sourceItemIndex < nextItemIndex || sourceBlockIndex != list._nextItemBlockIndex))) + { + int toCopy = sourceBlockIndex < lastTakenBlockIndex || nextItemIndex == 0 + ? blockSize - (sourceItemIndex >= targetItemIndex ? sourceItemIndex : targetItemIndex) + : Math.Min(nextItemIndex, blockSize - targetItemIndex); + + Array.Copy(memoryBlocks[sourceBlockIndex], sourceItemIndex, memoryBlocks[targetBlockIndex], targetItemIndex, toCopy); + + // We didn't have enough room in the target array block. There are still items in the source array block to copy. + if (sourceItemIndex + toCopy < blockSize) + { + sourceItemIndex += toCopy; + targetBlockIndex++; + targetItemIndex = 0; + } + // We copied all the source items in the current array block. But have we filled the target? + else + { + sourceItemIndex = 0; + sourceBlockIndex++; + if (targetItemIndex + toCopy < blockSize) + { + targetItemIndex += toCopy; + } + else + { + targetBlockIndex++; + targetItemIndex = 0; + } + } + } + } + + private static void ThrowArgumentOutOfRangeException(in string message) => throw new ArgumentOutOfRangeException("index", message); + + protected void AddRangeEnumerated(IEnumerable source, int growByCount) + { + long i, copied = _longCount; + int blockSize = _blockSize, targetItemIdx = _nextItemIndex, targetBlockIdx = _nextItemBlockIndex; + + using IEnumerator enumerator = source.GetEnumerator(); + if (!enumerator.MoveNext()) + { + return; + } + + long capacity = _capacity; + long available = capacity - copied; + + Span memoryBlocksSpan = new(_memoryBlocks, 0, ReservedBlockCount); + Span blockArraySpan; + var memoryBlocksCount = memoryBlocksSpan.Length; + while (true) + { + if (copied + growByCount > capacity) + { + capacity = EnsureCapacity(this, capacity + growByCount); + memoryBlocksCount = _reservedBlockCount; + if (blockSize != _blockSize) + { + blockSize = _blockSize; + } + + memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlocksCount); + available = capacity - copied; + } + + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + for (i = 1; i <= available; i++) + { + blockArraySpan[targetItemIdx++] = enumerator.Current; + + if (!enumerator.MoveNext()) + { + if (targetItemIdx < blockSize) + { + _nextItemBlockIndex = targetBlockIdx; + _nextItemIndex = targetItemIdx; + } + else + { + _nextItemBlockIndex = targetBlockIdx + 1; + _nextItemIndex = 0; + } + + _longCount += copied + i - _longCount; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + return; + } + + if (targetItemIdx == blockSize) + { + targetBlockIdx++; + targetItemIdx = 0; + + if (targetBlockIdx == memoryBlocksCount) + { + break; + } + + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + } + } + + copied += available; + } + } + + protected void AddRangeWithKnownCount(IEnumerable source, int requiredAdditionalCapacity) + { + long copied = _longCount; + int blockSize = _blockSize; + int targetItemIdx = _nextItemIndex; + int targetBlockIdx = _nextItemBlockIndex; + + Span memoryBlocksSpan; + Span blockArraySpan; + long requiredCapacity = copied + requiredAdditionalCapacity; + if (_capacity < requiredCapacity) + { + _ = EnsureCapacity(this, requiredCapacity); + blockSize = _blockSize; + } + + var memoryBlocksCount = ReservedBlockCount; + memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlocksCount); + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + foreach (var item in source) + { + blockArraySpan[targetItemIdx++] = item; + if (targetItemIdx == blockSize) + { + targetBlockIdx++; + targetItemIdx = 0; + if (targetBlockIdx == memoryBlocksCount) + { + break; + } + + blockArraySpan = new Span(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + } + } + + _longCount = requiredCapacity; + _longCountIndexOfStep = Math.Max((long)(requiredCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIdx; + _nextItemIndex = targetItemIdx; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] + private static long DoLongIndexOfParallel(in RecyclableListIndexOfV3 list, in ItemRangeV1 itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) + { + int blockSize = list._blockSize; + int blockIndex = itemRange.BlockIndex; + long itemsToSearchCount = itemRange.ItemsToSearchCount; + int index = Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, itemRange.StartingItemIndex, (int)Math.Min(blockSize - itemRange.StartingItemIndex, itemsToSearchCount)); + if (itemFoundSignal.IsSet) + { + return ItemNotFoundIndexLong; + } + else if (index >= 0) + { + return (blockIndex << list._blockSizePow2BitShift) + index; + } + + // We don't need Math.Min here, because if itemsToSearchCount < blockSize - itemRange.StartingItemIndex, then it would mean that + // there are no more items and we already scanned all. We want to break the loop. Subtracting a higher value from itemsToSearchCount + // will always result in a negative result. + itemsToSearchCount -= blockSize - itemRange.StartingItemIndex; + if (itemsToSearchCount <= 0) + { + return ItemNotFoundIndexLong; + } + + blockIndex++; + while (itemsToSearchCount > blockSize) + { + index = Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, blockSize); + if (itemFoundSignal.IsSet) + { + return ItemNotFoundIndexLong; + } + else if (index >= 0) + { + return ((long)blockIndex << list._blockSizePow2BitShift) + index; + } + + itemsToSearchCount -= blockSize; + blockIndex++; + } + + index = itemsToSearchCount <= 0 || blockIndex > list.LastBlockWithData + ? ItemNotFoundIndex + : blockIndex == list._nextItemBlockIndex + ? Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount) + : Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount); + + return !itemFoundSignal.IsSet && index >= 0 ? ((long)blockIndex << list._blockSizePow2BitShift) + index : ItemNotFoundIndexLong; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] + private long DoIndexOfParallel(T item) + { + var itemFoundSignal = ManualResetEventSlimmerPool.Create(false); + using var allDoneSignal = new Barrier(1); + long foundItemIndex = ItemNotFoundIndexLong; + Exception? ex = null; + + // (long)(_longCount * 0.329) => most efficient step, based on benchmarks + ItemRangesIteratorV1.Iterate(0, _blockSize, _blockSizePow2BitShift, _longCount, _longCountIndexOfStep, LocalIndexOf); + allDoneSignal.SignalAndWait(); + itemFoundSignal.Dispose(); + if (ex == null) + { + return foundItemIndex; + } + + ex.CaptureAndRethrow(); + return ItemNotFoundIndexLong; + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + bool LocalIndexOf(ItemRangeV1 itemRange) + { + _ = allDoneSignal.AddParticipant(); + _ = Task.Factory.StartNew(() => + { + try + { + var index = DoLongIndexOfParallel(this, itemRange, item, itemFoundSignal); + if (index >= 0) + { + itemFoundSignal.Set(); + foundItemIndex = index; + } + } + catch (Exception e) + { + //_ = Interlocked.Exchange(ref ex, e); + lock (allDoneSignal) + { + ex = e; + } + } + finally + { + itemRange.Dispose(); + allDoneSignal.RemoveParticipant(); + } + }); + //.ContinueWith(x => + //{ + // if (x.Exception != null) + // { + // lock (allDoneSignal) + // { + // ex = x.Exception; + // } + // } + //}); + + return !itemFoundSignal.IsSet; + } + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + protected static long DoIndexOf(T item, in T[][] memoryBlocks, int lastBlockIndex, int blockSize, int nextItemIndex) + { + Span memoryBlocksSpan = new(memoryBlocks, 0, lastBlockIndex + 1); + int itemIndex, blockIndex; + for (blockIndex = 0; blockIndex < lastBlockIndex; blockIndex++) + { + itemIndex = Array.IndexOf(memoryBlocksSpan[blockIndex], item, 0, blockSize); + if (itemIndex >= 0) + { + return itemIndex + ((long)blockIndex << (31 - BitOperations.LeadingZeroCount((uint)blockSize))); + } + } + + if (blockIndex == lastBlockIndex) + { + itemIndex = nextItemIndex != 0 + ? Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, nextItemIndex) + : Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, blockSize); + } + else + { + return ItemNotFoundIndexLong; + } + + return itemIndex >= 0 ? itemIndex + ((long)lastBlockIndex << (31 - BitOperations.LeadingZeroCount((uint)blockSize))) : ItemNotFoundIndexLong; + } + + /// + /// Creates a set of new memory buffers, if needed, to allow storing at minimum no. of items. + /// + /// that needs to be resized. + /// Minimal requested block size. It MUST be rounded to the power of 2, see remarks. + /// Pre-calculated bit shifting value for left & right shift operations against. + /// The minimum no. of items MUST be able to store after . + /// + /// For performance reasons, MUST a power of 2. This simplifies a lot block & item + /// index calculations, i.e. makes them logical operations on bits. + /// + /// This method checks for integral overflow. + /// + /// + /// This method doesn't support downsizing the memory block. As such it doesn't release excessive blocks. This is to for additional performance + /// gain - 1+ operation less for each call. If you need this functionality, you need to implement it on a higher level. + /// + /// The maximum no. of items can store. + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + protected static long Resize(RecyclableListIndexOfV3 list, int minBlockSize, byte minBlockSizePow2Shift, long newCapacity) + { + ArrayPool blockArrayPool = list._blockArrayPool; + int sourceBlockCount = list._reservedBlockCount; + int requiredBlockCount = checked((int)(newCapacity >> minBlockSizePow2Shift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0)); + int blockIndex; + + T[][] newMemoryBlocks; + if (requiredBlockCount > (list._memoryBlocks?.Length ?? 0)) + { + // Allocate new memory block for all arrays + newMemoryBlocks = requiredBlockCount >= RecyclableDefaults.MinPooledArrayLength ? list._memoryBlocksPool.Rent(requiredBlockCount) : new T[requiredBlockCount][]; + + // Copy arrays from the old memory block for all arrays + if (sourceBlockCount > 0) + { + Array.Copy(list._memoryBlocks!, newMemoryBlocks, sourceBlockCount); + // We can now return the old memory block for all arrays itself + if (sourceBlockCount >= RecyclableDefaults.MinPooledArrayLength) + { + list._memoryBlocksPool.Return(list._memoryBlocks!, true); + } + } + + list._memoryBlocks = newMemoryBlocks; + } + else + { + newMemoryBlocks = list._memoryBlocks!; + } + + // Allocate arrays for any new blocks + if (requiredBlockCount > sourceBlockCount) + { + if (minBlockSize >= RecyclableDefaults.MinPooledArrayLength) + { + if (sourceBlockCount == 0) + { + minBlockSize = GetEstimatedBlockSize(minBlockSize, blockArrayPool); + } + + blockIndex = sourceBlockCount; + while (true) + { + newMemoryBlocks[blockIndex] = blockArrayPool.Rent(minBlockSize); + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } + else + { + blockIndex = sourceBlockCount; + while (true) + { + newMemoryBlocks[blockIndex] = new T[minBlockSize]; + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } + } + + list._reservedBlockCount = requiredBlockCount; + return (long)requiredBlockCount << minBlockSizePow2Shift; + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static int GetEstimatedBlockSize(int minBlockSize, ArrayPool blockArrayPool) + { + var array = blockArrayPool.Rent(minBlockSize); + minBlockSize = array.Length; + blockArrayPool.Return(array); + return minBlockSize; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + protected static void SetupBlockArrayPooling(RecyclableListIndexOfV3 list, int blockSize, ArrayPool? blockArrayPool = null) + { + list._blockSize = blockSize; + list._blockSizePow2BitShift = (byte)(31 - BitOperations.LeadingZeroCount((uint)blockSize)); + list._blockArrayPool = blockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(blockSize) + : RecyclableArrayPool.Null; + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + protected static long EnsureCapacity(RecyclableListIndexOfV3 list, long requestedCapacity) + { + long newCapacity = list._capacity > 0 + ? checked((long)BitOperations.RoundUpToPowerOf2((ulong)requestedCapacity)) + : requestedCapacity; + + int blockSize = list._blockSize; + newCapacity = Resize(list, blockSize, list._blockSizePow2BitShift, newCapacity); + if (blockSize != list._memoryBlocks[0].Length && newCapacity > 0) + { + SetupBlockArrayPooling(list, list._memoryBlocks[0].Length); + list._blockSizeMinus1 = list._blockSize - 1; + } + + list._capacity = newCapacity; + return newCapacity; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] +#pragma warning disable CS8618 // It's set by SetupBlockArrayPooling + public RecyclableListIndexOfV3(int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) +#pragma warning restore CS8618 + { + _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; + + if (expectedItemsCount > 0) + { + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + SetupBlockArrayPooling(this, minBlockSize, blockArrayPool); + _capacity = Resize(this, minBlockSize, _blockSizePow2BitShift, expectedItemsCount.Value); + if (_blockSize != _memoryBlocks![0].Length && _capacity > 0) + { + SetupBlockArrayPooling(this, _memoryBlocks[0].Length, blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + } + else + { + _blockSizeMinus1 = minBlockSize - 1; + } + } + else + { + SetupBlockArrayPooling(this, checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)), blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + _memoryBlocks = _emptyMemoryBlocksArray; + } + } + +#pragma warning disable CS8618 // It's set by SetupBlockArrayPooling + public RecyclableListIndexOfV3(IEnumerable source, int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) +#pragma warning restore CS8618 + { + _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; + if (expectedItemsCount > 0) + { + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + SetupBlockArrayPooling(this, minBlockSize, blockArrayPool); + _capacity = Resize(this, minBlockSize, _blockSizePow2BitShift, expectedItemsCount.Value); + if (minBlockSize != _memoryBlocks![0].Length && _capacity > 0) + { + SetupBlockArrayPooling(this, _memoryBlocks[0].Length, blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + } + else + { + _blockSizeMinus1 = minBlockSize - 1; + } + } + else + { + SetupBlockArrayPooling(this, checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)), blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + _memoryBlocks = _emptyMemoryBlocksArray; + } + + AddRange(source); + } + + public T this[long index] + { + get => _memoryBlocks[index >> _blockSizePow2BitShift][index & _blockSizeMinus1]; + set => new Span(_memoryBlocks[checked((int)(index >> _blockSizePow2BitShift))])[checked((int)(index & _blockSizeMinus1))] = value; + } + + public T this[int index] + { + get => _memoryBlocks[index >> _blockSizePow2BitShift][index & _blockSizeMinus1]; + set => new Span(_memoryBlocks[index >> _blockSizePow2BitShift])[index & _blockSizeMinus1] = value; + } + + public T[][] AsArray { get => _memoryBlocks; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Add(T item) + { + if (_capacity < _longCount + 1) + { + _ = EnsureCapacity(this, _longCount + 1); + } + + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex++] = item; + if (_nextItemIndex == _blockSize) + { + _nextItemBlockIndex++; + _nextItemIndex = 0; + } + + _longCount++; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + } + + public void AddRange(in T[] items) + { + if (items.LongLength == 0) + { + return; + } + + long targetCapacity = _longCount + items.LongLength; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + Span itemsSpan = new(items); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + if (itemsSpan.Length > 0) + { + itemsSpan.CopyTo(targetBlockArraySpan); + } + + _nextItemIndex = itemsSpan.Length; + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + } + + public void AddRange(RecyclableArrayList items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + Span itemsSpan = new(items.AsArray, 0, items.Count); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + _nextItemIndex = itemsSpan.Length; + itemsSpan.CopyTo(targetBlockArraySpan); + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + } + + public void AddRange(RecyclableListIndexOfV3 items) + { + if (items.LongCount == 0) + { + return; + } + + long itemsCount = items.LongCount, + targetCapacity = _longCount + itemsCount; + + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + sourceBlockIndex = 0, + targetBlockIndex = _nextItemBlockIndex, + toCopy; + + Span sourceMemoryBlocksSpan = new(items._memoryBlocks, 0, items._reservedBlockCount); + Span itemsSpan = new(sourceMemoryBlocksSpan[sourceBlockIndex], 0, items._blockSize); + Span targetMemoryBlocksSpan = new(_memoryBlocks, 0, _reservedBlockCount); + Span targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + long copiedCount = 0; + while (copiedCount < itemsCount) + { + toCopy = checked((int)Math.Min(itemsCount - copiedCount, itemsSpan.Length)); + if (targetBlockArraySpan.Length < toCopy) + { + toCopy = targetBlockArraySpan.Length; + itemsSpan[..toCopy].CopyTo(targetBlockArraySpan); + targetBlockIndex++; + targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], 0, blockSize); + itemsSpan = itemsSpan[toCopy..]; + copiedCount += toCopy; + } + else + { + itemsSpan[..toCopy].CopyTo(targetBlockArraySpan); + targetBlockArraySpan = targetBlockArraySpan[toCopy..]; + itemsSpan = itemsSpan[toCopy..]; + if (itemsSpan.IsEmpty && sourceBlockIndex + 1 < sourceMemoryBlocksSpan.Length) + { + sourceBlockIndex++; + itemsSpan = new(sourceMemoryBlocksSpan[sourceBlockIndex], 0, items._blockSize); + } + + copiedCount += toCopy; + if (targetBlockArraySpan.IsEmpty && copiedCount < itemsCount) + { + targetBlockIndex++; + targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + } + } + + if ((targetCapacity & _blockSizeMinus1) == 0) + { + _nextItemIndex = 0; + _nextItemBlockIndex = targetBlockIndex + 1; + } + else + { + _nextItemIndex = checked((int)(targetCapacity & _blockSizeMinus1)); + _nextItemBlockIndex = targetBlockIndex; + } + + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + } + + public void AddRange(List items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + itemsCount = items.Count, + copied = Math.Min(blockSize - _nextItemIndex, itemsCount); + + Span memoryBlocksSpan = new(_memoryBlocks, 0, _reservedBlockCount); + items.CopyTo(0, memoryBlocksSpan[targetBlockIndex], _nextItemIndex, copied); + if (_nextItemIndex + copied == blockSize) + { + targetBlockIndex++; + _nextItemIndex = 0; + } + else + { + _nextItemIndex += copied; + } + + while (blockSize <= itemsCount - copied) + { + items.CopyTo(copied, memoryBlocksSpan[targetBlockIndex++], 0, blockSize); + copied += blockSize; + } + + if ((itemsCount - copied < blockSize) && (itemsCount - copied != 0)) + { + _nextItemIndex = itemsCount - copied; + items.CopyTo(copied, memoryBlocksSpan[targetBlockIndex], 0, _nextItemIndex); + } + + _nextItemBlockIndex = targetBlockIndex; + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + } + + public void AddRange(IList items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + // We're better off to temporarily copy it to a fixed array, + // than copying them item by item. We may run into OOMs, though. + T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _defaultBlockArrayPool.Rent(items.Count) : new T[items.Count]; + try + { + items.CopyTo(itemsBuffer, 0); + + Span itemsSpan = new(itemsBuffer, 0, items.Count); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + itemsSpan.CopyTo(targetBlockArraySpan); + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + _nextItemIndex = itemsSpan.Length; + } + finally + { + if (items.Count >= RecyclableDefaults.MinPooledArrayLength) + { + _defaultBlockArrayPool.Return(itemsBuffer, NeedsClearing); + } + } + } + + public void AddRange(IEnumerable source, int growByCount = RecyclableDefaults.BlockSize) + { + if (source is RecyclableList sourceRecyclableList) + { + AddRange(sourceRecyclableList); + return; + } + + if (source is RecyclableArrayList sourceRecyclableArrayList) + { + AddRange(sourceRecyclableArrayList); + return; + } + + if (source is T[] sourceArray) + { + AddRange(sourceArray); + return; + } + + if (source is List sourceList) + { + AddRange(sourceList); + return; + } + + if (source is IList sourceIList) + { + AddRange(sourceIList); + return; + } + + if (source.TryGetNonEnumeratedCount(out var requiredAdditionalCapacity)) + { + AddRangeWithKnownCount(source, requiredAdditionalCapacity); + return; + } + + AddRangeEnumerated(source, growByCount); + } + + public void Clear() + { + if (_longCount == 0) + { + return; + } + + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + T[][] memoryBlocks = _memoryBlocks; + int memoryBlocksCount = _reservedBlockCount; + ArrayPool blockArrayPool = _blockArrayPool; + int toRemoveIdx = 0; + while (toRemoveIdx < memoryBlocksCount) + { + blockArrayPool.Return(memoryBlocks[toRemoveIdx++], NeedsClearing); + } + } + + Array.Fill(_memoryBlocks, _emptyBlockArray, 0, _reservedBlockCount); + _capacity = 0; + _reservedBlockCount = 0; + _nextItemBlockIndex = 0; + _nextItemIndex = 0; + _longCount = 0; + _longCountIndexOfStep = 1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Contains(T item) + { + if (_longCount == 0) + { + return false; + } + + if (_nextItemBlockIndex == 0 || (_nextItemBlockIndex == 1 && _nextItemIndex == 0)) + { + return Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) >= 0; + } + + Span memoryBlocksSpan = new(_memoryBlocks); + int lastBlockIndex = _nextItemBlockIndex; + for (int blockIndex = 0; blockIndex < lastBlockIndex; blockIndex++) + { + if (Array.IndexOf(memoryBlocksSpan[blockIndex], item, 0, _blockSize) >= 0) + { + return true; + } + } + + return lastBlockIndex < memoryBlocksSpan.Length && Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, _nextItemIndex) >= 0; + } + + public void CopyTo(T[] array, int arrayIndex) => _memoryBlocks.CopyTo(0, _blockSize, _longCount, array, arrayIndex); + + public IEnumerator GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int IndexOf(T item) => _longCount == 0 + ? ItemNotFoundIndex + : checked((int)DoIndexOfParallel(item)); + + public void Insert(int index, T item) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public long LongIndexOf(T item) => _longCount == 0 + ? ItemNotFoundIndexLong + : _longCount <= _blockSize + ? Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) + : DoIndexOfParallel(item); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public bool Remove(T item) + { + var index = LongIndexOf(item); + if (index >= 0) + { + _longCount--; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + + return true; + } + + return false; + } + + public void RemoveBlock(int index) + { + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + _blockArrayPool.Return(_memoryBlocks[index], NeedsClearing); + _memoryBlocks[index] = _emptyBlockArray; + } + else + { + _memoryBlocks[index] = _emptyBlockArray; + } + + _capacity -= _blockSize; + if (_nextItemBlockIndex == index) + { + _nextItemIndex = 0; + } + else if (_nextItemBlockIndex > index) + { + _nextItemBlockIndex--; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public void RemoveAt(int index) + { + if (index >= _longCount || index < 0) + { + ThrowArgumentOutOfRangeException($"Argument \"{nameof(index)}\" = {index} is out of range. Expected value between 0 and {_longCount - 1}"); + } + + _longCount--; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public void RemoveAt(long index) + { + if (index >= _longCount || index < 0) + { + ThrowArgumentOutOfRangeException($"Argument \"{nameof(index)}\" = {index} is out of range. Expected value between 0 and {_longCount - 1}"); + } + + _longCount--; + _longCountIndexOfStep = Math.Max((long)Math.Floor(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + } + + IEnumerator IEnumerable.GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); + + ~RecyclableListIndexOfV3() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + if (_capacity > 0) + { + Clear(); + if (_memoryBlocks.Length >= RecyclableDefaults.MinPooledArrayLength) + { + _memoryBlocksPool.Return(_memoryBlocks, NeedsClearing); + } + + GC.SuppressFinalize(this); + } + } + } + + #endregion + + #region RecyclableList IndexOf v4 + + internal static class RecyclableListIndexOfV4Extensions + { + private const long ItemNotFoundIndexLong = -1L; + private const int ItemNotFoundIndex = -1; + + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] + public static long DoLongIndexOfParallel(this RecyclableListIndexOfV4 list, in ItemRangeV1 itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) + { + int blockSize = list._blockSize; + int blockIndex = itemRange.BlockIndex; + long itemsToSearchCount = itemRange.ItemsToSearchCount; + int index = Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, itemRange.StartingItemIndex, (int)Math.Min(blockSize - itemRange.StartingItemIndex, itemsToSearchCount)); + if (itemFoundSignal.IsSet) + { + return ItemNotFoundIndexLong; + } + else if (index >= 0) + { + return (blockIndex << list._blockSizePow2BitShift) + index; + } + + // We don't need Math.Min here, because if itemsToSearchCount < blockSize - itemRange.StartingItemIndex, then it would mean that + // there are no more items and we already scanned all. We want to break the loop. Subtracting a higher value from itemsToSearchCount + // will always result in a negative result. + itemsToSearchCount -= blockSize - itemRange.StartingItemIndex; + if (itemsToSearchCount <= 0) + { + return ItemNotFoundIndexLong; + } + + //T itemToFind = itemRange.ItemToFind; + //int lastBlockWithData = itemRange.LastBlockWithData; + blockIndex++; + while (itemsToSearchCount > blockSize) + { + index = Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, blockSize); + if (itemFoundSignal.IsSet) + { + return ItemNotFoundIndexLong; + } + else if (index >= 0) + { + return ((long)blockIndex << list._blockSizePow2BitShift) + index; + } + + itemsToSearchCount -= blockSize; + blockIndex++; + } + + index = itemsToSearchCount <= 0 + ? ItemNotFoundIndex + : Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount); + + return !itemFoundSignal.IsSet && index >= 0 ? ((long)blockIndex << list._blockSizePow2BitShift) + index : ItemNotFoundIndexLong; + } + } + + internal sealed class ParallelSynchronizationContextV4 : IDisposable + { + public readonly ManualResetEventSlimmer ItemFoundSignal; + public readonly Barrier AllDoneSignal; + public long FoundItemIndex; + public Exception? Exception; + + public ParallelSynchronizationContextV4(int participantCount) + { + ItemFoundSignal = ManualResetEventSlimmerPool.Create(false); + AllDoneSignal = new(participantCount); + } + + public void Dispose() + { + ItemFoundSignal.Dispose(); + AllDoneSignal.Dispose(); + + GC.SuppressFinalize(this); + } + } + + internal class RecyclableListIndexOfV4 : IDisposable, IList + { + private const long ItemNotFoundIndexLong = -1L; + private const int ItemNotFoundIndex = -1; + private const double OptimalParallelSearchStep = 0.329; + + private static readonly ArrayPool _defaultMemoryBlocksPool = ArrayPool.Create(); + private static readonly ArrayPool _defaultBlockArrayPool = ArrayPool.Create(); + private static readonly T[][] _emptyMemoryBlocksArray = new T[0][]; + private static readonly T[] _emptyBlockArray = new T[0]; + + internal int _blockSize; + internal byte _blockSizePow2BitShift; + internal int _blockSizeMinus1; + private int _nextItemBlockIndex; + private int _nextItemIndex; + private int _reservedBlockCount; + + protected static readonly bool NeedsClearing = !typeof(T).IsValueType; + + protected ArrayPool _memoryBlocksPool; + protected ArrayPool _blockArrayPool; + internal T[][] _memoryBlocks; + + private long _capacity; + public long Capacity + { + get => _capacity; + protected set => _capacity = value; + } + + public int Count => checked((int)_longCount); + public bool IsReadOnly { get; } + public int LastBlockWithData => _nextItemBlockIndex - (_nextItemIndex > 0 ? 0 : 1); + + internal long _longCountIndexOfStep; + internal long _longCount; + + public long LongCount + { + get => _longCount; + set + { + _longCount = value; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + } + } + + public int ReservedBlockCount => _reservedBlockCount; + public int BlockSize => _blockSize; + public byte BlockSizePow2BitShift => _blockSizePow2BitShift; + + public int NextItemBlockIndex => _nextItemBlockIndex; + public int NextItemIndex => _nextItemIndex; + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static void CopyItems(RecyclableListIndexOfV4 list, long index) + { + T[][] memoryBlocks = list._memoryBlocks; + int nextItemIndex = list._nextItemIndex; + int blockSize = list._blockSize; + int sourceBlockIndex = (int)((index + 1) >> list._blockSizePow2BitShift); + int sourceItemIndex = (int)((index + 1) & list._blockSizeMinus1); + + int targetBlockIndex = (int)(index >> list._blockSizePow2BitShift); + int targetItemIndex = (int)(index & list._blockSizeMinus1); + + int lastTakenBlockIndex = list.LastBlockWithData; + while (sourceBlockIndex < lastTakenBlockIndex || (sourceBlockIndex == lastTakenBlockIndex && (sourceItemIndex < nextItemIndex || sourceBlockIndex != list._nextItemBlockIndex))) + { + int toCopy = sourceBlockIndex < lastTakenBlockIndex || nextItemIndex == 0 + ? blockSize - (sourceItemIndex >= targetItemIndex ? sourceItemIndex : targetItemIndex) + : Math.Min(nextItemIndex, blockSize - targetItemIndex); + + Array.Copy(memoryBlocks[sourceBlockIndex], sourceItemIndex, memoryBlocks[targetBlockIndex], targetItemIndex, toCopy); + + // We didn't have enough room in the target array block. There are still items in the source array block to copy. + if (sourceItemIndex + toCopy < blockSize) + { + sourceItemIndex += toCopy; + targetBlockIndex++; + targetItemIndex = 0; + } + // We copied all the source items in the current array block. But have we filled the target? + else + { + sourceItemIndex = 0; + sourceBlockIndex++; + if (targetItemIndex + toCopy < blockSize) + { + targetItemIndex += toCopy; + } + else + { + targetBlockIndex++; + targetItemIndex = 0; + } + } + } + } + + private static void ThrowArgumentOutOfRangeException(in string message) => throw new ArgumentOutOfRangeException("index", message); + + protected void AddRangeEnumerated(IEnumerable source, int growByCount) + { + long i, copied = _longCount; + int blockSize = _blockSize, targetItemIdx = _nextItemIndex, targetBlockIdx = _nextItemBlockIndex; + + using IEnumerator enumerator = source.GetEnumerator(); + if (!enumerator.MoveNext()) + { + return; + } + + long capacity = _capacity; + long available = capacity - copied; + + Span memoryBlocksSpan = new(_memoryBlocks, 0, ReservedBlockCount); + Span blockArraySpan; + var memoryBlocksCount = memoryBlocksSpan.Length; + while (true) + { + if (copied + growByCount > capacity) + { + capacity = EnsureCapacity(this, capacity + growByCount); + memoryBlocksCount = _reservedBlockCount; + if (blockSize != _blockSize) + { + blockSize = _blockSize; + } + + memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlocksCount); + available = capacity - copied; + } + + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + for (i = 1; i <= available; i++) + { + blockArraySpan[targetItemIdx++] = enumerator.Current; + + if (!enumerator.MoveNext()) + { + if (targetItemIdx < blockSize) + { + _nextItemBlockIndex = targetBlockIdx; + _nextItemIndex = targetItemIdx; + } + else + { + _nextItemBlockIndex = targetBlockIdx + 1; + _nextItemIndex = 0; + } + + _longCount += copied + i - _longCount; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + return; + } + + if (targetItemIdx == blockSize) + { + targetBlockIdx++; + targetItemIdx = 0; + + if (targetBlockIdx == memoryBlocksCount) + { + break; + } + + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + } + } + + copied += available; + } + } + + protected void AddRangeWithKnownCount(IEnumerable source, int requiredAdditionalCapacity) + { + long copied = _longCount; + int blockSize = _blockSize; + int targetItemIdx = _nextItemIndex; + int targetBlockIdx = _nextItemBlockIndex; + + Span memoryBlocksSpan; + Span blockArraySpan; + long requiredCapacity = copied + requiredAdditionalCapacity; + if (_capacity < requiredCapacity) + { + _ = EnsureCapacity(this, requiredCapacity); + blockSize = _blockSize; + } + + var memoryBlocksCount = ReservedBlockCount; + memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlocksCount); + blockArraySpan = new(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + foreach (var item in source) + { + blockArraySpan[targetItemIdx++] = item; + if (targetItemIdx == blockSize) + { + targetBlockIdx++; + targetItemIdx = 0; + if (targetBlockIdx == memoryBlocksCount) + { + break; + } + + blockArraySpan = new Span(memoryBlocksSpan[targetBlockIdx], 0, blockSize); + } + } + + _longCount = requiredCapacity; + _longCountIndexOfStep = Math.Max((long)(requiredCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIdx; + _nextItemIndex = targetItemIdx; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] + private long DoIndexOfParallel(T item) + { + var synchronizationContext = new ParallelSynchronizationContextV4(1); + // (long)(_longCount * 0.329) => most efficient step, based on benchmarks + Iterate(synchronizationContext, this, item, ScheduleIndexOfTasks); + + synchronizationContext.AllDoneSignal.SignalAndWait(); + if (synchronizationContext.Exception == null) + { + return synchronizationContext.FoundItemIndex; + } + + synchronizationContext.Exception.CaptureAndRethrow(); + return ItemNotFoundIndexLong; + } + + private static bool ScheduleIndexOfTasks(ParallelSynchronizationContextV4 context, RecyclableListIndexOfV4 list, T itemToFind, ItemRangeV1 itemRange) + { + _ = context.AllDoneSignal.AddParticipant(); + _ = Task.Factory.StartNew(() => + { + try + { + var index = list.DoLongIndexOfParallel(itemRange, itemToFind, context.ItemFoundSignal); + if (index >= 0) + { + context.ItemFoundSignal.Set(); + _ = Interlocked.Exchange(ref context.FoundItemIndex, index); + } + } + catch (Exception e) + { + _ = Interlocked.Exchange(ref context.Exception, e); + } + finally + { + context.AllDoneSignal.RemoveParticipant(); + } + }); + + return !context.ItemFoundSignal.IsSet; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.NoInlining)] + public static void Iterate(in ParallelSynchronizationContextV4 context, RecyclableListIndexOfV4 list, in T itemToFind, Func, T, ItemRangeV1, bool> action) + { + int itemIndex = 0; + ItemRangeV1 itemRange; + int blockIndex = 0; + long itemsCount = list._longCount; + long step = list._longCountIndexOfStep; + while (itemsCount > step) + { + itemRange = new(blockIndex, itemIndex, step); + if (!action(context, list, itemToFind, itemRange)) + { + break; + } + + itemsCount -= step; + //blockIndex = (int)Math.DivRem(itemIndex + step, blockSize, out itemIndex); + blockIndex += (int)((itemIndex + step) >> list._blockSizePow2BitShift); + itemIndex = (int)((itemIndex + step) & list._blockSizeMinus1); + //itemIndex += step; + //while (itemIndex >= blockSize) + //{ + // blockIndex++; + // itemIndex -= blockSize; + //} + } + + itemRange = new(blockIndex, itemIndex, itemsCount); + _ = action(context, list, itemToFind, itemRange); + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + protected static long DoIndexOf(T item, in T[][] memoryBlocks, int lastBlockIndex, int blockSize, int nextItemIndex) + { + Span memoryBlocksSpan = new(memoryBlocks, 0, lastBlockIndex + 1); + int itemIndex, blockIndex; + for (blockIndex = 0; blockIndex < lastBlockIndex; blockIndex++) + { + itemIndex = Array.IndexOf(memoryBlocksSpan[blockIndex], item, 0, blockSize); + if (itemIndex >= 0) + { + return itemIndex + ((long)blockIndex << (31 - BitOperations.LeadingZeroCount((uint)blockSize))); + } + } + + if (blockIndex == lastBlockIndex) + { + itemIndex = nextItemIndex != 0 + ? Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, nextItemIndex) + : Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, blockSize); + } + else + { + return ItemNotFoundIndexLong; + } + + return itemIndex >= 0 ? itemIndex + ((long)lastBlockIndex << (31 - BitOperations.LeadingZeroCount((uint)blockSize))) : ItemNotFoundIndexLong; + } + + /// + /// Creates a set of new memory buffers, if needed, to allow storing at minimum no. of items. + /// + /// that needs to be resized. + /// Minimal requested block size. It MUST be rounded to the power of 2, see remarks. + /// Pre-calculated bit shifting value for left & right shift operations against. + /// The minimum no. of items MUST be able to store after . + /// + /// For performance reasons, MUST a power of 2. This simplifies a lot block & item + /// index calculations, i.e. makes them logical operations on bits. + /// + /// This method checks for integral overflow. + /// + /// + /// This method doesn't support downsizing the memory block. As such it doesn't release excessive blocks. This is to for additional performance + /// gain - 1+ operation less for each call. If you need this functionality, you need to implement it on a higher level. + /// + /// The maximum no. of items can store. + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + protected static long Resize(RecyclableListIndexOfV4 list, int minBlockSize, byte minBlockSizePow2Shift, long newCapacity) + { + ArrayPool blockArrayPool = list._blockArrayPool; + int sourceBlockCount = list._reservedBlockCount; + int requiredBlockCount = checked((int)(newCapacity >> minBlockSizePow2Shift) + ((newCapacity & (minBlockSize - 1)) > 0 ? 1 : 0)); + int blockIndex; + + T[][] newMemoryBlocks; + if (requiredBlockCount > (list._memoryBlocks?.Length ?? 0)) + { + // Allocate new memory block for all arrays + newMemoryBlocks = requiredBlockCount >= RecyclableDefaults.MinPooledArrayLength ? list._memoryBlocksPool.Rent(requiredBlockCount) : new T[requiredBlockCount][]; + + // Copy arrays from the old memory block for all arrays + if (sourceBlockCount > 0) + { + Array.Copy(list._memoryBlocks!, newMemoryBlocks, sourceBlockCount); + // We can now return the old memory block for all arrays itself + if (sourceBlockCount >= RecyclableDefaults.MinPooledArrayLength) + { + list._memoryBlocksPool.Return(list._memoryBlocks!, true); + } + } + + list._memoryBlocks = newMemoryBlocks; + } + else + { + newMemoryBlocks = list._memoryBlocks!; + } + + // Allocate arrays for any new blocks + if (requiredBlockCount > sourceBlockCount) + { + if (minBlockSize >= RecyclableDefaults.MinPooledArrayLength) + { + if (sourceBlockCount == 0) + { + minBlockSize = GetEstimatedBlockSize(minBlockSize, blockArrayPool); + } + + blockIndex = sourceBlockCount; + while (true) + { + newMemoryBlocks[blockIndex] = blockArrayPool.Rent(minBlockSize); + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } + else + { + blockIndex = sourceBlockCount; + while (true) + { + newMemoryBlocks[blockIndex] = new T[minBlockSize]; + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } + } + + list._reservedBlockCount = requiredBlockCount; + return (long)requiredBlockCount << minBlockSizePow2Shift; + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static int GetEstimatedBlockSize(int minBlockSize, ArrayPool blockArrayPool) + { + var array = blockArrayPool.Rent(minBlockSize); + minBlockSize = array.Length; + blockArrayPool.Return(array); + return minBlockSize; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + protected static void SetupBlockArrayPooling(RecyclableListIndexOfV4 list, int blockSize, ArrayPool? blockArrayPool = null) + { + list._blockSize = blockSize; + list._blockSizePow2BitShift = (byte)(31 - BitOperations.LeadingZeroCount((uint)blockSize)); + list._blockArrayPool = blockSize >= RecyclableDefaults.MinPooledArrayLength + ? blockArrayPool ?? RecyclableArrayPool.Shared(blockSize) + : RecyclableArrayPool.Null; + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + protected static long EnsureCapacity(RecyclableListIndexOfV4 list, long requestedCapacity) + { + long newCapacity = list._capacity > 0 + ? checked((long)BitOperations.RoundUpToPowerOf2((ulong)requestedCapacity)) + : requestedCapacity; + + int blockSize = list._blockSize; + newCapacity = Resize(list, blockSize, list._blockSizePow2BitShift, newCapacity); + if (blockSize != list._memoryBlocks[0].Length && newCapacity > 0) + { + SetupBlockArrayPooling(list, list._memoryBlocks[0].Length); + list._blockSizeMinus1 = list._blockSize - 1; + } + + list._capacity = newCapacity; + return newCapacity; + } + + [MethodImpl(MethodImplOptions.AggressiveOptimization)] +#pragma warning disable CS8618 // It's set by SetupBlockArrayPooling + public RecyclableListIndexOfV4(int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) +#pragma warning restore CS8618 + { + _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; + + if (expectedItemsCount > 0) + { + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + SetupBlockArrayPooling(this, minBlockSize, blockArrayPool); + _capacity = Resize(this, minBlockSize, _blockSizePow2BitShift, expectedItemsCount.Value); + if (_blockSize != _memoryBlocks![0].Length && _capacity > 0) + { + SetupBlockArrayPooling(this, _memoryBlocks[0].Length, blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + } + else + { + _blockSizeMinus1 = minBlockSize - 1; + } + } + else + { + SetupBlockArrayPooling(this, checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)), blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + _memoryBlocks = _emptyMemoryBlocksArray; + } + } + +#pragma warning disable CS8618 // It's set by SetupBlockArrayPooling + public RecyclableListIndexOfV4(IEnumerable source, int minBlockSize = RecyclableDefaults.BlockSize, long? expectedItemsCount = default, ArrayPool? memoryBlocksPool = default, ArrayPool? blockArrayPool = default) +#pragma warning restore CS8618 + { + _memoryBlocksPool = memoryBlocksPool ?? _defaultMemoryBlocksPool; + if (expectedItemsCount > 0) + { + minBlockSize = checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)); + SetupBlockArrayPooling(this, minBlockSize, blockArrayPool); + _capacity = Resize(this, minBlockSize, _blockSizePow2BitShift, expectedItemsCount.Value); + if (minBlockSize != _memoryBlocks![0].Length && _capacity > 0) + { + SetupBlockArrayPooling(this, _memoryBlocks[0].Length, blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + } + else + { + _blockSizeMinus1 = minBlockSize - 1; + } + } + else + { + SetupBlockArrayPooling(this, checked((int)BitOperations.RoundUpToPowerOf2((uint)minBlockSize)), blockArrayPool); + _blockSizeMinus1 = _blockSize - 1; + _memoryBlocks = _emptyMemoryBlocksArray; + } + + AddRange(source); + } + + public T this[long index] + { + get => _memoryBlocks[index >> _blockSizePow2BitShift][index & _blockSizeMinus1]; + set => new Span(_memoryBlocks[checked((int)(index >> _blockSizePow2BitShift))])[checked((int)(index & _blockSizeMinus1))] = value; + } + + public T this[int index] + { + get => _memoryBlocks[index >> _blockSizePow2BitShift][index & _blockSizeMinus1]; + set => new Span(_memoryBlocks[index >> _blockSizePow2BitShift])[index & _blockSizeMinus1] = value; + } + + public T[][] AsArray { get => _memoryBlocks; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Add(T item) + { + if (_capacity < _longCount + 1) + { + _ = EnsureCapacity(this, _longCount + 1); + } + + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex++] = item; + if (_nextItemIndex == _blockSize) + { + _nextItemBlockIndex++; + _nextItemIndex = 0; + } + + _longCount++; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + } + + public void AddRange(in T[] items) + { + if (items.LongLength == 0) + { + return; + } + + long targetCapacity = _longCount + items.LongLength; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + Span itemsSpan = new(items); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + if (itemsSpan.Length > 0) + { + itemsSpan.CopyTo(targetBlockArraySpan); + } + + _nextItemIndex = itemsSpan.Length; + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + } + + public void AddRange(RecyclableArrayList items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + Span itemsSpan = new(items.AsArray, 0, items.Count); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + _nextItemIndex = itemsSpan.Length; + itemsSpan.CopyTo(targetBlockArraySpan); + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + } + + public void AddRange(RecyclableListIndexOfV4 items) + { + if (items.LongCount == 0) + { + return; + } + + long itemsCount = items.LongCount, + targetCapacity = _longCount + itemsCount; + + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + sourceBlockIndex = 0, + targetBlockIndex = _nextItemBlockIndex, + toCopy; + + Span sourceMemoryBlocksSpan = new(items._memoryBlocks, 0, items._reservedBlockCount); + Span itemsSpan = new(sourceMemoryBlocksSpan[sourceBlockIndex], 0, items._blockSize); + Span targetMemoryBlocksSpan = new(_memoryBlocks, 0, _reservedBlockCount); + Span targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + long copiedCount = 0; + while (copiedCount < itemsCount) + { + toCopy = checked((int)Math.Min(itemsCount - copiedCount, itemsSpan.Length)); + if (targetBlockArraySpan.Length < toCopy) + { + toCopy = targetBlockArraySpan.Length; + itemsSpan[..toCopy].CopyTo(targetBlockArraySpan); + targetBlockIndex++; + targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], 0, blockSize); + itemsSpan = itemsSpan[toCopy..]; + copiedCount += toCopy; + } + else + { + itemsSpan[..toCopy].CopyTo(targetBlockArraySpan); + targetBlockArraySpan = targetBlockArraySpan[toCopy..]; + itemsSpan = itemsSpan[toCopy..]; + if (itemsSpan.IsEmpty && sourceBlockIndex + 1 < sourceMemoryBlocksSpan.Length) + { + sourceBlockIndex++; + itemsSpan = new(sourceMemoryBlocksSpan[sourceBlockIndex], 0, items._blockSize); + } + + copiedCount += toCopy; + if (targetBlockArraySpan.IsEmpty && copiedCount < itemsCount) + { + targetBlockIndex++; + targetBlockArraySpan = new(targetMemoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + } + } + + if ((targetCapacity & _blockSizeMinus1) == 0) + { + _nextItemIndex = 0; + _nextItemBlockIndex = targetBlockIndex + 1; + } + else + { + _nextItemIndex = checked((int)(targetCapacity & _blockSizeMinus1)); + _nextItemBlockIndex = targetBlockIndex; + } + + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + } + + public void AddRange(List items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + itemsCount = items.Count, + copied = Math.Min(blockSize - _nextItemIndex, itemsCount); + + Span memoryBlocksSpan = new(_memoryBlocks, 0, _reservedBlockCount); + items.CopyTo(0, memoryBlocksSpan[targetBlockIndex], _nextItemIndex, copied); + if (_nextItemIndex + copied == blockSize) + { + targetBlockIndex++; + _nextItemIndex = 0; + } + else + { + _nextItemIndex += copied; + } + + while (blockSize <= itemsCount - copied) + { + items.CopyTo(copied, memoryBlocksSpan[targetBlockIndex++], 0, blockSize); + copied += blockSize; + } + + if ((itemsCount - copied < blockSize) && (itemsCount - copied != 0)) + { + _nextItemIndex = itemsCount - copied; + items.CopyTo(copied, memoryBlocksSpan[targetBlockIndex], 0, _nextItemIndex); + } + + _nextItemBlockIndex = targetBlockIndex; + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + } + + public void AddRange(IList items) + { + if (items.Count == 0) + { + return; + } + + long targetCapacity = _longCount + items.Count; + if (_capacity < targetCapacity) + { + _ = EnsureCapacity(this, targetCapacity); + } + + int blockSize = _blockSize, + targetBlockIndex = _nextItemBlockIndex, + memoryBlockCount = _reservedBlockCount; + + // We're better off to temporarily copy it to a fixed array, + // than copying them item by item. We may run into OOMs, though. + T[] itemsBuffer = items.Count >= RecyclableDefaults.MinPooledArrayLength ? _defaultBlockArrayPool.Rent(items.Count) : new T[items.Count]; + try + { + items.CopyTo(itemsBuffer, 0); + + Span itemsSpan = new(itemsBuffer, 0, items.Count); + Span memoryBlocksSpan = new(_memoryBlocks, 0, memoryBlockCount); + Span targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], _nextItemIndex, blockSize - _nextItemIndex); + while (targetBlockArraySpan.Length <= itemsSpan.Length) + { + itemsSpan[..targetBlockArraySpan.Length].CopyTo(targetBlockArraySpan); + itemsSpan = itemsSpan[targetBlockArraySpan.Length..]; + targetBlockIndex++; + if (targetBlockIndex == memoryBlockCount) + { + break; + } + + targetBlockArraySpan = new(memoryBlocksSpan[targetBlockIndex], 0, blockSize); + } + + itemsSpan.CopyTo(targetBlockArraySpan); + _longCount = targetCapacity; + _longCountIndexOfStep = Math.Max((long)(targetCapacity * OptimalParallelSearchStep), 1L); + _nextItemBlockIndex = targetBlockIndex; + _nextItemIndex = itemsSpan.Length; + } + finally + { + if (items.Count >= RecyclableDefaults.MinPooledArrayLength) + { + _defaultBlockArrayPool.Return(itemsBuffer, NeedsClearing); + } + } + } + + public void AddRange(IEnumerable source, int growByCount = RecyclableDefaults.BlockSize) + { + if (source is RecyclableList sourceRecyclableList) + { + AddRange(sourceRecyclableList); + return; + } + + if (source is RecyclableArrayList sourceRecyclableArrayList) + { + AddRange(sourceRecyclableArrayList); + return; + } + + if (source is T[] sourceArray) + { + AddRange(sourceArray); + return; + } + + if (source is List sourceList) + { + AddRange(sourceList); + return; + } + + if (source is IList sourceIList) + { + AddRange(sourceIList); + return; + } + + if (source.TryGetNonEnumeratedCount(out var requiredAdditionalCapacity)) + { + AddRangeWithKnownCount(source, requiredAdditionalCapacity); + return; + } + + AddRangeEnumerated(source, growByCount); + } + + public void Clear() + { + if (_longCount == 0) + { + return; + } + + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + T[][] memoryBlocks = _memoryBlocks; + int memoryBlocksCount = _reservedBlockCount; + ArrayPool blockArrayPool = _blockArrayPool; + int toRemoveIdx = 0; + while (toRemoveIdx < memoryBlocksCount) + { + blockArrayPool.Return(memoryBlocks[toRemoveIdx++], NeedsClearing); + } + } + + Array.Fill(_memoryBlocks, _emptyBlockArray, 0, _reservedBlockCount); + _capacity = 0; + _reservedBlockCount = 0; + _nextItemBlockIndex = 0; + _nextItemIndex = 0; + _longCount = 0; + _longCountIndexOfStep = 1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Contains(T item) + { + if (_longCount == 0) + { + return false; + } + + if (_nextItemBlockIndex == 0 || (_nextItemBlockIndex == 1 && _nextItemIndex == 0)) + { + return Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) >= 0; + } + + Span memoryBlocksSpan = new(_memoryBlocks); + int lastBlockIndex = _nextItemBlockIndex; + for (int blockIndex = 0; blockIndex < lastBlockIndex; blockIndex++) + { + if (Array.IndexOf(memoryBlocksSpan[blockIndex], item, 0, _blockSize) >= 0) + { + return true; + } + } + + return lastBlockIndex < memoryBlocksSpan.Length && Array.IndexOf(memoryBlocksSpan[lastBlockIndex], item, 0, _nextItemIndex) >= 0; + } + + public void CopyTo(T[] array, int arrayIndex) => _memoryBlocks.CopyTo(0, _blockSize, _longCount, array, arrayIndex); + + public IEnumerator GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int IndexOf(T item) => _longCount == 0 + ? ItemNotFoundIndex + : checked((int)DoIndexOfParallel(item)); + + public void Insert(int index, T item) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public long LongIndexOf(T item) => _longCount == 0 + ? ItemNotFoundIndexLong + : _longCount <= _blockSize + ? Array.IndexOf(_memoryBlocks[0], item, 0, checked((int)_longCount)) + : DoIndexOfParallel(item); + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public bool Remove(T item) + { + var index = LongIndexOf(item); + if (index >= 0) + { + _longCount--; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + + return true; + } + + return false; + } + + public void RemoveBlock(int index) + { + if (_blockSize >= RecyclableDefaults.MinPooledArrayLength) + { + _blockArrayPool.Return(_memoryBlocks[index], NeedsClearing); + _memoryBlocks[index] = _emptyBlockArray; + } + else + { + _memoryBlocks[index] = _emptyBlockArray; + } + + _capacity -= _blockSize; + if (_nextItemBlockIndex == index) + { + _nextItemIndex = 0; + } + else if (_nextItemBlockIndex > index) + { + _nextItemBlockIndex--; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public void RemoveAt(int index) + { + if (index >= _longCount || index < 0) + { + ThrowArgumentOutOfRangeException($"Argument \"{nameof(index)}\" = {index} is out of range. Expected value between 0 and {_longCount - 1}"); + } + + _longCount--; + _longCountIndexOfStep = Math.Max((long)(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + public void RemoveAt(long index) + { + if (index >= _longCount || index < 0) + { + ThrowArgumentOutOfRangeException($"Argument \"{nameof(index)}\" = {index} is out of range. Expected value between 0 and {_longCount - 1}"); + } + + _longCount--; + _longCountIndexOfStep = Math.Max((long)Math.Floor(_longCount * OptimalParallelSearchStep), 1L); + + if (index < _longCount) + { + CopyItems(this, index); + } + + if (_nextItemIndex > 0) + { + _nextItemIndex--; + } + else + { + _nextItemIndex = _blockSizeMinus1; + _nextItemBlockIndex--; + } + + if (NeedsClearing) + { +#pragma warning disable CS8601 // In real use cases we'll never access it + _memoryBlocks[_nextItemBlockIndex][_nextItemIndex] = default; +#pragma warning restore CS8601 + } + } + + IEnumerator IEnumerable.GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); + + ~RecyclableListIndexOfV4() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + if (_capacity > 0) + { + Clear(); + if (_memoryBlocks.Length >= RecyclableDefaults.MinPooledArrayLength) + { + _memoryBlocksPool.Return(_memoryBlocks, NeedsClearing); + } + + GC.SuppressFinalize(this); + } + } + } + #endregion + public partial class RecyclableListPocBenchmarks { - [Benchmark(Baseline = true)] + // [Benchmark(Baseline = true)] public void RecyclableList_IndexOfV1() { var data = TestObjects; @@ -1166,6 +4670,54 @@ namespace Recyclable.Collections.Benchmarks.POC } } + // [Benchmark(Baseline = true)] + public void PooledList_IndexOf() + { + var data = TestObjects; + var list = TestPooledList; + var dataCount = TestObjectCount; + for (var i = 0; i < dataCount; i++) + { + DoNothing(list.IndexOf(data[i])); + } + } + + // [Benchmark(Baseline = false)] + public void RecyclableList_IndexOfV2() + { + var data = TestObjects; + var list = TestObjectsAsRecyclableListIndexOfV2; + var dataCount = TestObjectCount; + for (var i = 0; i < dataCount; i++) + { + DoNothing(list.IndexOf(data[i])); + } + } + + // [Benchmark(Baseline = false)] + public void RecyclableList_IndexOfV3() + { + var data = TestObjects; + var list = TestObjectsAsRecyclableListIndexOfV3; + var dataCount = TestObjectCount; + for (var i = 0; i < dataCount; i++) + { + DoNothing(list.IndexOf(data[i])); + } + } + + // [Benchmark(Baseline = true)] + public void RecyclableList_IndexOfV4() + { + var data = TestObjects; + var list = TestObjectsAsRecyclableListIndexOfV4; + var dataCount = TestObjectCount; + for (var i = 0; i < dataCount; i++) + { + DoNothing(list.IndexOf(data[i])); + } + } + [Benchmark(Baseline = false)] public void RecyclableList_IndexOf() { diff --git a/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.cs b/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.cs index ca2013c..69e381e 100644 --- a/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.cs +++ b/Recyclable.Collections.Benchmarks/POC/RecyclableListPoc.Benchmarks.cs @@ -1,6 +1,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; +using Collections.Pooled; using Recyclable.Collections.Benchmarks.Core; namespace Recyclable.Collections.Benchmarks.POC @@ -10,9 +11,17 @@ namespace Recyclable.Collections.Benchmarks.POC private long[]? _testArray; public long[] TestObjects => _testArray ?? throw new NullReferenceException("Something is wrong and test objects are null"); private RecyclableList? _testRecyclableList; - public RecyclableList TestObjectsAsRecyclableList => _testRecyclableList ?? throw new NullReferenceException("Something is wrong and RecyclableList is null"); + public RecyclableList TestObjectsAsRecyclableList => _testRecyclableList ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableList)} is null"); private RecyclableListIndexOfV1? _testRecyclableListIndexOfV1; - internal RecyclableListIndexOfV1 TestObjectsAsRecyclableListIndexOfV1 => _testRecyclableListIndexOfV1 ?? throw new NullReferenceException("Something is wrong and RecyclableListIndexOfV1 is null"); + internal RecyclableListIndexOfV1 TestObjectsAsRecyclableListIndexOfV1 => _testRecyclableListIndexOfV1 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV1)} is null"); + private RecyclableListIndexOfV2? _testRecyclableListIndexOfV2; + internal RecyclableListIndexOfV2 TestObjectsAsRecyclableListIndexOfV2 => _testRecyclableListIndexOfV2 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV2)} is null"); + private RecyclableListIndexOfV3? _testRecyclableListIndexOfV3; + internal RecyclableListIndexOfV3 TestObjectsAsRecyclableListIndexOfV3 => _testRecyclableListIndexOfV3 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV3)} is null"); + private RecyclableListIndexOfV4? _testRecyclableListIndexOfV4; + internal RecyclableListIndexOfV4 TestObjectsAsRecyclableListIndexOfV4 => _testRecyclableListIndexOfV4 ?? throw new NullReferenceException($"Something is wrong and {nameof(RecyclableListIndexOfV4)} is null"); + private PooledList? _testPooledList; + internal PooledList TestPooledList => _testPooledList ?? throw new NullReferenceException($"Something is wrong and {nameof(PooledList)} is null"); private static readonly MethodInfo _ensureCapacityNewFunc; private static readonly MethodInfo _ensureCapacityV1Func; @@ -47,6 +56,9 @@ namespace Recyclable.Collections.Benchmarks.POC benchmark.RecyclableList_EnsureCapacityV3_ByBlockSize(); benchmark.RecyclableList_IndexOf(); benchmark.RecyclableList_IndexOfV1(); + benchmark.RecyclableList_IndexOfV2(); + benchmark.RecyclableList_IndexOfV3(); + benchmark.RecyclableList_IndexOfV4(); } [GlobalSetup] @@ -55,6 +67,10 @@ namespace Recyclable.Collections.Benchmarks.POC _testArray = DataGenerator.EnumerateTestObjects(TestObjectCount); _testRecyclableList = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); _testRecyclableListIndexOfV1 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); + _testRecyclableListIndexOfV2 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); + _testRecyclableListIndexOfV3 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); + _testRecyclableListIndexOfV4 = new(TestObjects, BlockSize, expectedItemsCount: TestObjectCount); + _testPooledList = new PooledList(TestObjects, ClearMode.Auto); // Warm up memory pools for fair comparison with 1st run RecyclableList_EnsureCapacity_ByPowOf2(); @@ -63,6 +79,10 @@ namespace Recyclable.Collections.Benchmarks.POC RecyclableList_EnsureCapacityV3_ByPowOf2(); RecyclableList_IndexOf(); RecyclableList_IndexOfV1(); + RecyclableList_IndexOfV2(); + RecyclableList_IndexOfV3(); + RecyclableList_IndexOfV4(); + PooledList_IndexOf(); } [GlobalCleanup] @@ -73,6 +93,15 @@ namespace Recyclable.Collections.Benchmarks.POC _testRecyclableList = null; _testRecyclableListIndexOfV1?.Dispose(); _testRecyclableListIndexOfV1 = null; + _testRecyclableListIndexOfV2?.Dispose(); + _testRecyclableListIndexOfV2 = null; + _testRecyclableListIndexOfV3?.Dispose(); + _testRecyclableListIndexOfV3 = null; + _testRecyclableListIndexOfV4?.Dispose(); + _testRecyclableListIndexOfV4 = null; + _testPooledList?.Dispose(); + _testPooledList = null; + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true); } } } diff --git a/Recyclable.Collections/ItemRange.cs b/Recyclable.Collections/ItemRange.cs new file mode 100644 index 0000000..4a1d273 --- /dev/null +++ b/Recyclable.Collections/ItemRange.cs @@ -0,0 +1,32 @@ +using Recyclable.Collections.Pools; + +namespace Recyclable.Collections +{ + public sealed class ItemRange + { + public int BlockIndex; + public int StartingItemIndex; + public long ItemsToSearchCount; + internal bool _returned; + + public ItemRange() + { + } + + public ItemRange(int blockIndex, int startingItemIndex, long itemsToSearchCount) + { + BlockIndex = blockIndex; + ItemsToSearchCount = itemsToSearchCount; + StartingItemIndex = startingItemIndex; + } + + public void Dispose() + { + if (!_returned) + { + _returned = true; + ItemRangePool.Shared.Return(this); + } + } + } +} diff --git a/Recyclable.Collections/ItemRangesIterator.cs b/Recyclable.Collections/ItemRangesIterator.cs index 685f0b4..74c7d9e 100644 --- a/Recyclable.Collections/ItemRangesIterator.cs +++ b/Recyclable.Collections/ItemRangesIterator.cs @@ -1,237 +1,29 @@ -using Microsoft.Extensions.ObjectPool; -using System.Runtime.CompilerServices; +using System.Runtime.CompilerServices; +using Recyclable.Collections.Parallel; namespace Recyclable.Collections { - public sealed class ItemRange : IDisposable - { - private static readonly ObjectPool _defaultPool = ItemRangePool.Shared; - - public int BlockIndex; - public int StartingItemIndex; - public long ItemsToSearchCount; - private ObjectPool? _assignedPool; - - public ItemRange() - { - - } - - public ItemRange(int blockIndex, int startingItemIndex, long itemsToSearchCount) - { - BlockIndex = blockIndex; - ItemsToSearchCount = itemsToSearchCount; - StartingItemIndex = startingItemIndex; - } - - public void Dispose() - { - if (_assignedPool != null) - { - var assignedPool = _assignedPool; - _assignedPool = null; - assignedPool.Return(this); - } - - GC.SuppressFinalize(this); - } - - public static ItemRange Create(int blockIndex, int startingItemIndex, long itemsToSearchCount) - { - var itemRange = _defaultPool.Get(); - itemRange.BlockIndex = blockIndex; - itemRange.ItemsToSearchCount = itemsToSearchCount; - itemRange.StartingItemIndex = startingItemIndex; - return itemRange; - } - } - - public static class ItemRangePool - { - [ThreadStatic] - private static ObjectPool _shared; - public static ObjectPool Shared => _shared; - - static ItemRangePool() - { - _shared = new DefaultObjectPool(new DefaultPooledObjectPolicy()); - } - } - - public sealed class BarrierSlimmer : IDisposable - { - - private static readonly ObjectPool _defaultPool = new DefaultObjectPool(new DefaultPooledObjectPolicy()); - private ObjectPool? _assignedPool; - private readonly object _lock = new(); - - public bool HasParticipants; - public int Participants; - - public void AddParticipant() - { - Monitor.Enter(_lock); - if (++Participants == 1) - { - HasParticipants = true; - } - - Monitor.Exit(_lock); - } - - public void RemoveParticipant() - { - Monitor.Enter(_lock); - if (Participants == 0) - { - Monitor.Exit(_lock); - ThrowNoMoreParticipants(); - } - - if (--Participants == 0) - { - HasParticipants = false; - } - - Monitor.Exit(_lock); - } - - private static void ThrowNoMoreParticipants() - { - throw new InvalidOperationException("There are no more participants. The operation is not allowed"); - } - - public BarrierSlimmer() - { - } - - public BarrierSlimmer(int participants) - { - Participants = participants; - HasParticipants = participants > 0; - } - - public static BarrierSlimmer Create(int participants) - { - var result = _defaultPool.Get(); - result.Participants = participants; - result.HasParticipants = participants > 0; - return result; - } - - public void Dispose() - { - var assignedPool = _assignedPool; - _assignedPool = null; - assignedPool?.Return(this); - - GC.SuppressFinalize(this); - } - - public void SignalAndWait() - { - RemoveParticipant(); - - if (!HasParticipants) - { - return; - } - - if (!HasParticipants) - { - return; - } - - if (!HasParticipants) - { - return; - } - - var waitSpinner = new SpinWait(); - while (HasParticipants) - { - waitSpinner.SpinOnce(); - } - } - } - - public sealed class ManualResetEventSlimmer : IDisposable - { - private static readonly ObjectPool _defaultPool = ManualResetEventSlimmerPool.Shared; - - private bool _isSet; - public bool IsSet => _isSet; - private ObjectPool? _assignedPool; - - public ManualResetEventSlimmer(bool initialState) - { - _isSet = initialState; - } - - public ManualResetEventSlimmer() - { - - } - - public void Set() - { - _isSet = true; - } - - public void Reset() - { - _isSet = false; - } - - public void Dispose() - { - var assignedPool = _assignedPool; - _assignedPool = null; - assignedPool?.Return(this); - - GC.SuppressFinalize(this); - } - - public static ManualResetEventSlimmer Create(bool initialState) - { - ManualResetEventSlimmer result = _defaultPool.Get(); - result._assignedPool = _defaultPool; - result._isSet = initialState; - return result; - } - } - - public static class ManualResetEventSlimmerPool - { - [ThreadStatic] - private static ObjectPool _shared; - public static ObjectPool Shared => _shared; - - static ManualResetEventSlimmerPool() - { - _shared = new DefaultObjectPool(new DefaultPooledObjectPolicy()); - } - } - - public static class ItemRangesIterator + public static class ItemRangesIterator { [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.NoInlining)] - public static void Iterate(int blockIndex, int blockSize, byte blockSizePow2BitShift, long itemsCount, long step, Func action) + public static void Iterate(in ParallelSynchronizationContext context, in SearchConfiguration search, Func, ItemRange, bool> action) { int itemIndex = 0; ItemRange itemRange; - while (itemsCount > step) + int blockIndex = search.StartingBlockIndex; + long itemsCount = search.ItemsCount; + while (itemsCount > search.Step) { - itemRange = ItemRange.Create(blockIndex, itemIndex, step); - if (!action(itemRange)) + itemRange = new(blockIndex, itemIndex, search.Step); + if (!action(context, search, itemRange)) { break; } - itemsCount -= step; + itemsCount -= search.Step; //blockIndex = (int)Math.DivRem(itemIndex + step, blockSize, out itemIndex); - blockIndex += (int)((itemIndex + step) >> blockSizePow2BitShift); - itemIndex = (int)((itemIndex + step) & (blockSize - 1)); + blockIndex += (int)((itemIndex + search.Step) >> search.BlockSizePow2BitShift); + itemIndex = (int)((itemIndex + search.Step) & search.BlockSizeMinus1); //itemIndex += step; //while (itemIndex >= blockSize) //{ @@ -240,8 +32,8 @@ namespace Recyclable.Collections //} } - itemRange = ItemRange.Create(blockIndex, itemIndex, itemsCount); - _ = action(itemRange); + itemRange = new(blockIndex, itemIndex, itemsCount); + _ = action(context, search, itemRange); } } } diff --git a/Recyclable.Collections/MemoryBucket.cs b/Recyclable.Collections/MemoryBucket.cs new file mode 100644 index 0000000..d786d1a --- /dev/null +++ b/Recyclable.Collections/MemoryBucket.cs @@ -0,0 +1,22 @@ +namespace Recyclable.Collections +{ + public class MemoryBucket + { + public MemoryBucket(T[] memory, MemoryBucket nextBucket) + { + _memory = memory; + _nextBucket = nextBucket; + } + + public MemoryBucket(T[] memory) + { + _memory = memory; + } + + private T[] _memory; + public T[] Memory { get => _memory; set => _memory = value; } + + private MemoryBucket? _nextBucket; + public MemoryBucket? NextBucket { get => _nextBucket; set => _nextBucket = value; } + } +} diff --git a/Recyclable.Collections/OneSizeArrayPool.cs b/Recyclable.Collections/OneSizeArrayPool.cs deleted file mode 100644 index 3f54b31..0000000 --- a/Recyclable.Collections/OneSizeArrayPool.cs +++ /dev/null @@ -1,179 +0,0 @@ -using System.Buffers; -using System.Collections.Concurrent; - -namespace Recyclable.Collections -{ - public class MemoryBucket - { - public MemoryBucket(T[] memory, MemoryBucket nextBucket) - { - _memory = memory; - _nextBucket = nextBucket; - } - - public MemoryBucket(T[] memory) - { - _memory = memory; - } - - private T[] _memory; - public T[] Memory { get => _memory; set => _memory = value; } - - private MemoryBucket? _nextBucket; - public MemoryBucket? NextBucket { get => _nextBucket; set => _nextBucket = value; } - } - - public static class RecyclableArrayPool - { - public static readonly OneSizeArrayPool Null = new(0, 0); - - private static readonly ConcurrentDictionary> _pools = new(); - - public static ArrayPool Shared(int blockSize) - { - //lock (_lock) - { - if (!_pools.TryGetValue(blockSize, out ArrayPool? pool)) - { - pool = new OneSizeArrayPool(blockSize, 100); - pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize]; - } - - return pool; - } - - //_lock.Wait(); - //try - //{ - //} - //finally - //{ - // _lock.Release(); - //} - } - } - - public class OneSizeArrayPool : ArrayPool - { - private readonly int _blockSize; - private readonly int _minBlockCount; - private int _blockCount; - private MemoryBucket? _currentBucket; - //SemaphoreSlim _lock = new(1); - //ManualResetEventSlim _lock = new(true, 1); - private readonly object _updateLock = new(); - //private Task? _bufferFillUpTask; - - //private Task CreateAdditionalBucket(int minimumLength, int blockCount) - //{ - // return Task.Factory.StartNew(() => - // { - // MemoryBucket firstBucket = new(new T[minimumLength]); - // MemoryBucket currentBucket = firstBucket; - // for (var i = 0; i < blockCount; i++) - // { - // currentBucket = new(new T[minimumLength], currentBucket); - // } - - // lock (_updateLock) - // //_lock.Wait(); - // //try - // { - // currentBucket.NextBucket = _currentBucket; - // _currentBucket = firstBucket; - // _blockCount += blockCount; - // //_bufferFillUpTask = null; - // } - // //finally - // //{ - // // _ = _lock.Release(); - // // //_lock.Set(); - // //} - // }); - //} - - public OneSizeArrayPool(int blockSize, int minBlockCount) - { - _blockSize = blockSize; - _minBlockCount = minBlockCount; - } - - public override T[] Rent(int minimumLength) - { - if (minimumLength != _blockSize) - { - ThrowWrongRentedArraySizeException(minimumLength, _blockSize); - } - - MemoryBucket? bucket; - lock (_updateLock) - //_lock.Wait(); - //try - { - if (_currentBucket != null) - { - _blockCount--; - bucket = _currentBucket; - _currentBucket = _currentBucket.NextBucket; - } - else - { - bucket = null; - //if (_bufferFillUpTask == null) - //{ - // _bufferFillUpTask = CreateAdditionalBucket(minimumLength, _minBlockCount); - //} - } - } - //finally - //{ - // _ = _lock.Release(); - // //_lock.Set(); - //} - - return bucket != null ? bucket.Memory : new T[_blockSize]; - } - - public override void Return(T[] array, bool clearArray = false) - { - if (array.Length != _blockSize) - { - ThrowWrongReturnedArraySizeException(array.Length, _blockSize); - } - - MemoryBucket bucket = new(array); - if (clearArray) - { - Array.Clear(array, 0, array.Length); - } - - lock (_updateLock) - //lock (_lock) - //_lock.Wait(); - //try - { - if (_blockCount < _minBlockCount) - { - bucket.NextBucket = _currentBucket; - _currentBucket = bucket; - _blockCount++; - } - } - //finally - //{ - // _ = _lock.Release(); - // //_lock.Set(); - //} - } - - private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize) - { - throw new InvalidOperationException($"The array size {returnedLength} doesn't match the size {blockSize} of the pool buffer and cannot be returned to this pool."); - } - - private static void ThrowWrongRentedArraySizeException(int requestedSize, int blockSize) - { - throw new InvalidOperationException($"The requested size {requestedSize} doesn't match the size {blockSize} of the pool buffer and cannot be taken from this pool."); - } - } -} diff --git a/Recyclable.Collections/Parallel/BarrierSlimmer.cs b/Recyclable.Collections/Parallel/BarrierSlimmer.cs new file mode 100644 index 0000000..2c5e594 --- /dev/null +++ b/Recyclable.Collections/Parallel/BarrierSlimmer.cs @@ -0,0 +1,117 @@ +using Microsoft.Extensions.ObjectPool; +using System.Runtime.CompilerServices; + +namespace Recyclable.Collections.Parallel +{ + public sealed class BarrierSlimmer : IDisposable + { + private static readonly ObjectPool _defaultPool = new DefaultObjectPool(new DefaultPooledObjectPolicy()); + private ObjectPool? _assignedPool; + private readonly SpinLock _lock = new(false); + + public bool HasParticipants; + public int Participants; + + [MethodImpl(MethodImplOptions.NoInlining)] + public void AddParticipant() + { + bool isLocked = false; + _lock.Enter(ref isLocked); + + Participants++; + if (Participants == 1) + { + HasParticipants = true; + } + + if (isLocked) + { + _lock.Exit(false); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public void RemoveParticipant() + { + if (!HasParticipants) + { + ThrowNoMoreParticipants(); + } + + bool isLocked = false; + _lock.Enter(ref isLocked); + + Participants--; + if (Participants == 0) + { + HasParticipants = false; + } + + if (isLocked) + { + _lock.Exit(false); + } + } + + private static void ThrowNoMoreParticipants() + { + throw new InvalidOperationException("There are no more participants. The operation is not allowed"); + } + + public BarrierSlimmer() + { + _lock = new(false); + } + + public BarrierSlimmer(int participants) + { + _lock = new(false); + Participants = participants; + HasParticipants = participants > 0; + } + + public static BarrierSlimmer Create(int participants) + { + var result = _defaultPool.Get(); + result.Participants = participants; + result.HasParticipants = participants > 0; + return result; + } + + public void Dispose() + { + var assignedPool = _assignedPool; + _assignedPool = null; + assignedPool?.Return(this); + + GC.SuppressFinalize(this); + } + + public void SignalAndWait() + { + RemoveParticipant(); + + //if (!HasParticipants) + //{ + // return; + //} + + //if (!HasParticipants) + //{ + // return; + //} + + //if (!HasParticipants) + //{ + // return; + //} + + // SpinWait.SpinUntil(() => HasParticipants); + var waitSpinner = new SpinWait(); + while (HasParticipants) + { + waitSpinner.SpinOnce(0); + } + } + } +} diff --git a/Recyclable.Collections/Parallel/ManualResetEventSlimmer.cs b/Recyclable.Collections/Parallel/ManualResetEventSlimmer.cs new file mode 100644 index 0000000..c56260c --- /dev/null +++ b/Recyclable.Collections/Parallel/ManualResetEventSlimmer.cs @@ -0,0 +1,37 @@ +namespace Recyclable.Collections.Parallel +{ + public sealed class ManualResetEventSlimmer + { + internal bool _isSet; + public bool IsSet => _isSet; + internal bool _returned; + + public ManualResetEventSlimmer() + { + } + + public ManualResetEventSlimmer(bool initialState) + { + _isSet = initialState; + } + + public void Set() + { + _isSet = true; + } + + public void Reset() + { + _isSet = false; + } + + public void Dispose() + { + if (!_returned) + { + _returned = true; + ManualResetEventSlimmerPool.Shared.Return(this); + } + } + } +} diff --git a/Recyclable.Collections/Parallel/ManualResetEventSlimmerPool.cs b/Recyclable.Collections/Parallel/ManualResetEventSlimmerPool.cs new file mode 100644 index 0000000..5cee73e --- /dev/null +++ b/Recyclable.Collections/Parallel/ManualResetEventSlimmerPool.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.ObjectPool; + +namespace Recyclable.Collections.Parallel +{ + public static class ManualResetEventSlimmerPool + { + public static readonly SpinLock _updateLock = new(false); + public static readonly ObjectPool Shared = new DefaultObjectPool(new DefaultPooledObjectPolicy()); + + public static ManualResetEventSlimmer Create(bool initialState) + { + bool lockTaken = false; + _updateLock.Enter(ref lockTaken); + ManualResetEventSlimmer result = Shared.Get(); + if (lockTaken) + { + _updateLock.Exit(false); + } + + result._returned = false; + if (result._isSet != initialState) + { + result._isSet = initialState; + } + + return result; + } + } +} diff --git a/Recyclable.Collections/Parallel/ParallelSynchronizationContext.cs b/Recyclable.Collections/Parallel/ParallelSynchronizationContext.cs new file mode 100644 index 0000000..649027c --- /dev/null +++ b/Recyclable.Collections/Parallel/ParallelSynchronizationContext.cs @@ -0,0 +1,38 @@ +namespace Recyclable.Collections.Parallel +{ + public sealed class ParallelSynchronizationContext : IDisposable + { + public readonly ManualResetEventSlimmer ItemFoundSignal; + public readonly Barrier AllDoneSignal; + public long FoundItemIndex; + public Exception? Exception; + internal bool _returned; + + public ParallelSynchronizationContext() + { + ItemFoundSignal = ManualResetEventSlimmerPool.Create(false); + AllDoneSignal = new(0); + } + + public ParallelSynchronizationContext(int participantCount) + { + ItemFoundSignal = ManualResetEventSlimmerPool.Create(false); + AllDoneSignal = new(participantCount); + } + + public void Dispose() + { + if (!_returned) + { + _returned = true; + ParallelSynchronizationContextPool.Shared.Return(this); + } + else + { + ItemFoundSignal.Dispose(); + AllDoneSignal.Dispose(); + GC.SuppressFinalize(this); + } + } + } +} \ No newline at end of file diff --git a/Recyclable.Collections/Parallel/ParallelSynchronizationContextPool.cs b/Recyclable.Collections/Parallel/ParallelSynchronizationContextPool.cs new file mode 100644 index 0000000..88fa317 --- /dev/null +++ b/Recyclable.Collections/Parallel/ParallelSynchronizationContextPool.cs @@ -0,0 +1,50 @@ +using Microsoft.Extensions.ObjectPool; + +namespace Recyclable.Collections.Parallel +{ + public static class ParallelSynchronizationContextPool + { + private static readonly SpinLock _updateLock = new(false); + public static readonly ObjectPool Shared = new DefaultObjectPool(new DefaultPooledObjectPolicy()); + + public static ParallelSynchronizationContext GetWithOneParticipant() + { + bool lockTaken = false; + _updateLock.Enter(ref lockTaken); + var context = Shared.Get(); + if (lockTaken) + { + _updateLock.Exit(false); + } + + if (context.ItemFoundSignal._isSet) + { + context.ItemFoundSignal.Reset(); + } + + switch (context.AllDoneSignal.ParticipantsRemaining) + { + case > 1: + context.AllDoneSignal.RemoveParticipants(context.AllDoneSignal.ParticipantsRemaining - 1); + break; + + case 0: + context.AllDoneSignal.AddParticipant(); + break; + } + + if (context.Exception != null) + { + context.Exception = null; + } + + if (context.FoundItemIndex >= 0) + { + context.FoundItemIndex = -1L; + } + + context._returned = false; + return context; + } + } +} \ No newline at end of file diff --git a/Recyclable.Collections/Pools/ItemRangePool.cs b/Recyclable.Collections/Pools/ItemRangePool.cs new file mode 100644 index 0000000..532988f --- /dev/null +++ b/Recyclable.Collections/Pools/ItemRangePool.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.ObjectPool; + +namespace Recyclable.Collections.Pools +{ + public static class ItemRangePool + { + private static readonly SpinLock _updateLock = new(false); + public static readonly ObjectPool Shared = new DefaultObjectPool(new DefaultPooledObjectPolicy()); + + public static ItemRange Create(int blockIndex, int startingItemIndex, long itemsToSearchCount) + { + bool lockTaken = false; + _updateLock.Enter(ref lockTaken); + var itemRange = Shared.Get(); + if (lockTaken) + { + _updateLock.Exit(false); + } + + itemRange._returned = false; + itemRange.BlockIndex = blockIndex; + itemRange.ItemsToSearchCount = itemsToSearchCount; + itemRange.StartingItemIndex = startingItemIndex; + return itemRange; + } + } +} diff --git a/Recyclable.Collections/Pools/OneSizeArrayPool.cs b/Recyclable.Collections/Pools/OneSizeArrayPool.cs new file mode 100644 index 0000000..11859c5 --- /dev/null +++ b/Recyclable.Collections/Pools/OneSizeArrayPool.cs @@ -0,0 +1,86 @@ +using System.Buffers; + +namespace Recyclable.Collections.Pools +{ + public class OneSizeArrayPool : ArrayPool + { + private readonly int _blockSize; + private readonly int _minBlockCount; + private int _blockCount; + private MemoryBucket? _currentBucket; + private readonly SpinLock _updateLock = new(false); + + public OneSizeArrayPool(int blockSize, int minBlockCount) + { + _blockSize = blockSize; + _minBlockCount = minBlockCount; + } + + public override T[] Rent(int minimumLength) + { + if (minimumLength != _blockSize) + { + ThrowWrongRentedArraySizeException(minimumLength, _blockSize); + } + + MemoryBucket? bucket; + bool lockTaken = false; + _updateLock.Enter(ref lockTaken); + if (_currentBucket != null) + { + _blockCount--; + bucket = _currentBucket; + _currentBucket = _currentBucket.NextBucket; + } + else + { + bucket = null; + } + + if (lockTaken) + { + _updateLock.Exit(false); + } + + return bucket != null ? bucket.Memory : new T[_blockSize]; + } + + public override void Return(T[] array, bool clearArray = false) + { + if (array.Length != _blockSize) + { + ThrowWrongReturnedArraySizeException(array.Length, _blockSize); + } + + MemoryBucket bucket = new(array); + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + + bool lockTaken = false; + _updateLock.Enter(ref lockTaken); + if (_blockCount < _minBlockCount) + { + bucket.NextBucket = _currentBucket; + _currentBucket = bucket; + _blockCount++; + } + + if (lockTaken) + { + _updateLock.Exit(false); + } + } + + private static void ThrowWrongReturnedArraySizeException(int returnedLength, int blockSize) + { + throw new InvalidOperationException($"The array size {returnedLength} doesn't match the size {blockSize} of the pool buffer and cannot be returned to this pool."); + } + + private static void ThrowWrongRentedArraySizeException(int requestedSize, int blockSize) + { + throw new InvalidOperationException($"The requested size {requestedSize} doesn't match the size {blockSize} of the pool buffer and cannot be taken from this pool."); + } + } +} diff --git a/Recyclable.Collections/Pools/RecyclableArrayPool.cs b/Recyclable.Collections/Pools/RecyclableArrayPool.cs new file mode 100644 index 0000000..3e44463 --- /dev/null +++ b/Recyclable.Collections/Pools/RecyclableArrayPool.cs @@ -0,0 +1,35 @@ +using System.Buffers; +using System.Collections.Concurrent; + +namespace Recyclable.Collections.Pools +{ + public static class RecyclableArrayPool + { + public static readonly OneSizeArrayPool Null = new(0, 0); + + private static readonly ConcurrentDictionary> _pools = new(); + + public static ArrayPool Shared(int blockSize) + { + //lock (_lock) + { + if (!_pools.TryGetValue(blockSize, out ArrayPool? pool)) + { + pool = new OneSizeArrayPool(blockSize, 100); + pool = _pools.TryAdd(blockSize, pool) ? pool : _pools[blockSize]; + } + + return pool; + } + + //_lock.Wait(); + //try + //{ + //} + //finally + //{ + // _lock.Release(); + //} + } + } +} diff --git a/Recyclable.Collections/RecyclableDefaults.cs b/Recyclable.Collections/RecyclableDefaults.cs index 16c4468..03154ca 100644 --- a/Recyclable.Collections/RecyclableDefaults.cs +++ b/Recyclable.Collections/RecyclableDefaults.cs @@ -1,6 +1,6 @@ namespace Recyclable.Collections { - public class RecyclableDefaults + public static class RecyclableDefaults { public const int BlockSize = 16_384; public const int Capacity = 1; diff --git a/Recyclable.Collections/RecyclableList.cs b/Recyclable.Collections/RecyclableList.cs index a09658e..7e64195 100644 --- a/Recyclable.Collections/RecyclableList.cs +++ b/Recyclable.Collections/RecyclableList.cs @@ -2,6 +2,10 @@ using System.Collections; using System.Numerics; using System.Runtime.CompilerServices; +using Recyclable.Collections.Parallel; +using Recyclable.Collections.Pools; + +[assembly:InternalsVisibleTo("Recyclable.Collections.Benchmarks")] namespace Recyclable.Collections { @@ -9,16 +13,16 @@ namespace Recyclable.Collections { private const long ItemNotFoundIndexLong = -1L; private const int ItemNotFoundIndex = -1; - private const double OptimalParallelSearchStep = 0.329; + private const double OptimalParallelSearchStep = 0.1;// 0.329; private static readonly ArrayPool _defaultMemoryBlocksPool = ArrayPool.Create(); private static readonly ArrayPool _defaultBlockArrayPool = ArrayPool.Create(); private static readonly T[][] _emptyMemoryBlocksArray = new T[0][]; private static readonly T[] _emptyBlockArray = new T[0]; - private int _blockSize; - private byte _blockSizePow2BitShift; - private int _blockSizeMinus1; + internal int _blockSize; + internal byte _blockSizePow2BitShift; + internal int _blockSizeMinus1; private int _nextItemBlockIndex; private int _nextItemIndex; private int _reservedBlockCount; @@ -27,7 +31,7 @@ namespace Recyclable.Collections protected ArrayPool _memoryBlocksPool; protected ArrayPool _blockArrayPool; - protected T[][] _memoryBlocks; + internal T[][] _memoryBlocks; private long _capacity; public long Capacity @@ -40,8 +44,8 @@ namespace Recyclable.Collections public bool IsReadOnly { get; } public int LastBlockWithData => _nextItemBlockIndex - (_nextItemIndex > 0 ? 0 : 1); - protected long _longCountIndexOfStep; - protected long _longCount; + internal long _longCountIndexOfStep; + internal long _longCount; public long LongCount { @@ -56,7 +60,6 @@ namespace Recyclable.Collections public int ReservedBlockCount => _reservedBlockCount; public int BlockSize => _blockSize; public byte BlockSizePow2BitShift => _blockSizePow2BitShift; - public int NextItemBlockIndex => _nextItemBlockIndex; public int NextItemIndex => _nextItemIndex; @@ -220,12 +223,13 @@ namespace Recyclable.Collections _longCountIndexOfStep = Math.Max((long)(requiredCapacity * OptimalParallelSearchStep), 1L); _nextItemBlockIndex = targetBlockIdx; _nextItemIndex = targetItemIdx; - return; } - [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] - private static long DoLongIndexOfParallel(in RecyclableList list, in ItemRange itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) + // & WAS SLOWER + // [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] + private static long DoLongIndexOfParallel(RecyclableList list, in ItemRange itemRange, in T itemToFind, in ManualResetEventSlimmer itemFoundSignal) { + // & WAS SLOWER without int blockSize = list._blockSize; int blockIndex = itemRange.BlockIndex; long itemsToSearchCount = itemRange.ItemsToSearchCount; @@ -267,77 +271,76 @@ namespace Recyclable.Collections blockIndex++; } - index = itemsToSearchCount <= 0 || blockIndex > list.LastBlockWithData + index = itemsToSearchCount <= 0 //|| blockIndex > list.LastBlockWithData ? ItemNotFoundIndex - : blockIndex == list._nextItemBlockIndex - ? Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount) + // : blockIndex == list._nextItemBlockIndex + // ? Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount) : Array.IndexOf(list._memoryBlocks[blockIndex], itemToFind, 0, (int)itemsToSearchCount); - //? Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(itemRange.List._nextItemIndex, itemsToSearchCount))) + // ? Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(itemRange.List._nextItemIndex, itemsToSearchCount))) //: Array.IndexOf(memoryBlocks[blockIndex], itemToFind, 0, checked((int)Math.Min(blockSize, itemsToSearchCount))); return !itemFoundSignal.IsSet && index >= 0 ? ((long)blockIndex << list._blockSizePow2BitShift) + index : ItemNotFoundIndexLong; } - [MethodImpl(MethodImplOptions.AggressiveOptimization | MethodImplOptions.AggressiveInlining)] - private long DoIndexOfParallel(T item) + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + private long DoIndexOfParallel(in T item) { - using var itemFoundSignal = ManualResetEventSlimmer.Create(false); - using var allDoneSignal = new Barrier(1); - long foundItemIndex = ItemNotFoundIndexLong; - Exception? ex = null; - - // (long)(_longCount * 0.329) => most efficient step, based on benchmarks - ItemRangesIterator.Iterate(0, _blockSize, _blockSizePow2BitShift, _longCount, _longCountIndexOfStep, itemRange => + if (_longCount == 0) { - _ = allDoneSignal.AddParticipant(); - _ = Task.Factory.StartNew(() => - { - try - { - var index = DoLongIndexOfParallel(this, itemRange, item, itemFoundSignal); - if (index >= 0) - { - itemFoundSignal.Set(); - foundItemIndex = index; - } - } - catch (Exception e) - { - //_ = Interlocked.Exchange(ref ex, e); - lock (allDoneSignal) - { - ex = e; - } - } - finally - { - itemRange.Dispose(); - allDoneSignal.RemoveParticipant(); - } - }); - //.ContinueWith(x => - //{ - // if (x.Exception != null) - // { - // lock (allDoneSignal) - // { - // ex = x.Exception; - // } - // } - //}); - - return !itemFoundSignal.IsSet; - }); - allDoneSignal.SignalAndWait(); - if (ex == null) - { - return foundItemIndex; + return ItemNotFoundIndex; } - ex.CaptureAndRethrow(); + using var context = ParallelSynchronizationContextPool.GetWithOneParticipant(); + // (long)(_longCount * 0.329) => most efficient step, based on benchmarks + Iterate(context, item); + context.AllDoneSignal.SignalAndWait(); + if (context.Exception == null) + { + return context.FoundItemIndex; + } + + context.Exception.CaptureAndRethrow(); return ItemNotFoundIndexLong; } + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + // [MethodImpl(MethodImplOptions.AggressiveInlining)] + // [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.NoOptimization)] + private static bool ScheduleIndexOfTask(ParallelSynchronizationContext context, RecyclableList list, T itemToFind, ItemRange itemRange) + { + _ = context.AllDoneSignal.AddParticipant(); + // & WAS SLOWER + // _ = ThreadPool.QueueUserWorkItem((_) => + _ = Task.Factory.StartNew(() => + { + try + { + if (context.ItemFoundSignal.IsSet) + { + return; + } + + var index = DoLongIndexOfParallel(list, itemRange, itemToFind, context.ItemFoundSignal); + if (index >= 0) + { + context.ItemFoundSignal.Set(); + _ = Interlocked.Exchange(ref context.FoundItemIndex, index); + } + } + catch (Exception e) + { + _ = Interlocked.Exchange(ref context.Exception, e); + } + finally + { + context.AllDoneSignal.RemoveParticipant(); + itemRange.Dispose(); + } + }); + + return !context.ItemFoundSignal.IsSet; + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] protected static long DoIndexOf(T item, in T[][] memoryBlocks, int lastBlockIndex, int blockSize, int nextItemIndex) { @@ -427,35 +430,35 @@ namespace Recyclable.Collections } blockIndex = sourceBlockCount; - while (true) - { - newMemoryBlocks[blockIndex] = blockArrayPool.Rent(minBlockSize); - if (blockIndex + 1 < requiredBlockCount) - { - blockIndex++; - } - else - { - break; - } - } - } + while (true) + { + newMemoryBlocks[blockIndex] = blockArrayPool.Rent(minBlockSize); + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } else { blockIndex = sourceBlockCount; - while (true) - { - newMemoryBlocks[blockIndex] = new T[minBlockSize]; - if (blockIndex + 1 < requiredBlockCount) - { - blockIndex++; - } - else - { - break; - } - } - } + while (true) + { + newMemoryBlocks[blockIndex] = new T[minBlockSize]; + if (blockIndex + 1 < requiredBlockCount) + { + blockIndex++; + } + else + { + break; + } + } + } } list._reservedBlockCount = requiredBlockCount; @@ -471,6 +474,63 @@ namespace Recyclable.Collections return minBlockSize; } + // & WAS SLOWER + // [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + // & WAS SLOWER + // [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] + // & WAS SLOWER + // [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.NoOptimization)] + // & WAS SLOWER + // [MethodImpl(MethodImplOptions.NoInlining)] + private void Iterate(in ParallelSynchronizationContext context, in T itemToFind) + { + // TODO: Switch to use _blockSize instead of list._longCountIndeexOfStep for simplifications & better performance. + + // & WAS SLOWER without + ItemRange itemRange; + int blockIndex = 0; + // & WAS SLOWER + // int blockSize = _blockSize; + long itemIndex = 0; + long itemsCount = _longCount; + // & WAS SLOWER + // int blockSize = _blockSize; + // long step = list._longCountIndexOfStep; + // & WAS SLOWER + // while ((blockIndex << list._blockSizePow2BitShift) + itemIndex > step) + // & WAS SLOWER + // long longCountIndexOfStep = _longCountIndexOfStep; + while (itemsCount > _longCountIndexOfStep) + { + // At this point itemIndex is limited to blockSize range - int. + itemRange = ItemRangePool.Create(blockIndex, (int)itemIndex, _longCountIndexOfStep); + if (!ScheduleIndexOfTask(context, this, itemToFind, itemRange)) + { + break; + } + + itemsCount -= _longCountIndexOfStep; + + // & WAS SLOWER + // blockIndex = (int)Math.DivRem(itemIndex + step, step, out itemIndex); + // & WAS SLOWER + // blockIndex += (int)((itemIndex + list._longCountIndexOfStep) >> list._blockSizePow2BitShift); + // itemIndex = (int)((itemIndex + list._longCountIndexOfStep) & list._blockSizeMinus1); + + itemIndex += _longCountIndexOfStep; + while (itemIndex >= _blockSize) + { + blockIndex++; + itemIndex -= _blockSize; + } + } + + // & WAS SLOWER + // itemRange = new(blockIndex, (int)itemIndex, list._longCount - (blockIndex << list._blockSizePow2BitShift) - itemIndex); + itemRange = ItemRangePool.Create(blockIndex, (int)itemIndex, itemsCount); + _ = ScheduleIndexOfTask(context, this, itemToFind, itemRange); + } + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] protected static void SetupBlockArrayPooling(RecyclableList list, int blockSize, ArrayPool? blockArrayPool = null) { @@ -951,9 +1011,7 @@ namespace Recyclable.Collections public IEnumerator GetEnumerator() => _memoryBlocks.Enumerate(_blockSize, LongCount).GetEnumerator(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int IndexOf(T item) => _longCount == 0 - ? ItemNotFoundIndex - : checked((int)DoIndexOfParallel(item)); + public int IndexOf(T item) => checked((int)DoIndexOfParallel(item)); public void Insert(int index, T item) => throw new NotSupportedException(); diff --git a/Recyclable.Collections/RecyclableListExtensions.cs b/Recyclable.Collections/RecyclableListExtensions.cs index 2823cf7..fedf113 100644 --- a/Recyclable.Collections/RecyclableListExtensions.cs +++ b/Recyclable.Collections/RecyclableListExtensions.cs @@ -6,6 +6,8 @@ namespace Recyclable.Collections public static class RecyclableListExtensions { private const int _minPooledArraySize = RecyclableDefaults.MinPooledArrayLength; + private const long ItemNotFoundIndexLong = -1L; + private const int ItemNotFoundIndex = -1; public static void CopyItems(this T[] memory, long startingIndex, long count, ref T[] destArray, long offset = 1) { diff --git a/Recyclable.Collections/SearchConfiguration.cs b/Recyclable.Collections/SearchConfiguration.cs new file mode 100644 index 0000000..35c8a2c --- /dev/null +++ b/Recyclable.Collections/SearchConfiguration.cs @@ -0,0 +1,29 @@ +namespace Recyclable.Collections +{ + public readonly struct SearchConfiguration + { + public TItem ItemToFind { get; init; } + public TList List { get; init; } + public int BlockSizeMinus1 { get; init; } + public byte BlockSizePow2BitShift { get; init; } + public long ItemsCount { get; init; } + public int StartingBlockIndex { get; init; } + public long Step { get; init; } + + // public SearchConfiguration(TList list, in TItem itemToFind) + // { + // ItemToFind = itemToFind; + // List = list; + // } + + public static SearchConfiguration, TItem> CreateFrom(RecyclableList list, in TItem itemToFind) => new() + { + List = list, + ItemToFind = itemToFind, + BlockSizeMinus1 = list._blockSizeMinus1, + BlockSizePow2BitShift = list._blockSizePow2BitShift, + ItemsCount = list._longCount, + Step = list._longCountIndexOfStep + }; + } +} \ No newline at end of file diff --git a/Recyclable.CollectionsTests/ItemRangesIteratorTests.cs b/Recyclable.CollectionsTests/ItemRangesIteratorTests.cs index fb9b33c..586fbae 100644 --- a/Recyclable.CollectionsTests/ItemRangesIteratorTests.cs +++ b/Recyclable.CollectionsTests/ItemRangesIteratorTests.cs @@ -1,5 +1,6 @@ using FluentAssertions; using Recyclable.Collections; +using Recyclable.Collections.Parallel; using System.Numerics; namespace Recyclable.CollectionsTests @@ -11,18 +12,31 @@ namespace Recyclable.CollectionsTests public void IterateShouldYieldAllItems(int startingBlockIndex, int blockSize, long itemsCount, int step, IEnumerable<(int BlockIndex, int StartingItemIndex, long ItemsCount)> expected) { // Prepare + var mockList = new RecyclableList(); byte blockSizePow2BitShift = (byte)(31 - BitOperations.LeadingZeroCount((uint)blockSize)); - var actual = new List<(int BlockIndex, int StartingItemIndex, long ItemsCount)>(); + + var synchronizationContext = new ParallelSynchronizationContext(1); + var search = new SearchConfiguration, long>() + { + List = mockList, + ItemToFind = 0, + BlockSizeMinus1 = blockSize - 1, + BlockSizePow2BitShift = blockSizePow2BitShift, + ItemsCount = itemsCount, + Step = step, + }; + + var actualItemRanges = new List<(int BlockIndex, int StartingItemIndex, long ItemsCount)>(); // Act - ItemRangesIterator.Iterate(startingBlockIndex, blockSize, blockSizePow2BitShift, itemsCount, step, (searchInfo) => + ItemRangesIterator.Iterate(synchronizationContext, search, (context, search, itemRange) => { - actual.Add((searchInfo.BlockIndex, searchInfo.StartingItemIndex, searchInfo.ItemsToSearchCount)); + actualItemRanges.Add((itemRange.BlockIndex, itemRange.StartingItemIndex, itemRange.ItemsToSearchCount)); return true; }); // Validate - _ = actual.Should().HaveCount(expected.Count()).And.ContainInConsecutiveOrder(expected); + _ = actualItemRanges.Should().HaveCount(expected.Count()).And.ContainInConsecutiveOrder(expected); } public static IEnumerable TestCases => new[] diff --git a/Recyclable.CollectionsTests/RecyclableListTestData.cs b/Recyclable.CollectionsTests/RecyclableListTestData.cs index b4d17e3..ba0f50b 100644 --- a/Recyclable.CollectionsTests/RecyclableListTestData.cs +++ b/Recyclable.CollectionsTests/RecyclableListTestData.cs @@ -2,7 +2,7 @@ namespace Recyclable.CollectionsTests { - public class RecyclableListTestData + public static class RecyclableListTestData { private static IEnumerable DoCreateTestData(long itemsCount) {