Reduce GC pressure in divisor checking pipeline
This commit is contained in:
Marek Lemańczyk committed
1 parent
571b5a9938
commit
51730bcba3
2 files changed
+183
-142
No files matched your search
@@ -1,4 +1,5 @@
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Buffers.Binary;
|
||||
using System.Diagnostics;
|
||||
using System.Numerics;
|
||||
@@ -56,6 +57,9 @@ internal static partial class PrimeOrderCalculator
|
||||
[ThreadStatic]
|
||||
private static Dictionary<ulong, int>? s_partialFactorCounts;
|
||||
|
||||
[ThreadStatic]
|
||||
private static List<ulong>? s_heuristicCandidateList;
|
||||
|
||||
private static bool IsGpuHeuristicDevice => s_pow2ModeInitialized && s_deviceMode == PrimeOrderHeuristicDevice.Gpu;
|
||||
|
||||
private readonly struct Pow2ModeScope
|
||||
@@ -402,124 +406,131 @@ internal static partial class PrimeOrderCalculator
|
||||
}
|
||||
|
||||
int capacity = config.MaxPowChecks <= 0 ? 64 : config.MaxPowChecks * 4;
|
||||
List<ulong> candidates = new(capacity);
|
||||
FactorEntry[] factorArray = orderFactors.Factors!;
|
||||
// DebugLog("Building candidates list");
|
||||
BuildCandidates(order, factorArray, orderFactors.Count, candidates, capacity);
|
||||
if (candidates.Count == 0)
|
||||
List<ulong> candidates = AcquireHeuristicCandidateList(capacity);
|
||||
try
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// DebugLog("Sorting candidates");
|
||||
SortCandidates(prime, previousOrder, candidates);
|
||||
|
||||
int powBudget = config.MaxPowChecks <= 0 ? candidates.Count : config.MaxPowChecks;
|
||||
int powUsed = 0;
|
||||
int candidateCount = candidates.Count;
|
||||
bool allowGpuBatch = true;
|
||||
Span<ulong> candidateSpan = CollectionsMarshal.AsSpan(candidates);
|
||||
|
||||
// DebugLog(() => $"Checking candidates ({candidateCount} candidates, {powBudget} pow budget)");
|
||||
int index = 0;
|
||||
const int MaxGpuBatchSize = 256;
|
||||
const int StackGpuBatchSize = 64;
|
||||
Span<ulong> stackGpuRemainders = stackalloc ulong[StackGpuBatchSize];
|
||||
ArrayPool<ulong> pool = ThreadStaticPools.UlongPool;
|
||||
while (index < candidateCount && powUsed < powBudget)
|
||||
{
|
||||
int remaining = candidateCount - index;
|
||||
int budgetRemaining = powBudget - powUsed;
|
||||
int batchSize = Math.Min(remaining, Math.Min(budgetRemaining, MaxGpuBatchSize));
|
||||
if (batchSize <= 0)
|
||||
FactorEntry[] factorArray = orderFactors.Factors!;
|
||||
// DebugLog("Building candidates list");
|
||||
BuildCandidates(order, factorArray, orderFactors.Count, candidates, capacity);
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
break;
|
||||
return false;
|
||||
}
|
||||
|
||||
ReadOnlySpan<ulong> batch = candidateSpan.Slice(index, batchSize);
|
||||
ulong[]? gpuPool = null;
|
||||
Span<ulong> pooledGpuRemainders = default;
|
||||
bool gpuSuccess = false;
|
||||
bool gpuStackRemainders = false;
|
||||
GpuPow2ModStatus status = GpuPow2ModStatus.Unavailable;
|
||||
|
||||
if (allowGpuBatch && IsGpuPow2Allowed)
|
||||
|
||||
// DebugLog("Sorting candidates");
|
||||
SortCandidates(prime, previousOrder, candidates);
|
||||
|
||||
int powBudget = config.MaxPowChecks <= 0 ? candidates.Count : config.MaxPowChecks;
|
||||
int powUsed = 0;
|
||||
int candidateCount = candidates.Count;
|
||||
bool allowGpuBatch = true;
|
||||
Span<ulong> candidateSpan = CollectionsMarshal.AsSpan(candidates);
|
||||
|
||||
// DebugLog(() => $"Checking candidates ({candidateCount} candidates, {powBudget} pow budget)");
|
||||
int index = 0;
|
||||
const int MaxGpuBatchSize = 256;
|
||||
const int StackGpuBatchSize = 64;
|
||||
Span<ulong> stackGpuRemainders = stackalloc ulong[StackGpuBatchSize];
|
||||
ArrayPool<ulong> pool = ThreadStaticPools.UlongPool;
|
||||
while (index < candidateCount && powUsed < powBudget)
|
||||
{
|
||||
if (batchSize <= StackGpuBatchSize)
|
||||
int remaining = candidateCount - index;
|
||||
int budgetRemaining = powBudget - powUsed;
|
||||
int batchSize = Math.Min(remaining, Math.Min(budgetRemaining, MaxGpuBatchSize));
|
||||
if (batchSize <= 0)
|
||||
{
|
||||
Span<ulong> localRemainders = stackGpuRemainders.Slice(0, batchSize);
|
||||
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, localRemainders, divisorData);
|
||||
if (status == GpuPow2ModStatus.Success)
|
||||
{
|
||||
gpuSuccess = true;
|
||||
gpuStackRemainders = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
|
||||
ReadOnlySpan<ulong> batch = candidateSpan.Slice(index, batchSize);
|
||||
ulong[]? gpuPool = null;
|
||||
Span<ulong> pooledGpuRemainders = default;
|
||||
bool gpuSuccess = false;
|
||||
bool gpuStackRemainders = false;
|
||||
GpuPow2ModStatus status = GpuPow2ModStatus.Unavailable;
|
||||
|
||||
if (allowGpuBatch && IsGpuPow2Allowed)
|
||||
{
|
||||
gpuPool = pool.Rent(batchSize);
|
||||
Span<ulong> pooledRemainders = gpuPool.AsSpan(0, batchSize);
|
||||
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, pooledRemainders, divisorData);
|
||||
if (status == GpuPow2ModStatus.Success)
|
||||
if (batchSize <= StackGpuBatchSize)
|
||||
{
|
||||
pooledGpuRemainders = pooledRemainders;
|
||||
gpuSuccess = true;
|
||||
Span<ulong> localRemainders = stackGpuRemainders.Slice(0, batchSize);
|
||||
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, localRemainders, divisorData);
|
||||
if (status == GpuPow2ModStatus.Success)
|
||||
{
|
||||
gpuSuccess = true;
|
||||
gpuStackRemainders = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
pool.Return(gpuPool, clearArray: false);
|
||||
gpuPool = null;
|
||||
gpuPool = pool.Rent(batchSize);
|
||||
Span<ulong> pooledRemainders = gpuPool.AsSpan(0, batchSize);
|
||||
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, pooledRemainders, divisorData);
|
||||
if (status == GpuPow2ModStatus.Success)
|
||||
{
|
||||
pooledGpuRemainders = pooledRemainders;
|
||||
gpuSuccess = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
pool.Return(gpuPool, clearArray: false);
|
||||
gpuPool = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gpuSuccess && (status == GpuPow2ModStatus.Overflow || status == GpuPow2ModStatus.Unavailable))
|
||||
{
|
||||
allowGpuBatch = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gpuSuccess && (status == GpuPow2ModStatus.Overflow || status == GpuPow2ModStatus.Unavailable))
|
||||
|
||||
for (int i = 0; i < batchSize && powUsed < powBudget; i++)
|
||||
{
|
||||
allowGpuBatch = false;
|
||||
ulong candidate = batch[i];
|
||||
powUsed++;
|
||||
|
||||
bool equalsOne;
|
||||
if (gpuSuccess)
|
||||
{
|
||||
ulong remainderValue = gpuStackRemainders ? stackGpuRemainders[i] : pooledGpuRemainders[i];
|
||||
equalsOne = remainderValue == 1UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
equalsOne = Pow2EqualsOneCpu(candidate, prime, divisorData);
|
||||
}
|
||||
if (!equalsOne)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryConfirmCandidateCpu(prime, candidate, divisorData, config, ref powUsed, powBudget))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (gpuPool is not null)
|
||||
{
|
||||
pool.Return(gpuPool, clearArray: false);
|
||||
}
|
||||
|
||||
result = candidate;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < batchSize && powUsed < powBudget; i++)
|
||||
{
|
||||
ulong candidate = batch[i];
|
||||
powUsed++;
|
||||
|
||||
bool equalsOne;
|
||||
if (gpuSuccess)
|
||||
{
|
||||
ulong remainderValue = gpuStackRemainders ? stackGpuRemainders[i] : pooledGpuRemainders[i];
|
||||
equalsOne = remainderValue == 1UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
equalsOne = Pow2EqualsOneCpu(candidate, prime, divisorData);
|
||||
}
|
||||
if (!equalsOne)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryConfirmCandidateCpu(prime, candidate, divisorData, config, ref powUsed, powBudget))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
if (gpuPool is not null)
|
||||
{
|
||||
pool.Return(gpuPool, clearArray: false);
|
||||
}
|
||||
|
||||
result = candidate;
|
||||
return true;
|
||||
|
||||
index += batchSize;
|
||||
}
|
||||
|
||||
if (gpuPool is not null)
|
||||
{
|
||||
pool.Return(gpuPool, clearArray: false);
|
||||
}
|
||||
|
||||
index += batchSize;
|
||||
|
||||
}
|
||||
finally
|
||||
{
|
||||
candidates.Clear();
|
||||
}
|
||||
|
||||
// DebugLog("No candidate confirmed");
|
||||
return false;
|
||||
}
|
||||
@@ -823,6 +834,21 @@ internal static partial class PrimeOrderCalculator
|
||||
return dictionary;
|
||||
}
|
||||
|
||||
private static List<ulong> AcquireHeuristicCandidateList(int capacityHint)
|
||||
{
|
||||
List<ulong>? list = s_heuristicCandidateList;
|
||||
if (list is null)
|
||||
{
|
||||
list = new List<ulong>(capacityHint);
|
||||
s_heuristicCandidateList = list;
|
||||
return list;
|
||||
}
|
||||
|
||||
list.Clear();
|
||||
list.EnsureCapacity(capacityHint);
|
||||
return list;
|
||||
}
|
||||
|
||||
private static PartialFactorResult PartialFactor(ulong value, in PrimeOrderSearchConfig config)
|
||||
{
|
||||
if (value <= 1UL)
|
||||
|
||||
@@ -346,48 +346,67 @@ public static class ULongExtensions
|
||||
|
||||
ulong result = divisor.MontgomeryOne;
|
||||
ulong nPrime = divisor.NPrime;
|
||||
ulong[] oddPowers = InitializeMontgomeryOddPowersCpu(divisor, modulus, nPrime, oddPowerCount);
|
||||
|
||||
const int StackOddPowerThreshold = PerfectNumberConstants.PooledArrayThreshold;
|
||||
Span<ulong> oddPowersSpan = oddPowerCount <= StackOddPowerThreshold
|
||||
? stackalloc ulong[StackOddPowerThreshold]
|
||||
: default;
|
||||
|
||||
Span<ulong> oddPowers;
|
||||
ulong[]? oddPowersArray = null;
|
||||
|
||||
if (!oddPowersSpan.IsEmpty)
|
||||
{
|
||||
oddPowers = oddPowersSpan.Slice(0, oddPowerCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
oddPowersArray = ThreadStaticPools.UlongPool.Rent(oddPowerCount);
|
||||
oddPowers = oddPowersArray.AsSpan(0, oddPowerCount);
|
||||
}
|
||||
|
||||
InitializeMontgomeryOddPowersCpu(divisor, modulus, nPrime, oddPowers);
|
||||
|
||||
int index = bitLength - 1;
|
||||
while (index >= 0)
|
||||
{
|
||||
if (((exponent >> index) & 1UL) == 0UL)
|
||||
{
|
||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||
index--;
|
||||
continue;
|
||||
}
|
||||
while (index >= 0)
|
||||
{
|
||||
if (((exponent >> index) & 1UL) == 0UL)
|
||||
{
|
||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||
index--;
|
||||
continue;
|
||||
}
|
||||
|
||||
int windowStart = index - windowSize + 1;
|
||||
if (windowStart < 0)
|
||||
{
|
||||
windowStart = 0;
|
||||
}
|
||||
int windowStart = index - windowSize + 1;
|
||||
if (windowStart < 0)
|
||||
{
|
||||
windowStart = 0;
|
||||
}
|
||||
|
||||
while (((exponent >> windowStart) & 1UL) == 0UL)
|
||||
{
|
||||
windowStart++;
|
||||
}
|
||||
while (((exponent >> windowStart) & 1UL) == 0UL)
|
||||
{
|
||||
windowStart++;
|
||||
}
|
||||
|
||||
int windowLength = index - windowStart + 1;
|
||||
for (int square = 0; square < windowLength; square++)
|
||||
{
|
||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||
}
|
||||
int windowLength = index - windowStart + 1;
|
||||
for (int square = 0; square < windowLength; square++)
|
||||
{
|
||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||
}
|
||||
|
||||
ulong mask = (1UL << windowLength) - 1UL;
|
||||
ulong windowValue = (exponent >> windowStart) & mask;
|
||||
int tableIndex = (int)((windowValue - 1UL) >> 1);
|
||||
ulong multiplier = oddPowers[tableIndex];
|
||||
result = result.MontgomeryMultiply(multiplier, modulus, nPrime);
|
||||
ulong mask = (1UL << windowLength) - 1UL;
|
||||
ulong windowValue = (exponent >> windowStart) & mask;
|
||||
int tableIndex = (int)((windowValue - 1UL) >> 1);
|
||||
ulong multiplier = oddPowers[tableIndex];
|
||||
result = result.MontgomeryMultiply(multiplier, modulus, nPrime);
|
||||
|
||||
index = windowStart - 1;
|
||||
}
|
||||
|
||||
if (oddPowerCount >= PerfectNumberConstants.PooledArrayThreshold)
|
||||
{
|
||||
ArrayPool<ulong>.Shared.Return(oddPowers);
|
||||
}
|
||||
index = windowStart - 1;
|
||||
}
|
||||
|
||||
if (oddPowersArray is not null)
|
||||
{
|
||||
ThreadStaticPools.UlongPool.Return(oddPowersArray, clearArray: false);
|
||||
}
|
||||
|
||||
if (keepMontgomery)
|
||||
{
|
||||
@@ -533,26 +552,22 @@ public static class ULongExtensions
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong[] InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, int oddPowerCount)
|
||||
private static void InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, Span<ulong> oddPowers)
|
||||
{
|
||||
ulong[] oddPowers = oddPowerCount < PerfectNumberConstants.PooledArrayThreshold ? new ulong[oddPowerCount] : ArrayPool<ulong>.Shared.Rent(oddPowerCount);
|
||||
|
||||
oddPowers[0] = divisor.MontgomeryTwo;
|
||||
if (oddPowerCount == 1)
|
||||
if (oddPowers.Length == 1)
|
||||
{
|
||||
return oddPowers;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong previous,
|
||||
square = divisor.MontgomeryTwoSquared;
|
||||
ulong previous;
|
||||
ulong square = divisor.MontgomeryTwoSquared;
|
||||
|
||||
for (int i = 1; i < oddPowerCount; i++)
|
||||
for (int i = 1; i < oddPowers.Length; i++)
|
||||
{
|
||||
previous = oddPowers[i - 1];
|
||||
oddPowers[i] = previous.MontgomeryMultiply(square, modulus, nPrime);
|
||||
}
|
||||
|
||||
return oddPowers;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
|
||||
Reference in new issue
Block a user