Reduce GC pressure in divisor checking pipeline
This commit is contained in:
1 parent
571b5a9938
commit
51730bcba3
2 files changed
+173
-132
No files matched your search
@@ -1,4 +1,5 @@
|
|||||||
using System.Buffers;
|
using System.Buffers;
|
||||||
|
using System.Collections.Generic;
|
||||||
using System.Buffers.Binary;
|
using System.Buffers.Binary;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Numerics;
|
using System.Numerics;
|
||||||
@@ -56,6 +57,9 @@ internal static partial class PrimeOrderCalculator
|
|||||||
[ThreadStatic]
|
[ThreadStatic]
|
||||||
private static Dictionary<ulong, int>? s_partialFactorCounts;
|
private static Dictionary<ulong, int>? s_partialFactorCounts;
|
||||||
|
|
||||||
|
[ThreadStatic]
|
||||||
|
private static List<ulong>? s_heuristicCandidateList;
|
||||||
|
|
||||||
private static bool IsGpuHeuristicDevice => s_pow2ModeInitialized && s_deviceMode == PrimeOrderHeuristicDevice.Gpu;
|
private static bool IsGpuHeuristicDevice => s_pow2ModeInitialized && s_deviceMode == PrimeOrderHeuristicDevice.Gpu;
|
||||||
|
|
||||||
private readonly struct Pow2ModeScope
|
private readonly struct Pow2ModeScope
|
||||||
@@ -402,105 +406,116 @@ internal static partial class PrimeOrderCalculator
|
|||||||
}
|
}
|
||||||
|
|
||||||
int capacity = config.MaxPowChecks <= 0 ? 64 : config.MaxPowChecks * 4;
|
int capacity = config.MaxPowChecks <= 0 ? 64 : config.MaxPowChecks * 4;
|
||||||
List<ulong> candidates = new(capacity);
|
List<ulong> candidates = AcquireHeuristicCandidateList(capacity);
|
||||||
FactorEntry[] factorArray = orderFactors.Factors!;
|
try
|
||||||
// DebugLog("Building candidates list");
|
|
||||||
BuildCandidates(order, factorArray, orderFactors.Count, candidates, capacity);
|
|
||||||
if (candidates.Count == 0)
|
|
||||||
{
|
{
|
||||||
return false;
|
FactorEntry[] factorArray = orderFactors.Factors!;
|
||||||
}
|
// DebugLog("Building candidates list");
|
||||||
|
BuildCandidates(order, factorArray, orderFactors.Count, candidates, capacity);
|
||||||
// DebugLog("Sorting candidates");
|
if (candidates.Count == 0)
|
||||||
SortCandidates(prime, previousOrder, candidates);
|
|
||||||
|
|
||||||
int powBudget = config.MaxPowChecks <= 0 ? candidates.Count : config.MaxPowChecks;
|
|
||||||
int powUsed = 0;
|
|
||||||
int candidateCount = candidates.Count;
|
|
||||||
bool allowGpuBatch = true;
|
|
||||||
Span<ulong> candidateSpan = CollectionsMarshal.AsSpan(candidates);
|
|
||||||
|
|
||||||
// DebugLog(() => $"Checking candidates ({candidateCount} candidates, {powBudget} pow budget)");
|
|
||||||
int index = 0;
|
|
||||||
const int MaxGpuBatchSize = 256;
|
|
||||||
const int StackGpuBatchSize = 64;
|
|
||||||
Span<ulong> stackGpuRemainders = stackalloc ulong[StackGpuBatchSize];
|
|
||||||
ArrayPool<ulong> pool = ThreadStaticPools.UlongPool;
|
|
||||||
while (index < candidateCount && powUsed < powBudget)
|
|
||||||
{
|
|
||||||
int remaining = candidateCount - index;
|
|
||||||
int budgetRemaining = powBudget - powUsed;
|
|
||||||
int batchSize = Math.Min(remaining, Math.Min(budgetRemaining, MaxGpuBatchSize));
|
|
||||||
if (batchSize <= 0)
|
|
||||||
{
|
{
|
||||||
break;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
ReadOnlySpan<ulong> batch = candidateSpan.Slice(index, batchSize);
|
// DebugLog("Sorting candidates");
|
||||||
ulong[]? gpuPool = null;
|
SortCandidates(prime, previousOrder, candidates);
|
||||||
Span<ulong> pooledGpuRemainders = default;
|
|
||||||
bool gpuSuccess = false;
|
|
||||||
bool gpuStackRemainders = false;
|
|
||||||
GpuPow2ModStatus status = GpuPow2ModStatus.Unavailable;
|
|
||||||
|
|
||||||
if (allowGpuBatch && IsGpuPow2Allowed)
|
int powBudget = config.MaxPowChecks <= 0 ? candidates.Count : config.MaxPowChecks;
|
||||||
|
int powUsed = 0;
|
||||||
|
int candidateCount = candidates.Count;
|
||||||
|
bool allowGpuBatch = true;
|
||||||
|
Span<ulong> candidateSpan = CollectionsMarshal.AsSpan(candidates);
|
||||||
|
|
||||||
|
// DebugLog(() => $"Checking candidates ({candidateCount} candidates, {powBudget} pow budget)");
|
||||||
|
int index = 0;
|
||||||
|
const int MaxGpuBatchSize = 256;
|
||||||
|
const int StackGpuBatchSize = 64;
|
||||||
|
Span<ulong> stackGpuRemainders = stackalloc ulong[StackGpuBatchSize];
|
||||||
|
ArrayPool<ulong> pool = ThreadStaticPools.UlongPool;
|
||||||
|
while (index < candidateCount && powUsed < powBudget)
|
||||||
{
|
{
|
||||||
if (batchSize <= StackGpuBatchSize)
|
int remaining = candidateCount - index;
|
||||||
|
int budgetRemaining = powBudget - powUsed;
|
||||||
|
int batchSize = Math.Min(remaining, Math.Min(budgetRemaining, MaxGpuBatchSize));
|
||||||
|
if (batchSize <= 0)
|
||||||
{
|
{
|
||||||
Span<ulong> localRemainders = stackGpuRemainders.Slice(0, batchSize);
|
break;
|
||||||
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, localRemainders, divisorData);
|
|
||||||
if (status == GpuPow2ModStatus.Success)
|
|
||||||
{
|
|
||||||
gpuSuccess = true;
|
|
||||||
gpuStackRemainders = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
ReadOnlySpan<ulong> batch = candidateSpan.Slice(index, batchSize);
|
||||||
|
ulong[]? gpuPool = null;
|
||||||
|
Span<ulong> pooledGpuRemainders = default;
|
||||||
|
bool gpuSuccess = false;
|
||||||
|
bool gpuStackRemainders = false;
|
||||||
|
GpuPow2ModStatus status = GpuPow2ModStatus.Unavailable;
|
||||||
|
|
||||||
|
if (allowGpuBatch && IsGpuPow2Allowed)
|
||||||
{
|
{
|
||||||
gpuPool = pool.Rent(batchSize);
|
if (batchSize <= StackGpuBatchSize)
|
||||||
Span<ulong> pooledRemainders = gpuPool.AsSpan(0, batchSize);
|
|
||||||
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, pooledRemainders, divisorData);
|
|
||||||
if (status == GpuPow2ModStatus.Success)
|
|
||||||
{
|
{
|
||||||
pooledGpuRemainders = pooledRemainders;
|
Span<ulong> localRemainders = stackGpuRemainders.Slice(0, batchSize);
|
||||||
gpuSuccess = true;
|
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, localRemainders, divisorData);
|
||||||
|
if (status == GpuPow2ModStatus.Success)
|
||||||
|
{
|
||||||
|
gpuSuccess = true;
|
||||||
|
gpuStackRemainders = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
pool.Return(gpuPool, clearArray: false);
|
gpuPool = pool.Rent(batchSize);
|
||||||
gpuPool = null;
|
Span<ulong> pooledRemainders = gpuPool.AsSpan(0, batchSize);
|
||||||
|
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, pooledRemainders, divisorData);
|
||||||
|
if (status == GpuPow2ModStatus.Success)
|
||||||
|
{
|
||||||
|
pooledGpuRemainders = pooledRemainders;
|
||||||
|
gpuSuccess = true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
pool.Return(gpuPool, clearArray: false);
|
||||||
|
gpuPool = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!gpuSuccess && (status == GpuPow2ModStatus.Overflow || status == GpuPow2ModStatus.Unavailable))
|
||||||
|
{
|
||||||
|
allowGpuBatch = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!gpuSuccess && (status == GpuPow2ModStatus.Overflow || status == GpuPow2ModStatus.Unavailable))
|
for (int i = 0; i < batchSize && powUsed < powBudget; i++)
|
||||||
{
|
{
|
||||||
allowGpuBatch = false;
|
ulong candidate = batch[i];
|
||||||
}
|
powUsed++;
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < batchSize && powUsed < powBudget; i++)
|
bool equalsOne;
|
||||||
{
|
if (gpuSuccess)
|
||||||
ulong candidate = batch[i];
|
{
|
||||||
powUsed++;
|
ulong remainderValue = gpuStackRemainders ? stackGpuRemainders[i] : pooledGpuRemainders[i];
|
||||||
|
equalsOne = remainderValue == 1UL;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
equalsOne = Pow2EqualsOneCpu(candidate, prime, divisorData);
|
||||||
|
}
|
||||||
|
if (!equalsOne)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
bool equalsOne;
|
if (!TryConfirmCandidateCpu(prime, candidate, divisorData, config, ref powUsed, powBudget))
|
||||||
if (gpuSuccess)
|
{
|
||||||
{
|
continue;
|
||||||
ulong remainderValue = gpuStackRemainders ? stackGpuRemainders[i] : pooledGpuRemainders[i];
|
}
|
||||||
equalsOne = remainderValue == 1UL;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
equalsOne = Pow2EqualsOneCpu(candidate, prime, divisorData);
|
|
||||||
}
|
|
||||||
if (!equalsOne)
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!TryConfirmCandidateCpu(prime, candidate, divisorData, config, ref powUsed, powBudget))
|
if (gpuPool is not null)
|
||||||
{
|
{
|
||||||
continue;
|
pool.Return(gpuPool, clearArray: false);
|
||||||
|
}
|
||||||
|
|
||||||
|
result = candidate;
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (gpuPool is not null)
|
if (gpuPool is not null)
|
||||||
@@ -508,18 +523,14 @@ internal static partial class PrimeOrderCalculator
|
|||||||
pool.Return(gpuPool, clearArray: false);
|
pool.Return(gpuPool, clearArray: false);
|
||||||
}
|
}
|
||||||
|
|
||||||
result = candidate;
|
index += batchSize;
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (gpuPool is not null)
|
|
||||||
{
|
|
||||||
pool.Return(gpuPool, clearArray: false);
|
|
||||||
}
|
|
||||||
|
|
||||||
index += batchSize;
|
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
candidates.Clear();
|
||||||
|
}
|
||||||
// DebugLog("No candidate confirmed");
|
// DebugLog("No candidate confirmed");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -823,6 +834,21 @@ internal static partial class PrimeOrderCalculator
|
|||||||
return dictionary;
|
return dictionary;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static List<ulong> AcquireHeuristicCandidateList(int capacityHint)
|
||||||
|
{
|
||||||
|
List<ulong>? list = s_heuristicCandidateList;
|
||||||
|
if (list is null)
|
||||||
|
{
|
||||||
|
list = new List<ulong>(capacityHint);
|
||||||
|
s_heuristicCandidateList = list;
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
list.Clear();
|
||||||
|
list.EnsureCapacity(capacityHint);
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
private static PartialFactorResult PartialFactor(ulong value, in PrimeOrderSearchConfig config)
|
private static PartialFactorResult PartialFactor(ulong value, in PrimeOrderSearchConfig config)
|
||||||
{
|
{
|
||||||
if (value <= 1UL)
|
if (value <= 1UL)
|
||||||
|
|||||||
@@ -346,48 +346,67 @@ public static class ULongExtensions
|
|||||||
|
|
||||||
ulong result = divisor.MontgomeryOne;
|
ulong result = divisor.MontgomeryOne;
|
||||||
ulong nPrime = divisor.NPrime;
|
ulong nPrime = divisor.NPrime;
|
||||||
ulong[] oddPowers = InitializeMontgomeryOddPowersCpu(divisor, modulus, nPrime, oddPowerCount);
|
|
||||||
|
const int StackOddPowerThreshold = PerfectNumberConstants.PooledArrayThreshold;
|
||||||
|
Span<ulong> oddPowersSpan = oddPowerCount <= StackOddPowerThreshold
|
||||||
|
? stackalloc ulong[StackOddPowerThreshold]
|
||||||
|
: default;
|
||||||
|
|
||||||
|
Span<ulong> oddPowers;
|
||||||
|
ulong[]? oddPowersArray = null;
|
||||||
|
|
||||||
|
if (!oddPowersSpan.IsEmpty)
|
||||||
|
{
|
||||||
|
oddPowers = oddPowersSpan.Slice(0, oddPowerCount);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
oddPowersArray = ThreadStaticPools.UlongPool.Rent(oddPowerCount);
|
||||||
|
oddPowers = oddPowersArray.AsSpan(0, oddPowerCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
InitializeMontgomeryOddPowersCpu(divisor, modulus, nPrime, oddPowers);
|
||||||
|
|
||||||
int index = bitLength - 1;
|
int index = bitLength - 1;
|
||||||
while (index >= 0)
|
while (index >= 0)
|
||||||
{
|
{
|
||||||
if (((exponent >> index) & 1UL) == 0UL)
|
if (((exponent >> index) & 1UL) == 0UL)
|
||||||
{
|
{
|
||||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||||
index--;
|
index--;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
int windowStart = index - windowSize + 1;
|
int windowStart = index - windowSize + 1;
|
||||||
if (windowStart < 0)
|
if (windowStart < 0)
|
||||||
{
|
{
|
||||||
windowStart = 0;
|
windowStart = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
while (((exponent >> windowStart) & 1UL) == 0UL)
|
while (((exponent >> windowStart) & 1UL) == 0UL)
|
||||||
{
|
{
|
||||||
windowStart++;
|
windowStart++;
|
||||||
}
|
}
|
||||||
|
|
||||||
int windowLength = index - windowStart + 1;
|
int windowLength = index - windowStart + 1;
|
||||||
for (int square = 0; square < windowLength; square++)
|
for (int square = 0; square < windowLength; square++)
|
||||||
{
|
{
|
||||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||||
}
|
}
|
||||||
|
|
||||||
ulong mask = (1UL << windowLength) - 1UL;
|
ulong mask = (1UL << windowLength) - 1UL;
|
||||||
ulong windowValue = (exponent >> windowStart) & mask;
|
ulong windowValue = (exponent >> windowStart) & mask;
|
||||||
int tableIndex = (int)((windowValue - 1UL) >> 1);
|
int tableIndex = (int)((windowValue - 1UL) >> 1);
|
||||||
ulong multiplier = oddPowers[tableIndex];
|
ulong multiplier = oddPowers[tableIndex];
|
||||||
result = result.MontgomeryMultiply(multiplier, modulus, nPrime);
|
result = result.MontgomeryMultiply(multiplier, modulus, nPrime);
|
||||||
|
|
||||||
index = windowStart - 1;
|
index = windowStart - 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (oddPowerCount >= PerfectNumberConstants.PooledArrayThreshold)
|
if (oddPowersArray is not null)
|
||||||
{
|
{
|
||||||
ArrayPool<ulong>.Shared.Return(oddPowers);
|
ThreadStaticPools.UlongPool.Return(oddPowersArray, clearArray: false);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (keepMontgomery)
|
if (keepMontgomery)
|
||||||
{
|
{
|
||||||
@@ -533,26 +552,22 @@ public static class ULongExtensions
|
|||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static ulong[] InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, int oddPowerCount)
|
private static void InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, Span<ulong> oddPowers)
|
||||||
{
|
{
|
||||||
ulong[] oddPowers = oddPowerCount < PerfectNumberConstants.PooledArrayThreshold ? new ulong[oddPowerCount] : ArrayPool<ulong>.Shared.Rent(oddPowerCount);
|
|
||||||
|
|
||||||
oddPowers[0] = divisor.MontgomeryTwo;
|
oddPowers[0] = divisor.MontgomeryTwo;
|
||||||
if (oddPowerCount == 1)
|
if (oddPowers.Length == 1)
|
||||||
{
|
{
|
||||||
return oddPowers;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
ulong previous,
|
ulong previous;
|
||||||
square = divisor.MontgomeryTwoSquared;
|
ulong square = divisor.MontgomeryTwoSquared;
|
||||||
|
|
||||||
for (int i = 1; i < oddPowerCount; i++)
|
for (int i = 1; i < oddPowers.Length; i++)
|
||||||
{
|
{
|
||||||
previous = oddPowers[i - 1];
|
previous = oddPowers[i - 1];
|
||||||
oddPowers[i] = previous.MontgomeryMultiply(square, modulus, nPrime);
|
oddPowers[i] = previous.MontgomeryMultiply(square, modulus, nPrime);
|
||||||
}
|
}
|
||||||
|
|
||||||
return oddPowers;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
|
|||||||
Reference in new issue
Block a user