Reduce GC pressure in divisor checking pipeline

This commit is contained in:
Marek Lemańczyk committed 2025-10-15 23:43:27 +02:00
1 parent 571b5a9938
commit 51730bcba3
2 files changed
+173 -132

No files matched your search

+114 -88
View File
@@ -1,4 +1,5 @@
using System.Buffers; using System.Buffers;
using System.Collections.Generic;
using System.Buffers.Binary; using System.Buffers.Binary;
using System.Diagnostics; using System.Diagnostics;
using System.Numerics; using System.Numerics;
@@ -56,6 +57,9 @@ internal static partial class PrimeOrderCalculator
[ThreadStatic] [ThreadStatic]
private static Dictionary<ulong, int>? s_partialFactorCounts; private static Dictionary<ulong, int>? s_partialFactorCounts;
[ThreadStatic]
private static List<ulong>? s_heuristicCandidateList;
private static bool IsGpuHeuristicDevice => s_pow2ModeInitialized && s_deviceMode == PrimeOrderHeuristicDevice.Gpu; private static bool IsGpuHeuristicDevice => s_pow2ModeInitialized && s_deviceMode == PrimeOrderHeuristicDevice.Gpu;
private readonly struct Pow2ModeScope private readonly struct Pow2ModeScope
@@ -402,105 +406,116 @@ internal static partial class PrimeOrderCalculator
} }
int capacity = config.MaxPowChecks <= 0 ? 64 : config.MaxPowChecks * 4; int capacity = config.MaxPowChecks <= 0 ? 64 : config.MaxPowChecks * 4;
List<ulong> candidates = new(capacity); List<ulong> candidates = AcquireHeuristicCandidateList(capacity);
FactorEntry[] factorArray = orderFactors.Factors!; try
// DebugLog("Building candidates list");
BuildCandidates(order, factorArray, orderFactors.Count, candidates, capacity);
if (candidates.Count == 0)
{ {
return false; FactorEntry[] factorArray = orderFactors.Factors!;
} // DebugLog("Building candidates list");
BuildCandidates(order, factorArray, orderFactors.Count, candidates, capacity);
// DebugLog("Sorting candidates"); if (candidates.Count == 0)
SortCandidates(prime, previousOrder, candidates);
int powBudget = config.MaxPowChecks <= 0 ? candidates.Count : config.MaxPowChecks;
int powUsed = 0;
int candidateCount = candidates.Count;
bool allowGpuBatch = true;
Span<ulong> candidateSpan = CollectionsMarshal.AsSpan(candidates);
// DebugLog(() => $"Checking candidates ({candidateCount} candidates, {powBudget} pow budget)");
int index = 0;
const int MaxGpuBatchSize = 256;
const int StackGpuBatchSize = 64;
Span<ulong> stackGpuRemainders = stackalloc ulong[StackGpuBatchSize];
ArrayPool<ulong> pool = ThreadStaticPools.UlongPool;
while (index < candidateCount && powUsed < powBudget)
{
int remaining = candidateCount - index;
int budgetRemaining = powBudget - powUsed;
int batchSize = Math.Min(remaining, Math.Min(budgetRemaining, MaxGpuBatchSize));
if (batchSize <= 0)
{ {
break; return false;
} }
ReadOnlySpan<ulong> batch = candidateSpan.Slice(index, batchSize); // DebugLog("Sorting candidates");
ulong[]? gpuPool = null; SortCandidates(prime, previousOrder, candidates);
Span<ulong> pooledGpuRemainders = default;
bool gpuSuccess = false;
bool gpuStackRemainders = false;
GpuPow2ModStatus status = GpuPow2ModStatus.Unavailable;
if (allowGpuBatch && IsGpuPow2Allowed) int powBudget = config.MaxPowChecks <= 0 ? candidates.Count : config.MaxPowChecks;
int powUsed = 0;
int candidateCount = candidates.Count;
bool allowGpuBatch = true;
Span<ulong> candidateSpan = CollectionsMarshal.AsSpan(candidates);
// DebugLog(() => $"Checking candidates ({candidateCount} candidates, {powBudget} pow budget)");
int index = 0;
const int MaxGpuBatchSize = 256;
const int StackGpuBatchSize = 64;
Span<ulong> stackGpuRemainders = stackalloc ulong[StackGpuBatchSize];
ArrayPool<ulong> pool = ThreadStaticPools.UlongPool;
while (index < candidateCount && powUsed < powBudget)
{ {
if (batchSize <= StackGpuBatchSize) int remaining = candidateCount - index;
int budgetRemaining = powBudget - powUsed;
int batchSize = Math.Min(remaining, Math.Min(budgetRemaining, MaxGpuBatchSize));
if (batchSize <= 0)
{ {
Span<ulong> localRemainders = stackGpuRemainders.Slice(0, batchSize); break;
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, localRemainders, divisorData);
if (status == GpuPow2ModStatus.Success)
{
gpuSuccess = true;
gpuStackRemainders = true;
}
} }
else
ReadOnlySpan<ulong> batch = candidateSpan.Slice(index, batchSize);
ulong[]? gpuPool = null;
Span<ulong> pooledGpuRemainders = default;
bool gpuSuccess = false;
bool gpuStackRemainders = false;
GpuPow2ModStatus status = GpuPow2ModStatus.Unavailable;
if (allowGpuBatch && IsGpuPow2Allowed)
{ {
gpuPool = pool.Rent(batchSize); if (batchSize <= StackGpuBatchSize)
Span<ulong> pooledRemainders = gpuPool.AsSpan(0, batchSize);
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, pooledRemainders, divisorData);
if (status == GpuPow2ModStatus.Success)
{ {
pooledGpuRemainders = pooledRemainders; Span<ulong> localRemainders = stackGpuRemainders.Slice(0, batchSize);
gpuSuccess = true; status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, localRemainders, divisorData);
if (status == GpuPow2ModStatus.Success)
{
gpuSuccess = true;
gpuStackRemainders = true;
}
} }
else else
{ {
pool.Return(gpuPool, clearArray: false); gpuPool = pool.Rent(batchSize);
gpuPool = null; Span<ulong> pooledRemainders = gpuPool.AsSpan(0, batchSize);
status = PrimeOrderGpuHeuristics.TryPow2ModBatch(batch, prime, pooledRemainders, divisorData);
if (status == GpuPow2ModStatus.Success)
{
pooledGpuRemainders = pooledRemainders;
gpuSuccess = true;
}
else
{
pool.Return(gpuPool, clearArray: false);
gpuPool = null;
}
}
if (!gpuSuccess && (status == GpuPow2ModStatus.Overflow || status == GpuPow2ModStatus.Unavailable))
{
allowGpuBatch = false;
} }
} }
if (!gpuSuccess && (status == GpuPow2ModStatus.Overflow || status == GpuPow2ModStatus.Unavailable)) for (int i = 0; i < batchSize && powUsed < powBudget; i++)
{ {
allowGpuBatch = false; ulong candidate = batch[i];
} powUsed++;
}
for (int i = 0; i < batchSize && powUsed < powBudget; i++) bool equalsOne;
{ if (gpuSuccess)
ulong candidate = batch[i]; {
powUsed++; ulong remainderValue = gpuStackRemainders ? stackGpuRemainders[i] : pooledGpuRemainders[i];
equalsOne = remainderValue == 1UL;
}
else
{
equalsOne = Pow2EqualsOneCpu(candidate, prime, divisorData);
}
if (!equalsOne)
{
continue;
}
bool equalsOne; if (!TryConfirmCandidateCpu(prime, candidate, divisorData, config, ref powUsed, powBudget))
if (gpuSuccess) {
{ continue;
ulong remainderValue = gpuStackRemainders ? stackGpuRemainders[i] : pooledGpuRemainders[i]; }
equalsOne = remainderValue == 1UL;
}
else
{
equalsOne = Pow2EqualsOneCpu(candidate, prime, divisorData);
}
if (!equalsOne)
{
continue;
}
if (!TryConfirmCandidateCpu(prime, candidate, divisorData, config, ref powUsed, powBudget)) if (gpuPool is not null)
{ {
continue; pool.Return(gpuPool, clearArray: false);
}
result = candidate;
return true;
} }
if (gpuPool is not null) if (gpuPool is not null)
@@ -508,18 +523,14 @@ internal static partial class PrimeOrderCalculator
pool.Return(gpuPool, clearArray: false); pool.Return(gpuPool, clearArray: false);
} }
result = candidate; index += batchSize;
return true;
} }
if (gpuPool is not null)
{
pool.Return(gpuPool, clearArray: false);
}
index += batchSize;
} }
finally
{
candidates.Clear();
}
// DebugLog("No candidate confirmed"); // DebugLog("No candidate confirmed");
return false; return false;
} }
@@ -823,6 +834,21 @@ internal static partial class PrimeOrderCalculator
return dictionary; return dictionary;
} }
private static List<ulong> AcquireHeuristicCandidateList(int capacityHint)
{
List<ulong>? list = s_heuristicCandidateList;
if (list is null)
{
list = new List<ulong>(capacityHint);
s_heuristicCandidateList = list;
return list;
}
list.Clear();
list.EnsureCapacity(capacityHint);
return list;
}
private static PartialFactorResult PartialFactor(ulong value, in PrimeOrderSearchConfig config) private static PartialFactorResult PartialFactor(ulong value, in PrimeOrderSearchConfig config)
{ {
if (value <= 1UL) if (value <= 1UL)
+59 -44
View File
@@ -346,48 +346,67 @@ public static class ULongExtensions
ulong result = divisor.MontgomeryOne; ulong result = divisor.MontgomeryOne;
ulong nPrime = divisor.NPrime; ulong nPrime = divisor.NPrime;
ulong[] oddPowers = InitializeMontgomeryOddPowersCpu(divisor, modulus, nPrime, oddPowerCount);
const int StackOddPowerThreshold = PerfectNumberConstants.PooledArrayThreshold;
Span<ulong> oddPowersSpan = oddPowerCount <= StackOddPowerThreshold
? stackalloc ulong[StackOddPowerThreshold]
: default;
Span<ulong> oddPowers;
ulong[]? oddPowersArray = null;
if (!oddPowersSpan.IsEmpty)
{
oddPowers = oddPowersSpan.Slice(0, oddPowerCount);
}
else
{
oddPowersArray = ThreadStaticPools.UlongPool.Rent(oddPowerCount);
oddPowers = oddPowersArray.AsSpan(0, oddPowerCount);
}
InitializeMontgomeryOddPowersCpu(divisor, modulus, nPrime, oddPowers);
int index = bitLength - 1; int index = bitLength - 1;
while (index >= 0) while (index >= 0)
{ {
if (((exponent >> index) & 1UL) == 0UL) if (((exponent >> index) & 1UL) == 0UL)
{ {
result = result.MontgomeryMultiply(result, modulus, nPrime); result = result.MontgomeryMultiply(result, modulus, nPrime);
index--; index--;
continue; continue;
} }
int windowStart = index - windowSize + 1; int windowStart = index - windowSize + 1;
if (windowStart < 0) if (windowStart < 0)
{ {
windowStart = 0; windowStart = 0;
} }
while (((exponent >> windowStart) & 1UL) == 0UL) while (((exponent >> windowStart) & 1UL) == 0UL)
{ {
windowStart++; windowStart++;
} }
int windowLength = index - windowStart + 1; int windowLength = index - windowStart + 1;
for (int square = 0; square < windowLength; square++) for (int square = 0; square < windowLength; square++)
{ {
result = result.MontgomeryMultiply(result, modulus, nPrime); result = result.MontgomeryMultiply(result, modulus, nPrime);
} }
ulong mask = (1UL << windowLength) - 1UL; ulong mask = (1UL << windowLength) - 1UL;
ulong windowValue = (exponent >> windowStart) & mask; ulong windowValue = (exponent >> windowStart) & mask;
int tableIndex = (int)((windowValue - 1UL) >> 1); int tableIndex = (int)((windowValue - 1UL) >> 1);
ulong multiplier = oddPowers[tableIndex]; ulong multiplier = oddPowers[tableIndex];
result = result.MontgomeryMultiply(multiplier, modulus, nPrime); result = result.MontgomeryMultiply(multiplier, modulus, nPrime);
index = windowStart - 1; index = windowStart - 1;
} }
if (oddPowerCount >= PerfectNumberConstants.PooledArrayThreshold) if (oddPowersArray is not null)
{ {
ArrayPool<ulong>.Shared.Return(oddPowers); ThreadStaticPools.UlongPool.Return(oddPowersArray, clearArray: false);
} }
if (keepMontgomery) if (keepMontgomery)
{ {
@@ -533,26 +552,22 @@ public static class ULongExtensions
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ulong[] InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, int oddPowerCount) private static void InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, Span<ulong> oddPowers)
{ {
ulong[] oddPowers = oddPowerCount < PerfectNumberConstants.PooledArrayThreshold ? new ulong[oddPowerCount] : ArrayPool<ulong>.Shared.Rent(oddPowerCount);
oddPowers[0] = divisor.MontgomeryTwo; oddPowers[0] = divisor.MontgomeryTwo;
if (oddPowerCount == 1) if (oddPowers.Length == 1)
{ {
return oddPowers; return;
} }
ulong previous, ulong previous;
square = divisor.MontgomeryTwoSquared; ulong square = divisor.MontgomeryTwoSquared;
for (int i = 1; i < oddPowerCount; i++) for (int i = 1; i < oddPowers.Length; i++)
{ {
previous = oddPowers[i - 1]; previous = oddPowers[i - 1];
oddPowers[i] = previous.MontgomeryMultiply(square, modulus, nPrime); oddPowers[i] = previous.MontgomeryMultiply(square, modulus, nPrime);
} }
return oddPowers;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]