Fuse GPU by-divisor candidate filtering into one kernel

This commit is contained in:
Marek Lemańczyk committed 2025-10-24 01:23:25 +02:00
1 parent ab1a069e57
commit 40d3b4b3ac
3 files changed
+64 -196

No files matched your search

@@ -78,13 +78,9 @@ public class GpuKernelCompilationTests
typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetStepperKernel", BindingFlags.NonPublic | BindingFlags.Instance) typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetStepperKernel", BindingFlags.NonPublic | BindingFlags.Instance)
?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetKernelByPrimeExponent not found."); ?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetKernelByPrimeExponent not found.");
private static readonly MethodInfo DivisorByDivisorGetCandidateMaskKernelMethod = private static readonly MethodInfo DivisorByDivisorGetFilteredCandidateKernelMethod =
typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetCandidateMaskKernel", BindingFlags.NonPublic | BindingFlags.Instance) typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetFilteredCandidateKernel", BindingFlags.NonPublic | BindingFlags.Instance)
?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetCandidateMaskKernel not found."); ?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetFilteredCandidateKernel not found.");
private static readonly MethodInfo DivisorByDivisorGetCandidateGenerationKernelMethod =
typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetCandidateGenerationKernel", BindingFlags.NonPublic | BindingFlags.Instance)
?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetCandidateGenerationKernel not found.");
private static readonly MethodInfo DivisorTesterGetKernelMethod = private static readonly MethodInfo DivisorTesterGetKernelMethod =
typeof(MersenneNumberDivisorGpuTester).GetMethod("GetKernel", BindingFlags.NonPublic | BindingFlags.Instance) typeof(MersenneNumberDivisorGpuTester).GetMethod("GetKernel", BindingFlags.NonPublic | BindingFlags.Instance)
@@ -158,8 +154,7 @@ public class GpuKernelCompilationTests
yield return Loader("NttGpuMath.InverseKernel", CompileNttInverseKernel); yield return Loader("NttGpuMath.InverseKernel", CompileNttInverseKernel);
yield return Loader("MersenneNumberDivisorByDivisorGpuTester.CheckKernel", CompileDivisorByDivisorCheckKernel); yield return Loader("MersenneNumberDivisorByDivisorGpuTester.CheckKernel", CompileDivisorByDivisorCheckKernel);
yield return Loader("MersenneNumberDivisorByDivisorGpuTester.EvaluateDivisorWithStepperKernel", CompileDivisorByDivisorExponentKernel); yield return Loader("MersenneNumberDivisorByDivisorGpuTester.EvaluateDivisorWithStepperKernel", CompileDivisorByDivisorExponentKernel);
yield return Loader("MersenneNumberDivisorByDivisorGpuTester.EvaluateCandidateMaskKernel", CompileDivisorByDivisorCandidateMaskKernel); yield return Loader("MersenneNumberDivisorByDivisorGpuTester.GenerateFilteredCandidatesKernel", CompileDivisorByDivisorFilteredCandidateKernel);
yield return Loader("MersenneNumberDivisorByDivisorGpuTester.GenerateCandidatesKernel", CompileDivisorByDivisorCandidateGenerationKernel);
yield return Loader("MersenneNumberDivisorGpuTester.Kernel", CompileDivisorTesterKernel); yield return Loader("MersenneNumberDivisorGpuTester.Kernel", CompileDivisorTesterKernel);
yield return Loader("MersenneNumberLucasLehmerGpuTester.Kernel", CompileLucasLehmerKernel); yield return Loader("MersenneNumberLucasLehmerGpuTester.Kernel", CompileLucasLehmerKernel);
yield return Loader("MersenneNumberLucasLehmerGpuTester.AddSmallKernel", CompileLucasLehmerAddSmallKernel); yield return Loader("MersenneNumberLucasLehmerGpuTester.AddSmallKernel", CompileLucasLehmerAddSmallKernel);
@@ -287,16 +282,10 @@ public class GpuKernelCompilationTests
_ = DivisorByDivisorGetExponentKernelMethod.Invoke(tester, new object[] { accelerator }); _ = DivisorByDivisorGetExponentKernelMethod.Invoke(tester, new object[] { accelerator });
} }
private static void CompileDivisorByDivisorCandidateMaskKernel(Accelerator accelerator) private static void CompileDivisorByDivisorFilteredCandidateKernel(Accelerator accelerator)
{ {
var tester = new MersenneNumberDivisorByDivisorGpuTester(); var tester = new MersenneNumberDivisorByDivisorGpuTester();
_ = DivisorByDivisorGetCandidateMaskKernelMethod.Invoke(tester, new object[] { accelerator }); _ = DivisorByDivisorGetFilteredCandidateKernelMethod.Invoke(tester, new object[] { accelerator });
}
private static void CompileDivisorByDivisorCandidateGenerationKernel(Accelerator accelerator)
{
var tester = new MersenneNumberDivisorByDivisorGpuTester();
_ = DivisorByDivisorGetCandidateGenerationKernelMethod.Invoke(tester, new object[] { accelerator });
} }
private static void CompileDivisorTesterKernel(Accelerator accelerator) private static void CompileDivisorTesterKernel(Accelerator accelerator)
@@ -74,63 +74,37 @@ internal static class DivisorByDivisorKernels
} }
} }
public static void GenerateCandidatesKernel( public static void GenerateFilteredCandidatesKernel(
Index1D index, Index1D index,
ArrayView<ulong> candidates, ArrayView<ulong> candidates,
ArrayView<int> filteredCount,
ArrayView<GpuUInt128> nextStarts, ArrayView<GpuUInt128> nextStarts,
GpuUInt128 startValue, GpuUInt128 startValue,
GpuUInt128 stride) GpuUInt128 stride,
byte lastIsSevenFlag)
{ {
int globalIndex = index; int globalIndex = index;
// EvenPerfectBitScanner always launches this kernel with an extent matching the candidate view length, int localIndex = Group.IdxX;
// so the bounds guard stays commented out to keep the hot path branch-free. int groupSize = Group.Dimension.X;
// int candidateLength = (int)candidates.Length;
// if (globalIndex >= candidateLength)
// {
// return;
// }
ulong offset = (ulong)globalIndex; ulong candidateOffset = (ulong)globalIndex;
GpuUInt128 candidateValue = stride; GpuUInt128 candidateValue = stride;
candidateValue.Mul(offset); candidateValue.Mul(candidateOffset);
candidateValue.Add(startValue); candidateValue.Add(startValue);
candidates[globalIndex] = candidateValue.Low; ulong candidate = candidateValue.Low;
candidates[globalIndex] = candidate;
if (globalIndex == 0) if (globalIndex == 0)
{ {
// EvenPerfectBitScanner provides a single-element nextStarts view per launch, so the guard stays commented out. ulong candidateCount = (ulong)groupSize;
// int nextStartLength = (int)nextStarts.Length;
// if (nextStartLength <= 0)
// {
// return;
// }
ulong candidateCount = (ulong)candidates.Length;
GpuUInt128 nextValue = stride; GpuUInt128 nextValue = stride;
nextValue.Mul(candidateCount); nextValue.Mul(candidateCount);
nextValue.Add(startValue); nextValue.Add(startValue);
nextStarts[0] = nextValue; nextStarts[0] = nextValue;
} }
}
public static void EvaluateCandidateMaskKernel(
Index1D index,
ArrayView<ulong> candidates,
ArrayView<byte> mask,
byte lastIsSevenFlag)
{
int globalIndex = index;
// EvenPerfectBitScanner launches this auto-grouped kernel with an extent matching the mask length,
// so the defensive guard stays commented out to keep the kernel branch-free.
// int length = (int)mask.Length;
// if (globalIndex >= length)
// {
// return;
// }
ulong candidate = candidates[globalIndex];
byte remainder10 = (byte)(candidate % 10UL);
bool lastIsSeven = lastIsSevenFlag != 0; bool lastIsSeven = lastIsSevenFlag != 0;
byte remainder10 = (byte)(candidate % 10UL);
bool acceptSeven = remainder10 == 3 || remainder10 == 7 || remainder10 == 9; bool acceptSeven = remainder10 == 3 || remainder10 == 7 || remainder10 == 9;
bool acceptNonSeven = remainder10 == 1 || remainder10 == 3 || remainder10 == 7 || remainder10 == 9; bool acceptNonSeven = remainder10 == 1 || remainder10 == 3 || remainder10 == 7 || remainder10 == 9;
bool accept10 = lastIsSeven ? acceptSeven : acceptNonSeven; bool accept10 = lastIsSeven ? acceptSeven : acceptNonSeven;
@@ -140,60 +114,36 @@ internal static class DivisorByDivisorKernels
bool accept5 = remainder5 != 0; bool accept5 = remainder5 != 0;
byte remainder3 = (byte)(candidate % 3UL); byte remainder3 = (byte)(candidate % 3UL);
bool accept3 = remainder3 != 0; bool accept3 = remainder3 != 0;
byte accepted = (byte)((accept10 && accept8 && accept3 && accept5) ? 1 : 0); int accepted = (accept10 && accept8 && accept3 && accept5) ? 1 : 0;
mask[globalIndex] = accepted;
}
public static void CompactCandidateMaskKernel(
Index1D index,
ArrayView<byte> mask,
ArrayView<ulong> sourceCandidates,
ArrayView<ulong> compactedCandidates,
ArrayView<int> compactedCount)
{
int globalIndex = index;
int length = (int)mask.Length;
// EvenPerfectBitScanner launches this stream kernel with an extent matching the mask length,
// so the defensive guard stays commented out to keep the kernel branch-free.
// if (globalIndex >= length)
// {
// return;
// }
int localIndex = Group.IdxX;
int groupSize = Group.Dimension.X;
var shared = SharedMemory.GetDynamic<int>(); var shared = SharedMemory.GetDynamic<int>();
int length = groupSize;
int prefixBase = 0; int prefixBase = 0;
int mappingBase = length; int mappingBase = length;
int fallbackBase = length * 2; int fallbackBase = length * 2;
// The dynamic shared-memory slice stores prefix sums, compacted index mappings, and fallback slots
// in separate regions so the scatter stage can avoid divergent branches.
byte maskValue = mask[globalIndex];
int accepted = maskValue != 0 ? 1 : 0;
shared[prefixBase + localIndex] = accepted; shared[prefixBase + localIndex] = accepted;
shared[fallbackBase + localIndex] = globalIndex; shared[fallbackBase + localIndex] = globalIndex;
Group.Barrier(); Group.Barrier();
int offset = 1; int scanOffset = 1;
while (offset < groupSize) while (scanOffset < groupSize)
{ {
int addendIndex = prefixBase + localIndex - offset; int addendIndex = prefixBase + localIndex - scanOffset;
int addend = localIndex >= offset ? shared[addendIndex] : 0; int addend = localIndex >= scanOffset ? shared[addendIndex] : 0;
Group.Barrier(); Group.Barrier();
int currentIndex = prefixBase + localIndex; int currentIndex = prefixBase + localIndex;
int current = shared[currentIndex]; int current = shared[currentIndex];
int sum = current + addend; int sum = current + addend;
int shouldUpdate = localIndex >= offset ? 1 : 0; int shouldUpdate = localIndex >= scanOffset ? 1 : 0;
shared[currentIndex] = shouldUpdate != 0 ? sum : current; shared[currentIndex] = shouldUpdate != 0 ? sum : current;
Group.Barrier(); Group.Barrier();
offset <<= 1; scanOffset <<= 1;
} }
int inclusivePrefix = shared[prefixBase + localIndex]; int inclusivePrefix = shared[prefixBase + localIndex];
@@ -209,16 +159,17 @@ internal static class DivisorByDivisorKernels
int lookupBase = writerMask != 0 ? mappingBase : fallbackBase; int lookupBase = writerMask != 0 ? mappingBase : fallbackBase;
int sourceIndex = shared[lookupBase + localIndex]; int sourceIndex = shared[lookupBase + localIndex];
ulong candidateValue = sourceCandidates[sourceIndex]; ulong candidateValueToStore = candidates[sourceIndex];
int destinationIndex = localIndex; if (writerMask != 0)
ulong existingValue = compactedCandidates[destinationIndex]; {
compactedCandidates[destinationIndex] = writerMask != 0 ? candidateValue : existingValue; candidates[localIndex] = candidateValueToStore;
}
int lastLaneMask = globalIndex == length - 1 ? 1 : 0; int lastLaneMask = globalIndex == length - 1 ? 1 : 0;
int countValue = shared[prefixBase + localIndex]; int countValue = shared[prefixBase + localIndex];
if (lastLaneMask != 0) if (lastLaneMask != 0)
{ {
compactedCount[0] = countValue; filteredCount[0] = countValue;
} }
} }
@@ -25,10 +25,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<MontgomeryDivisorData>, ArrayView<ulong>, ArrayView<byte>>> _kernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<MontgomeryDivisorData>, ArrayView<ulong>, ArrayView<byte>>> _kernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, MontgomeryDivisorData, ulong, ulong, ArrayView<ulong>, ArrayView<byte>>> _stepperKernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary<Accelerator, Action<Index1D, MontgomeryDivisorData, ulong, ulong, ArrayView<ulong>, ArrayView<byte>>> _stepperKernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte>> _candidateMaskKernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary<Accelerator, Action<KernelConfig, Index1D, ArrayView<ulong>, ArrayView<int>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128, byte>> _filteredCandidateKernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<int>>> _candidateCompactionKernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<byte>, ArrayView<int>>> _hitDetectionKernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<byte>, ArrayView<int>>> _hitDetectionKernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> _candidateGenerationKernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, ConcurrentBag<BatchResources>> _resourcePools = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary<Accelerator, ConcurrentBag<BatchResources>> _resourcePools = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentBag<GpuContextPool.GpuContextLease> _acceleratorPool = new(); private readonly ConcurrentBag<GpuContextPool.GpuContextLease> _acceleratorPool = new();
private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new(); private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new();
@@ -39,26 +37,16 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private Action<Index1D, MontgomeryDivisorData, ulong, ulong, ArrayView<ulong>, ArrayView<byte>> GetStepperKernel(Accelerator accelerator) => private Action<Index1D, MontgomeryDivisorData, ulong, ulong, ArrayView<ulong>, ArrayView<byte>> GetStepperKernel(Accelerator accelerator) =>
_stepperKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, MontgomeryDivisorData, ulong, ulong, ArrayView<ulong>, ArrayView<byte>>(DivisorByDivisorKernels.EvaluateDivisorWithStepperKernel)); _stepperKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, MontgomeryDivisorData, ulong, ulong, ArrayView<ulong>, ArrayView<byte>>(DivisorByDivisorKernels.EvaluateDivisorWithStepperKernel));
private Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte> GetCandidateMaskKernel(Accelerator accelerator) => private Action<KernelConfig, Index1D, ArrayView<ulong>, ArrayView<int>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128, byte> GetFilteredCandidateKernel(Accelerator accelerator) =>
_candidateMaskKernelCache.GetOrAdd( _filteredCandidateKernelCache.GetOrAdd(
accelerator, accelerator,
acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>, byte>(DivisorByDivisorKernels.EvaluateCandidateMaskKernel)); acc => acc.LoadStreamKernel<Index1D, ArrayView<ulong>, ArrayView<int>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128, byte>(DivisorByDivisorKernels.GenerateFilteredCandidatesKernel));
private Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<int>> GetCandidateCompactionKernel(Accelerator accelerator) =>
_candidateCompactionKernelCache.GetOrAdd(
accelerator,
acc => acc.LoadStreamKernel<Index1D, ArrayView<byte>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<int>>(DivisorByDivisorKernels.CompactCandidateMaskKernel));
private Action<Index1D, ArrayView<byte>, ArrayView<int>> GetHitDetectionKernel(Accelerator accelerator) => private Action<Index1D, ArrayView<byte>, ArrayView<int>> GetHitDetectionKernel(Accelerator accelerator) =>
_hitDetectionKernelCache.GetOrAdd( _hitDetectionKernelCache.GetOrAdd(
accelerator, accelerator,
acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.PublishFirstHitKernel)); acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.PublishFirstHitKernel));
private Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetCandidateGenerationKernel(Accelerator accelerator) =>
_candidateGenerationKernelCache.GetOrAdd(
accelerator,
acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(DivisorByDivisorKernels.GenerateCandidatesKernel));
public int GpuBatchSize public int GpuBatchSize
{ {
get => _gpuBatchSize; get => _gpuBatchSize;
@@ -137,14 +125,11 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
resources.CandidateBuffer, resources.CandidateBuffer,
resources.NextStartBuffer, resources.NextStartBuffer,
resources.FilterCountBuffer, resources.FilterCountBuffer,
resources.CandidateMaskKernel, resources.FilteredCandidateKernel,
resources.CandidateCompactionKernel,
resources.CandidateGenerationKernel,
resources.HitDetectionKernel, resources.HitDetectionKernel,
resources.Divisors, resources.Divisors,
resources.Exponents, resources.Exponents,
resources.DivisorData, resources.DivisorData,
resources.CycleCandidates,
resources.CycleLengths, resources.CycleLengths,
resources.CycleCapacity, resources.CycleCapacity,
out lastProcessed, out lastProcessed,
@@ -206,14 +191,11 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
MemoryBuffer1D<ulong, Stride1D.Dense> candidateBuffer, MemoryBuffer1D<ulong, Stride1D.Dense> candidateBuffer,
MemoryBuffer1D<GpuUInt128, Stride1D.Dense> nextStartBuffer, MemoryBuffer1D<GpuUInt128, Stride1D.Dense> nextStartBuffer,
MemoryBuffer1D<int, Stride1D.Dense> filterCountBuffer, MemoryBuffer1D<int, Stride1D.Dense> filterCountBuffer,
Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte> candidateMaskKernel, Action<KernelConfig, Index1D, ArrayView<ulong>, ArrayView<int>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128, byte> filteredCandidateKernel,
Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<int>> candidateCompactionKernel,
Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> candidateGenerationKernel,
Action<Index1D, ArrayView<byte>, ArrayView<int>> hitDetectionKernel, Action<Index1D, ArrayView<byte>, ArrayView<int>> hitDetectionKernel,
in ulong[] divisors, in ulong[] divisors,
ulong[] exponents, ulong[] exponents,
MontgomeryDivisorData[] divisorData, MontgomeryDivisorData[] divisorData,
ulong[] cycleCandidates,
ulong[] cycleLengths, ulong[] cycleLengths,
int cycleCapacity, int cycleCapacity,
out ulong lastProcessed, out ulong lastProcessed,
@@ -257,7 +239,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ArrayPool<ulong> ulongPool = ThreadStaticPools.UlongPool; ArrayPool<ulong> ulongPool = ThreadStaticPools.UlongPool;
ulong[] filteredDivisors = ulongPool.Rent(chunkCapacity); ulong[] filteredDivisors = ulongPool.Rent(chunkCapacity);
Span<ulong> candidateStorage = cycleCandidates.AsSpan();
Span<ulong> filteredStorage = filteredDivisors.AsSpan(); Span<ulong> filteredStorage = filteredDivisors.AsSpan();
Span<ulong> cycleLengthStorage = cycleLengths.AsSpan(); Span<ulong> cycleLengthStorage = cycleLengths.AsSpan();
Span<MontgomeryDivisorData> divisorDataStorage = divisorData.AsSpan(); Span<MontgomeryDivisorData> divisorDataStorage = divisorData.AsSpan();
@@ -278,38 +259,33 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
break; break;
} }
Span<ulong> candidateSpan = candidateStorage[..effectiveCount];
ArrayView1D<ulong, Stride1D.Dense> candidateView = candidateBuffer.View.SubView(0, effectiveCount); ArrayView1D<ulong, Stride1D.Dense> candidateView = candidateBuffer.View.SubView(0, effectiveCount);
ArrayView1D<GpuUInt128, Stride1D.Dense> nextStartView = nextStartBuffer.View.SubView(0, 1); ArrayView1D<GpuUInt128, Stride1D.Dense> nextStartView = nextStartBuffer.View.SubView(0, 1);
ArrayView1D<int, Stride1D.Dense> filteredCountSubView = filteredCountView.SubView(0, 1);
GpuUInt128 gpuStartValue = new GpuUInt128(currentDivisor128); GpuUInt128 gpuStartValue = new GpuUInt128(currentDivisor128);
GpuUInt128 gpuStride = new GpuUInt128(twoP128); GpuUInt128 gpuStride = new GpuUInt128(twoP128);
GpuUInt128 nextDivisorGpu = GenerateCandidatesOnGpu( GpuUInt128 nextDivisorGpu = default;
int filteredCount = GenerateFilteredCandidatesOnGpu(
effectiveCount, effectiveCount,
gpuStartValue, gpuStartValue,
gpuStride, gpuStride,
lastIsSevenFlag,
candidateView, candidateView,
filteredCountSubView,
nextStartView, nextStartView,
candidateGenerationKernel); filteredCandidateKernel,
out nextDivisorGpu);
if (effectiveCount > 0) if (effectiveCount > 0)
{ {
candidateView.CopyToCPU(ref MemoryMarshal.GetReference(candidateSpan), effectiveCount);
processedCount += (ulong)effectiveCount; processedCount += (ulong)effectiveCount;
lastProcessed = candidateSpan[effectiveCount - 1]; UInt128 nextDivisor128 = (UInt128)nextDivisorGpu;
currentDivisor128 = (UInt128)nextDivisorGpu; UInt128 stride128 = (UInt128)gpuStride;
UInt128 lastDivisor128 = nextDivisor128 - stride128;
lastProcessed = (ulong)lastDivisor128;
currentDivisor128 = nextDivisor128;
} }
ArrayView1D<byte, Stride1D.Dense> maskView = hitsBuffer.View.SubView(0, effectiveCount);
EvaluateCandidateMaskOnGpu(effectiveCount, candidateView, maskView, lastIsSevenFlag, candidateMaskKernel);
int filteredCount = CompactCandidatesOnGpu(
effectiveCount,
maskView,
candidateView,
candidateView,
filteredCountView,
candidateCompactionKernel);
if (filteredCount > 0) if (filteredCount > 0)
{ {
Span<ulong> filteredSlice = filteredStorage[..filteredCount]; Span<ulong> filteredSlice = filteredStorage[..filteredCount];
@@ -380,63 +356,22 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static GpuUInt128 GenerateCandidatesOnGpu( private static int GenerateFilteredCandidatesOnGpu(
int count, int count,
GpuUInt128 startValue, GpuUInt128 startValue,
GpuUInt128 stride, GpuUInt128 stride,
byte lastIsSevenFlag,
ArrayView1D<ulong, Stride1D.Dense> candidateView, ArrayView1D<ulong, Stride1D.Dense> candidateView,
ArrayView1D<int, Stride1D.Dense> filteredCountView,
ArrayView1D<GpuUInt128, Stride1D.Dense> nextStartView, ArrayView1D<GpuUInt128, Stride1D.Dense> nextStartView,
Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> kernel) Action<KernelConfig, Index1D, ArrayView<ulong>, ArrayView<int>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128, byte> kernel,
out GpuUInt128 nextStart)
{ {
// EvenPerfectBitScanner always launches this helper with a positive count, // EvenPerfectBitScanner always launches this helper with a positive count,
// so the guard stays commented out to keep the hot path branch-free. // so the guard stays commented out to keep the hot path branch-free.
// if (count <= 0) // if (count <= 0)
// { // {
// return GpuUInt128.Zero; // nextStart = GpuUInt128.Zero;
// }
kernel(count, candidateView, nextStartView, startValue, stride);
GpuUInt128 nextStart = default;
nextStartView.SubView(0, 1).CopyToCPU(ref nextStart, 1);
return nextStart;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void EvaluateCandidateMaskOnGpu(
int count,
ArrayView1D<ulong, Stride1D.Dense> candidateView,
ArrayView1D<byte, Stride1D.Dense> maskView,
byte lastIsSevenFlag,
Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte> maskKernel)
{
// EvenPerfectBitScanner always supplies a positive count here,
// so the guard stays commented out to keep the hot path branch-free.
// if (count <= 0)
// {
// return;
// }
maskKernel(count, candidateView, maskView, lastIsSevenFlag);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int CompactCandidatesOnGpu(
int count,
ArrayView1D<byte, Stride1D.Dense> maskView,
ArrayView1D<ulong, Stride1D.Dense> sourceView,
ArrayView1D<ulong, Stride1D.Dense> destinationView,
ArrayView1D<int, Stride1D.Dense> countView,
Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<int>> compactionKernel)
{
// EvenPerfectBitScanner always compacts with a positive count,
// so the guard stays commented out to keep the hot path branch-free.
// if (count <= 0)
// {
// return 0; // return 0;
// } // }
@@ -445,10 +380,13 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
Index1D groupDim = new Index1D(count); Index1D groupDim = new Index1D(count);
KernelConfig kernelConfig = new KernelConfig(gridDim, groupDim, sharedMemoryConfig); KernelConfig kernelConfig = new KernelConfig(gridDim, groupDim, sharedMemoryConfig);
compactionKernel(kernelConfig, new Index1D(count), maskView, sourceView, destinationView, countView); kernel(kernelConfig, new Index1D(count), candidateView, filteredCountView, nextStartView, startValue, stride, lastIsSevenFlag);
int filteredCount = 0; int filteredCount = 0;
countView.SubView(0, 1).CopyToCPU(ref filteredCount, 1); filteredCountView.SubView(0, 1).CopyToCPU(ref filteredCount, 1);
nextStart = default;
nextStartView.SubView(0, 1).CopyToCPU(ref nextStart, 1);
return filteredCount; return filteredCount;
} }
@@ -720,11 +658,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
Divisors = ulongPool.Rent(actualCapacity); Divisors = ulongPool.Rent(actualCapacity);
Exponents = ulongPool.Rent(actualCapacity); Exponents = ulongPool.Rent(actualCapacity);
DivisorData = ThreadStaticPools.MontgomeryDivisorDataPool.Rent(actualCapacity); DivisorData = ThreadStaticPools.MontgomeryDivisorDataPool.Rent(actualCapacity);
CycleCandidates = ulongPool.Rent(cycleCapacity);
CycleLengths = ulongPool.Rent(cycleCapacity); CycleLengths = ulongPool.Rent(cycleCapacity);
CandidateMaskKernel = owner.GetCandidateMaskKernel(accelerator); FilteredCandidateKernel = owner.GetFilteredCandidateKernel(accelerator);
CandidateCompactionKernel = owner.GetCandidateCompactionKernel(accelerator);
CandidateGenerationKernel = owner.GetCandidateGenerationKernel(accelerator);
HitDetectionKernel = owner.GetHitDetectionKernel(accelerator); HitDetectionKernel = owner.GetHitDetectionKernel(accelerator);
Capacity = actualCapacity; Capacity = actualCapacity;
CycleCapacity = cycleCapacity; CycleCapacity = cycleCapacity;
@@ -750,15 +685,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
internal readonly MontgomeryDivisorData[] DivisorData; internal readonly MontgomeryDivisorData[] DivisorData;
internal readonly ulong[] CycleCandidates;
internal readonly ulong[] CycleLengths; internal readonly ulong[] CycleLengths;
internal readonly Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte> CandidateMaskKernel; internal readonly Action<KernelConfig, Index1D, ArrayView<ulong>, ArrayView<int>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128, byte> FilteredCandidateKernel;
internal readonly Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<int>> CandidateCompactionKernel;
internal readonly Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> CandidateGenerationKernel;
internal readonly Action<Index1D, ArrayView<byte>, ArrayView<int>> HitDetectionKernel; internal readonly Action<Index1D, ArrayView<byte>, ArrayView<int>> HitDetectionKernel;
@@ -780,7 +709,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ulongPool.Return(Divisors, clearArray: false); ulongPool.Return(Divisors, clearArray: false);
ulongPool.Return(Exponents, clearArray: false); ulongPool.Return(Exponents, clearArray: false);
divisorPool.Return(DivisorData, clearArray: false); divisorPool.Return(DivisorData, clearArray: false);
ulongPool.Return(CycleCandidates, clearArray: false);
ulongPool.Return(CycleLengths, clearArray: false); ulongPool.Return(CycleLengths, clearArray: false);
} }
} }