From 40d3b4b3ac648dba1815ec75b9c699f4b94a5ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Lema=C5=84czyk?= Date: Fri, 24 Oct 2025 01:23:25 +0200 Subject: [PATCH] Fuse GPU by-divisor candidate filtering into one kernel --- .../Gpu/GpuKernelCompilationTests.cs | 23 +-- .../Gpu/Kernels/DivisorByDivisorKernels.cs | 101 ++++--------- ...MersenneNumberDivisorByDivisorGpuTester.cs | 136 +++++------------- 3 files changed, 64 insertions(+), 196 deletions(-) diff --git a/PerfectNumbers.Core.Tests/Gpu/GpuKernelCompilationTests.cs b/PerfectNumbers.Core.Tests/Gpu/GpuKernelCompilationTests.cs index 469ce8e..8583473 100644 --- a/PerfectNumbers.Core.Tests/Gpu/GpuKernelCompilationTests.cs +++ b/PerfectNumbers.Core.Tests/Gpu/GpuKernelCompilationTests.cs @@ -78,13 +78,9 @@ public class GpuKernelCompilationTests typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetStepperKernel", BindingFlags.NonPublic | BindingFlags.Instance) ?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetKernelByPrimeExponent not found."); - private static readonly MethodInfo DivisorByDivisorGetCandidateMaskKernelMethod = - typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetCandidateMaskKernel", BindingFlags.NonPublic | BindingFlags.Instance) - ?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetCandidateMaskKernel not found."); - - private static readonly MethodInfo DivisorByDivisorGetCandidateGenerationKernelMethod = - typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetCandidateGenerationKernel", BindingFlags.NonPublic | BindingFlags.Instance) - ?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetCandidateGenerationKernel not found."); + private static readonly MethodInfo DivisorByDivisorGetFilteredCandidateKernelMethod = + typeof(MersenneNumberDivisorByDivisorGpuTester).GetMethod("GetFilteredCandidateKernel", BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("MersenneNumberDivisorByDivisorGpuTester.GetFilteredCandidateKernel not found."); private static readonly MethodInfo DivisorTesterGetKernelMethod = typeof(MersenneNumberDivisorGpuTester).GetMethod("GetKernel", BindingFlags.NonPublic | BindingFlags.Instance) @@ -158,8 +154,7 @@ public class GpuKernelCompilationTests yield return Loader("NttGpuMath.InverseKernel", CompileNttInverseKernel); yield return Loader("MersenneNumberDivisorByDivisorGpuTester.CheckKernel", CompileDivisorByDivisorCheckKernel); yield return Loader("MersenneNumberDivisorByDivisorGpuTester.EvaluateDivisorWithStepperKernel", CompileDivisorByDivisorExponentKernel); - yield return Loader("MersenneNumberDivisorByDivisorGpuTester.EvaluateCandidateMaskKernel", CompileDivisorByDivisorCandidateMaskKernel); - yield return Loader("MersenneNumberDivisorByDivisorGpuTester.GenerateCandidatesKernel", CompileDivisorByDivisorCandidateGenerationKernel); + yield return Loader("MersenneNumberDivisorByDivisorGpuTester.GenerateFilteredCandidatesKernel", CompileDivisorByDivisorFilteredCandidateKernel); yield return Loader("MersenneNumberDivisorGpuTester.Kernel", CompileDivisorTesterKernel); yield return Loader("MersenneNumberLucasLehmerGpuTester.Kernel", CompileLucasLehmerKernel); yield return Loader("MersenneNumberLucasLehmerGpuTester.AddSmallKernel", CompileLucasLehmerAddSmallKernel); @@ -287,16 +282,10 @@ public class GpuKernelCompilationTests _ = DivisorByDivisorGetExponentKernelMethod.Invoke(tester, new object[] { accelerator }); } - private static void CompileDivisorByDivisorCandidateMaskKernel(Accelerator accelerator) + private static void CompileDivisorByDivisorFilteredCandidateKernel(Accelerator accelerator) { var tester = new MersenneNumberDivisorByDivisorGpuTester(); - _ = DivisorByDivisorGetCandidateMaskKernelMethod.Invoke(tester, new object[] { accelerator }); - } - - private static void CompileDivisorByDivisorCandidateGenerationKernel(Accelerator accelerator) - { - var tester = new MersenneNumberDivisorByDivisorGpuTester(); - _ = DivisorByDivisorGetCandidateGenerationKernelMethod.Invoke(tester, new object[] { accelerator }); + _ = DivisorByDivisorGetFilteredCandidateKernelMethod.Invoke(tester, new object[] { accelerator }); } private static void CompileDivisorTesterKernel(Accelerator accelerator) diff --git a/PerfectNumbers.Core/Gpu/Kernels/DivisorByDivisorKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/DivisorByDivisorKernels.cs index d2ec9bf..557a624 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/DivisorByDivisorKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/DivisorByDivisorKernels.cs @@ -74,63 +74,37 @@ internal static class DivisorByDivisorKernels } } - public static void GenerateCandidatesKernel( + public static void GenerateFilteredCandidatesKernel( Index1D index, ArrayView candidates, + ArrayView filteredCount, ArrayView nextStarts, GpuUInt128 startValue, - GpuUInt128 stride) + GpuUInt128 stride, + byte lastIsSevenFlag) { int globalIndex = index; - // EvenPerfectBitScanner always launches this kernel with an extent matching the candidate view length, - // so the bounds guard stays commented out to keep the hot path branch-free. - // int candidateLength = (int)candidates.Length; - // if (globalIndex >= candidateLength) - // { - // return; - // } + int localIndex = Group.IdxX; + int groupSize = Group.Dimension.X; - ulong offset = (ulong)globalIndex; + ulong candidateOffset = (ulong)globalIndex; GpuUInt128 candidateValue = stride; - candidateValue.Mul(offset); + candidateValue.Mul(candidateOffset); candidateValue.Add(startValue); - candidates[globalIndex] = candidateValue.Low; + ulong candidate = candidateValue.Low; + candidates[globalIndex] = candidate; if (globalIndex == 0) { - // EvenPerfectBitScanner provides a single-element nextStarts view per launch, so the guard stays commented out. - // int nextStartLength = (int)nextStarts.Length; - // if (nextStartLength <= 0) - // { - // return; - // } - - ulong candidateCount = (ulong)candidates.Length; + ulong candidateCount = (ulong)groupSize; GpuUInt128 nextValue = stride; nextValue.Mul(candidateCount); nextValue.Add(startValue); nextStarts[0] = nextValue; } - } - public static void EvaluateCandidateMaskKernel( - Index1D index, - ArrayView candidates, - ArrayView mask, - byte lastIsSevenFlag) - { - int globalIndex = index; - // EvenPerfectBitScanner launches this auto-grouped kernel with an extent matching the mask length, - // so the defensive guard stays commented out to keep the kernel branch-free. - // int length = (int)mask.Length; - // if (globalIndex >= length) - // { - // return; - // } - - ulong candidate = candidates[globalIndex]; - byte remainder10 = (byte)(candidate % 10UL); bool lastIsSeven = lastIsSevenFlag != 0; + byte remainder10 = (byte)(candidate % 10UL); bool acceptSeven = remainder10 == 3 || remainder10 == 7 || remainder10 == 9; bool acceptNonSeven = remainder10 == 1 || remainder10 == 3 || remainder10 == 7 || remainder10 == 9; bool accept10 = lastIsSeven ? acceptSeven : acceptNonSeven; @@ -140,60 +114,36 @@ internal static class DivisorByDivisorKernels bool accept5 = remainder5 != 0; byte remainder3 = (byte)(candidate % 3UL); bool accept3 = remainder3 != 0; - byte accepted = (byte)((accept10 && accept8 && accept3 && accept5) ? 1 : 0); - mask[globalIndex] = accepted; - } + int accepted = (accept10 && accept8 && accept3 && accept5) ? 1 : 0; - public static void CompactCandidateMaskKernel( - Index1D index, - ArrayView mask, - ArrayView sourceCandidates, - ArrayView compactedCandidates, - ArrayView compactedCount) - { - int globalIndex = index; - int length = (int)mask.Length; - // EvenPerfectBitScanner launches this stream kernel with an extent matching the mask length, - // so the defensive guard stays commented out to keep the kernel branch-free. - // if (globalIndex >= length) - // { - // return; - // } - - int localIndex = Group.IdxX; - int groupSize = Group.Dimension.X; var shared = SharedMemory.GetDynamic(); - + int length = groupSize; int prefixBase = 0; int mappingBase = length; int fallbackBase = length * 2; - // The dynamic shared-memory slice stores prefix sums, compacted index mappings, and fallback slots - // in separate regions so the scatter stage can avoid divergent branches. - byte maskValue = mask[globalIndex]; - int accepted = maskValue != 0 ? 1 : 0; shared[prefixBase + localIndex] = accepted; shared[fallbackBase + localIndex] = globalIndex; Group.Barrier(); - int offset = 1; - while (offset < groupSize) + int scanOffset = 1; + while (scanOffset < groupSize) { - int addendIndex = prefixBase + localIndex - offset; - int addend = localIndex >= offset ? shared[addendIndex] : 0; + int addendIndex = prefixBase + localIndex - scanOffset; + int addend = localIndex >= scanOffset ? shared[addendIndex] : 0; Group.Barrier(); int currentIndex = prefixBase + localIndex; int current = shared[currentIndex]; int sum = current + addend; - int shouldUpdate = localIndex >= offset ? 1 : 0; + int shouldUpdate = localIndex >= scanOffset ? 1 : 0; shared[currentIndex] = shouldUpdate != 0 ? sum : current; Group.Barrier(); - offset <<= 1; + scanOffset <<= 1; } int inclusivePrefix = shared[prefixBase + localIndex]; @@ -209,16 +159,17 @@ internal static class DivisorByDivisorKernels int lookupBase = writerMask != 0 ? mappingBase : fallbackBase; int sourceIndex = shared[lookupBase + localIndex]; - ulong candidateValue = sourceCandidates[sourceIndex]; - int destinationIndex = localIndex; - ulong existingValue = compactedCandidates[destinationIndex]; - compactedCandidates[destinationIndex] = writerMask != 0 ? candidateValue : existingValue; + ulong candidateValueToStore = candidates[sourceIndex]; + if (writerMask != 0) + { + candidates[localIndex] = candidateValueToStore; + } int lastLaneMask = globalIndex == length - 1 ? 1 : 0; int countValue = shared[prefixBase + localIndex]; if (lastLaneMask != 0) { - compactedCount[0] = countValue; + filteredCount[0] = countValue; } } diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs index 4305f0e..90df7b0 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs @@ -25,10 +25,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv private readonly ConcurrentDictionary, ArrayView, ArrayView>> _kernelCache = new(); private readonly ConcurrentDictionary, ArrayView>> _stepperKernelCache = new(AcceleratorReferenceComparer.Instance); - private readonly ConcurrentDictionary, ArrayView, byte>> _candidateMaskKernelCache = new(AcceleratorReferenceComparer.Instance); - private readonly ConcurrentDictionary, ArrayView, ArrayView, ArrayView>> _candidateCompactionKernelCache = new(AcceleratorReferenceComparer.Instance); + private readonly ConcurrentDictionary, ArrayView, ArrayView, GpuUInt128, GpuUInt128, byte>> _filteredCandidateKernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary, ArrayView>> _hitDetectionKernelCache = new(AcceleratorReferenceComparer.Instance); - private readonly ConcurrentDictionary, ArrayView, GpuUInt128, GpuUInt128>> _candidateGenerationKernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary> _resourcePools = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentBag _acceleratorPool = new(); private readonly ConcurrentBag _sessionPool = new(); @@ -39,26 +37,16 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv private Action, ArrayView> GetStepperKernel(Accelerator accelerator) => _stepperKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView>(DivisorByDivisorKernels.EvaluateDivisorWithStepperKernel)); - private Action, ArrayView, byte> GetCandidateMaskKernel(Accelerator accelerator) => - _candidateMaskKernelCache.GetOrAdd( + private Action, ArrayView, ArrayView, GpuUInt128, GpuUInt128, byte> GetFilteredCandidateKernel(Accelerator accelerator) => + _filteredCandidateKernelCache.GetOrAdd( accelerator, - acc => acc.LoadAutoGroupedStreamKernel, ArrayView, byte>(DivisorByDivisorKernels.EvaluateCandidateMaskKernel)); - - private Action, ArrayView, ArrayView, ArrayView> GetCandidateCompactionKernel(Accelerator accelerator) => - _candidateCompactionKernelCache.GetOrAdd( - accelerator, - acc => acc.LoadStreamKernel, ArrayView, ArrayView, ArrayView>(DivisorByDivisorKernels.CompactCandidateMaskKernel)); + acc => acc.LoadStreamKernel, ArrayView, ArrayView, GpuUInt128, GpuUInt128, byte>(DivisorByDivisorKernels.GenerateFilteredCandidatesKernel)); private Action, ArrayView> GetHitDetectionKernel(Accelerator accelerator) => _hitDetectionKernelCache.GetOrAdd( accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView>(DivisorByDivisorKernels.PublishFirstHitKernel)); - private Action, ArrayView, GpuUInt128, GpuUInt128> GetCandidateGenerationKernel(Accelerator accelerator) => - _candidateGenerationKernelCache.GetOrAdd( - accelerator, - acc => acc.LoadAutoGroupedStreamKernel, ArrayView, GpuUInt128, GpuUInt128>(DivisorByDivisorKernels.GenerateCandidatesKernel)); - public int GpuBatchSize { get => _gpuBatchSize; @@ -137,14 +125,11 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv resources.CandidateBuffer, resources.NextStartBuffer, resources.FilterCountBuffer, - resources.CandidateMaskKernel, - resources.CandidateCompactionKernel, - resources.CandidateGenerationKernel, + resources.FilteredCandidateKernel, resources.HitDetectionKernel, resources.Divisors, resources.Exponents, resources.DivisorData, - resources.CycleCandidates, resources.CycleLengths, resources.CycleCapacity, out lastProcessed, @@ -206,14 +191,11 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv MemoryBuffer1D candidateBuffer, MemoryBuffer1D nextStartBuffer, MemoryBuffer1D filterCountBuffer, - Action, ArrayView, byte> candidateMaskKernel, - Action, ArrayView, ArrayView, ArrayView> candidateCompactionKernel, - Action, ArrayView, GpuUInt128, GpuUInt128> candidateGenerationKernel, + Action, ArrayView, ArrayView, GpuUInt128, GpuUInt128, byte> filteredCandidateKernel, Action, ArrayView> hitDetectionKernel, in ulong[] divisors, ulong[] exponents, MontgomeryDivisorData[] divisorData, - ulong[] cycleCandidates, ulong[] cycleLengths, int cycleCapacity, out ulong lastProcessed, @@ -257,7 +239,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv ArrayPool ulongPool = ThreadStaticPools.UlongPool; ulong[] filteredDivisors = ulongPool.Rent(chunkCapacity); - Span candidateStorage = cycleCandidates.AsSpan(); Span filteredStorage = filteredDivisors.AsSpan(); Span cycleLengthStorage = cycleLengths.AsSpan(); Span divisorDataStorage = divisorData.AsSpan(); @@ -278,38 +259,33 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv break; } - Span candidateSpan = candidateStorage[..effectiveCount]; - ArrayView1D candidateView = candidateBuffer.View.SubView(0, effectiveCount); ArrayView1D nextStartView = nextStartBuffer.View.SubView(0, 1); + ArrayView1D filteredCountSubView = filteredCountView.SubView(0, 1); GpuUInt128 gpuStartValue = new GpuUInt128(currentDivisor128); GpuUInt128 gpuStride = new GpuUInt128(twoP128); - GpuUInt128 nextDivisorGpu = GenerateCandidatesOnGpu( + GpuUInt128 nextDivisorGpu = default; + int filteredCount = GenerateFilteredCandidatesOnGpu( effectiveCount, gpuStartValue, gpuStride, + lastIsSevenFlag, candidateView, + filteredCountSubView, nextStartView, - candidateGenerationKernel); + filteredCandidateKernel, + out nextDivisorGpu); + if (effectiveCount > 0) { - candidateView.CopyToCPU(ref MemoryMarshal.GetReference(candidateSpan), effectiveCount); processedCount += (ulong)effectiveCount; - lastProcessed = candidateSpan[effectiveCount - 1]; - currentDivisor128 = (UInt128)nextDivisorGpu; + UInt128 nextDivisor128 = (UInt128)nextDivisorGpu; + UInt128 stride128 = (UInt128)gpuStride; + UInt128 lastDivisor128 = nextDivisor128 - stride128; + lastProcessed = (ulong)lastDivisor128; + currentDivisor128 = nextDivisor128; } - ArrayView1D maskView = hitsBuffer.View.SubView(0, effectiveCount); - EvaluateCandidateMaskOnGpu(effectiveCount, candidateView, maskView, lastIsSevenFlag, candidateMaskKernel); - - int filteredCount = CompactCandidatesOnGpu( - effectiveCount, - maskView, - candidateView, - candidateView, - filteredCountView, - candidateCompactionKernel); - if (filteredCount > 0) { Span filteredSlice = filteredStorage[..filteredCount]; @@ -380,63 +356,22 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static GpuUInt128 GenerateCandidatesOnGpu( + private static int GenerateFilteredCandidatesOnGpu( int count, GpuUInt128 startValue, GpuUInt128 stride, + byte lastIsSevenFlag, ArrayView1D candidateView, + ArrayView1D filteredCountView, ArrayView1D nextStartView, - Action, ArrayView, GpuUInt128, GpuUInt128> kernel) + Action, ArrayView, ArrayView, GpuUInt128, GpuUInt128, byte> kernel, + out GpuUInt128 nextStart) { // EvenPerfectBitScanner always launches this helper with a positive count, // so the guard stays commented out to keep the hot path branch-free. // if (count <= 0) // { - // return GpuUInt128.Zero; - // } - - kernel(count, candidateView, nextStartView, startValue, stride); - - GpuUInt128 nextStart = default; - nextStartView.SubView(0, 1).CopyToCPU(ref nextStart, 1); - return nextStart; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void EvaluateCandidateMaskOnGpu( - int count, - ArrayView1D candidateView, - ArrayView1D maskView, - byte lastIsSevenFlag, - Action, ArrayView, byte> maskKernel) - { - // EvenPerfectBitScanner always supplies a positive count here, - // so the guard stays commented out to keep the hot path branch-free. - // if (count <= 0) - // { - // return; - // } - - maskKernel(count, candidateView, maskView, lastIsSevenFlag); - } - - - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - - - private static int CompactCandidatesOnGpu( - int count, - ArrayView1D maskView, - ArrayView1D sourceView, - ArrayView1D destinationView, - ArrayView1D countView, - Action, ArrayView, ArrayView, ArrayView> compactionKernel) - { - // EvenPerfectBitScanner always compacts with a positive count, - // so the guard stays commented out to keep the hot path branch-free. - // if (count <= 0) - // { + // nextStart = GpuUInt128.Zero; // return 0; // } @@ -445,10 +380,13 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv Index1D groupDim = new Index1D(count); KernelConfig kernelConfig = new KernelConfig(gridDim, groupDim, sharedMemoryConfig); - compactionKernel(kernelConfig, new Index1D(count), maskView, sourceView, destinationView, countView); + kernel(kernelConfig, new Index1D(count), candidateView, filteredCountView, nextStartView, startValue, stride, lastIsSevenFlag); int filteredCount = 0; - countView.SubView(0, 1).CopyToCPU(ref filteredCount, 1); + filteredCountView.SubView(0, 1).CopyToCPU(ref filteredCount, 1); + + nextStart = default; + nextStartView.SubView(0, 1).CopyToCPU(ref nextStart, 1); return filteredCount; } @@ -720,11 +658,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv Divisors = ulongPool.Rent(actualCapacity); Exponents = ulongPool.Rent(actualCapacity); DivisorData = ThreadStaticPools.MontgomeryDivisorDataPool.Rent(actualCapacity); - CycleCandidates = ulongPool.Rent(cycleCapacity); CycleLengths = ulongPool.Rent(cycleCapacity); - CandidateMaskKernel = owner.GetCandidateMaskKernel(accelerator); - CandidateCompactionKernel = owner.GetCandidateCompactionKernel(accelerator); - CandidateGenerationKernel = owner.GetCandidateGenerationKernel(accelerator); + FilteredCandidateKernel = owner.GetFilteredCandidateKernel(accelerator); HitDetectionKernel = owner.GetHitDetectionKernel(accelerator); Capacity = actualCapacity; CycleCapacity = cycleCapacity; @@ -750,15 +685,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv internal readonly MontgomeryDivisorData[] DivisorData; - internal readonly ulong[] CycleCandidates; - internal readonly ulong[] CycleLengths; - internal readonly Action, ArrayView, byte> CandidateMaskKernel; - - internal readonly Action, ArrayView, ArrayView, ArrayView> CandidateCompactionKernel; - - internal readonly Action, ArrayView, GpuUInt128, GpuUInt128> CandidateGenerationKernel; + internal readonly Action, ArrayView, ArrayView, GpuUInt128, GpuUInt128, byte> FilteredCandidateKernel; internal readonly Action, ArrayView> HitDetectionKernel; @@ -780,7 +709,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv ulongPool.Return(Divisors, clearArray: false); ulongPool.Return(Exponents, clearArray: false); divisorPool.Return(DivisorData, clearArray: false); - ulongPool.Return(CycleCandidates, clearArray: false); ulongPool.Return(CycleLengths, clearArray: false); } }