mlemanczyk/even-perfect-numbers-scanner
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Streams added to all kernels and GPU operations to separate threads from each other.

Browse Source
This commit is contained in:
mlemanczyk committed 2025-11-06 18:53:55 +01:00
1 parent 64d615c402
commit de2a78b27b
30 files changed
+1097 -1002
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PerfectNumbers.Core/Gpu/GpuContextPool.cs
-1
View File
@@ -102,7 +102,6 @@ public static class GpuContextPool
internal static void Return(PooledContext ctx)
{
// ctx.Accelerator.Synchronize();
GpuPool.Enqueue(ctx);
}
}
PerfectNumbers.Core/Gpu/GpuKernelLease.cs
+4 -3
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -19,7 +20,7 @@ public sealed class GpuKernelLease
Kernels = kernels;
}
internal static GpuKernelLease Rent(GpuContextLease gpu, KernelContainer kernels)
internal static GpuKernelLease Rent(GpuContextLease gpu, AcceleratorStream stream, KernelContainer kernels)
{
if (!Pool.TryDequeue(out var lease))
{
@@ -28,7 +29,7 @@ public sealed class GpuKernelLease
lease._gpu = gpu;
lease.Kernels = kernels;
lease.Stream = gpu.Accelerator.CreateStream();
lease.Stream = stream;
return lease;
}
@@ -66,7 +67,7 @@ public sealed class GpuKernelLease
{
get
{
var accel = Accelerator;
var accel = Accelerator; // avoid capturing 'this' in lambda
return GpuKernelPool.InitOnce(ref Kernels!.Incremental, () =>
{
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernels.IncrementalKernelScan);
PerfectNumbers.Core/Gpu/GpuKernelPool.cs
+20 -19
View File
@@ -111,26 +111,26 @@ public class GpuKernelPool
return Interlocked.CompareExchange(ref target, newValue, null) ?? newValue;
}
internal static KernelContainer GetKernels(Accelerator accelerator)
internal static KernelContainer GetKernels(Accelerator accelerator, AcceleratorStream stream)
{
return _kernels.GetOrAdd(accelerator, static (accelerator) =>
return _kernels.GetOrAdd(accelerator, (accelerator) =>
{
var kernels = new KernelContainer();
PreloadStaticTables(kernels, accelerator);
PreloadStaticTables(kernels, accelerator, stream);
return kernels;
});
}
internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator)
internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
EnsureSmallCyclesOnDevice(kernels, accelerator);
EnsureSmallPrimesOnDevice(kernels, accelerator);
EnsureSmallPrimeFactorTables(kernels, accelerator);
EnsureSmallCyclesOnDevice(kernels, accelerator, stream);
EnsureSmallPrimesOnDevice(kernels, accelerator, stream);
EnsureSmallPrimeFactorTables(kernels, accelerator, stream);
}
// Ensures the small cycles table is uploaded to the device for the given accelerator.
// Returns the ArrayView to pass into kernels (when kernels are extended to accept it).
public static ArrayView1D<ulong, Stride1D.Dense> EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator)
public static ArrayView1D<ulong, Stride1D.Dense> EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
if (kernels.SmallCycles is { } buffer)
{
@@ -139,12 +139,12 @@ public class GpuKernelPool
var host = MersenneDivisorCycles.Shared.ExportSmallCyclesSnapshot(); // TODO: Preload this device buffer during startup and keep it immutable so we can delete the lock above in favor of the preloaded snapshot.
var device = accelerator.Allocate1D<ulong>(host.Length);
device.View.CopyFromCPU(host);
device.View.CopyFromCPU(stream, host);
kernels.SmallCycles = device;
return device.View;
}
public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator)
public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
if (kernels.SmallPrimesLastOne is { } lastOne &&
kernels.SmallPrimesLastSeven is { } lastSeven &&
@@ -168,13 +168,13 @@ public class GpuKernelPool
var hostLastSevenPow2 = PrimesGenerator.SmallPrimesPow2LastSeven;
var deviceLastOne = accelerator.Allocate1D<uint>(hostLastOne.Length);
deviceLastOne.View.CopyFromCPU(hostLastOne);
deviceLastOne.View.CopyFromCPU(stream, hostLastOne);
var deviceLastSeven = accelerator.Allocate1D<uint>(hostLastSeven.Length);
deviceLastSeven.View.CopyFromCPU(hostLastSeven);
deviceLastSeven.View.CopyFromCPU(stream, hostLastSeven);
var deviceLastOnePow2 = accelerator.Allocate1D<ulong>(hostLastOnePow2.Length);
deviceLastOnePow2.View.CopyFromCPU(hostLastOnePow2);
deviceLastOnePow2.View.CopyFromCPU(stream, hostLastOnePow2);
var deviceLastSevenPow2 = accelerator.Allocate1D<ulong>(hostLastSevenPow2.Length);
deviceLastSevenPow2.View.CopyFromCPU(hostLastSevenPow2);
deviceLastSevenPow2.View.CopyFromCPU(stream, hostLastSevenPow2);
kernels.SmallPrimesLastOne = deviceLastOne;
kernels.SmallPrimesLastSeven = deviceLastSeven;
@@ -184,7 +184,7 @@ public class GpuKernelPool
return new ResiduePrimeViews(deviceLastOne.View, deviceLastSeven.View, deviceLastOnePow2.View, deviceLastSevenPow2.View);
}
public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator)
public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
if (kernels.SmallPrimeFactorsPrimes is { } primeBuffer && kernels.SmallPrimeFactorsSquares is { } squareBuffer)
{
@@ -200,9 +200,9 @@ public class GpuKernelPool
var hostSquares = PrimesGenerator.SmallPrimesPow2;
var devicePrimes = accelerator.Allocate1D<uint>(hostPrimes.Length);
devicePrimes.View.CopyFromCPU(hostPrimes);
devicePrimes.View.CopyFromCPU(stream, hostPrimes);
var deviceSquares = accelerator.Allocate1D<ulong>(hostSquares.Length);
deviceSquares.View.CopyFromCPU(hostSquares);
deviceSquares.View.CopyFromCPU(stream, hostSquares);
kernels.SmallPrimeFactorsPrimes = devicePrimes;
kernels.SmallPrimeFactorsSquares = deviceSquares;
@@ -218,8 +218,9 @@ public class GpuKernelPool
{
GpuPrimeWorkLimiter.Acquire();
var gpu = Rent();
var kernels = GetKernels(gpu.Accelerator);
return GpuKernelLease.Rent(gpu, kernels);
var stream = gpu.Accelerator.CreateStream();
var kernels = GetKernels(gpu.Accelerator, stream);
return GpuKernelLease.Rent(gpu, stream, kernels);
}
/// <summary>
PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs
-10
View File
@@ -44,11 +44,6 @@ public sealed class ScratchBuffer
SmallPrimeFactorPrimeSlots = _accelerator.Allocate1D<ulong>(newSize);
SmallPrimeFactorExponentSlots = _accelerator.Allocate1D<int>(newSize);
}
// else
// {
// SmallPrimeFactorPrimeSlots.MemSetToZero();
// SmallPrimeFactorExponentSlots.MemSetToZero();
// }
}
public void ResizeSpecialMaxFactors(int newSize)
@@ -61,11 +56,6 @@ public sealed class ScratchBuffer
SpecialMaxFactors = _accelerator.Allocate1D<ulong>(newSize);
SpecialMaxCandidates = _accelerator.Allocate1D<ulong>(newSize);
}
// else
// {
// SpecialMaxFactors.MemSetToZero();
// SpecialMaxCandidates.MemSetToZero();
// }
}
}
PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs
+4 -4
View File
@@ -11,13 +11,13 @@ internal static class GpuStaticTableInitializer
{
private static readonly ConcurrentDictionary<Accelerator, byte> Initialized = new(AcceleratorReferenceComparer.Instance);
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator)
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
Initialized.GetOrAdd(accelerator, acc =>
{
PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc);
GpuKernelPool.PreloadStaticTables(kernels, acc);
PrimeOrderGpuHeuristics.PreloadStaticTables( acc);
PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc, stream);
GpuKernelPool.PreloadStaticTables(kernels, acc, stream);
PrimeOrderGpuHeuristics.PreloadStaticTables( acc, stream);
return 0;
});
}
PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs
+1 -1
View File
@@ -305,7 +305,7 @@ internal static class LucasLehmerKernels
states[idx] = s;
}
public static void Kernel(Index1D index, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
public static void Kernel(Index1D _, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
{
// Lucas–Lehmer iteration in the field GF(2^p-1).
// For p < 128, use a Mersenne-specific squaring + reduction to avoid
PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs
+2 -1
View File
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices;
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -32,7 +33,7 @@ internal static class NttBarrettKernels
}
// Barrett-based squaring: v = v^2 mod n for 128-bit n
public static void SquareBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
public static void SquareBarrett128Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
{
var v = data[index];
Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0);
PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs
+3 -2
View File
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices;
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -29,7 +30,7 @@ internal static class NttMontgomeryKernels
data[i2] = new GpuUInt128(0UL, diff);
}
public static void ToMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
public static void ToMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
{
ulong a = data[index].Low;
ulong v = MontMul64(a, r2, modulus, nPrime);
@@ -43,7 +44,7 @@ internal static class NttMontgomeryKernels
data[index] = new GpuUInt128(0UL, v);
}
public static void SquareMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
public static void SquareMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
{
ulong aR = data[index].Low;
ulong v = MontMul64(aR, aR, modulus, nPrime);
PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs
+41 -41
View File
@@ -24,14 +24,17 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private ulong _divisorLimit;
// private bool _isConfigured;
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>> _kernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>> _kernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentBag<BatchResources> _resourcePool = new();
private readonly ConcurrentBag<GpuContextLease> _acceleratorPool = new();
private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new();
private Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc =>
acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.CheckKernel));
private Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.CheckKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>>();
});
public int GpuBatchSize
@@ -92,8 +95,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ulong processedCount;
ulong lastProcessed;
var gpuLease = RentAccelerator();
var gpuLease = GpuKernelPool.GetKernel();
var accelerator = gpuLease.Accelerator;
var stream = gpuLease.Stream;
// Monitor.Enter(gpuLease.ExecutionLock);
@@ -104,6 +108,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
prime,
allowedMax,
kernel,
stream,
resources.DivisorDataBuffer,
resources.OffsetBuffer,
resources.CountBuffer,
@@ -122,9 +127,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
out processedCount);
ReturnBatchResources(resources);
// Monitor.Exit(gpuLease.ExecutionLock);
ReturnAccelerator(gpuLease);
// Monitor.Exit(gpuLease.ExecutionLock);
gpuLease.Dispose();
if (composite)
{
divisorsExhausted = true;
@@ -167,7 +172,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private static bool CheckDivisors(
ulong prime,
ulong allowedMax,
Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> kernel,
Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> kernel,
AcceleratorStream stream,
MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> divisorDataBuffer,
MemoryBuffer1D<int, Stride1D.Dense> offsetBuffer,
MemoryBuffer1D<int, Stride1D.Dense> countBuffer,
@@ -329,19 +335,20 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
continue;
}
divisorDataView.CopyFromCPU(ref divisorDataRef, admissibleCount);
offsetView.CopyFromCPU(ref offsetRef, admissibleCount);
countView.CopyFromCPU(ref countRef, admissibleCount);
cycleView.CopyFromCPU(ref cycleRef, admissibleCount);
exponentViewDevice.CopyFromCPU(ref exponentRef, admissibleCount);
divisorDataView.CopyFromCPU(stream, ref divisorDataRef, admissibleCount);
offsetView.CopyFromCPU(stream, ref offsetRef, admissibleCount);
countView.CopyFromCPU(stream, ref countRef, admissibleCount);
cycleView.CopyFromCPU(stream, ref cycleRef, admissibleCount);
exponentViewDevice.CopyFromCPU(stream, ref exponentRef, admissibleCount);
int sentinel = int.MaxValue;
hitIndexView.CopyFromCPU(ref sentinel, 1);
hitIndexView.CopyFromCPU(stream, ref sentinel, 1);
kernel(admissibleCount, divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView);
kernel(stream, new Index1D(admissibleCount), divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView);
int firstHit = sentinel;
hitIndexView.CopyToCPU(ref firstHit, 1);
hitIndexView.CopyToCPU(stream, ref firstHit, 1);
stream.Synchronize();
int hitIndex = firstHit >= admissibleCount ? -1 : firstHit;
bool hitFound = hitIndex >= 0;
composite = hitFound;
@@ -477,9 +484,10 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
public sealed class DivisorScanSession : IMersenneNumberDivisorByDivisorTester.IDivisorScanSession
{
private readonly MersenneNumberDivisorByDivisorGpuTester _owner;
private readonly GpuContextLease _lease;
private readonly GpuKernelLease _lease;
private readonly Accelerator _accelerator;
private Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> _kernel = null!;
private readonly AcceleratorStream _stream;
private Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> _kernel = null!;
private MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> _divisorBuffer = null!;
private MemoryBuffer1D<int, Stride1D.Dense> _offsetBuffer = null!;
private MemoryBuffer1D<int, Stride1D.Dense> _countBuffer = null!;
@@ -493,8 +501,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
internal DivisorScanSession(MersenneNumberDivisorByDivisorGpuTester owner)
{
_owner = owner;
_lease = GpuContextPool.Rent();
_lease = GpuKernelPool.GetKernel();
_accelerator = _lease.Accelerator;
_stream = _lease.Stream;
_capacity = Math.Max(1, owner._gpuBatchSize);
}
@@ -593,35 +602,36 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ArrayView1D<ulong, Stride1D.Dense> exponentView = _exponentsBuffer.View;
ArrayView1D<byte, Stride1D.Dense> hitView = _hitBuffer.View;
GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor);
divisorView.CopyFromCPU(ref partialData, 1);
AcceleratorStream stream = _stream;
GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor);
divisorView.CopyFromCPU(stream, ref partialData, 1);
int offsetValue = 0;
offsetView.CopyFromCPU(ref offsetValue, 1);
offsetView.CopyFromCPU(stream, ref offsetValue, 1);
int countValue = length;
countView.CopyFromCPU(ref countValue, 1);
countView.CopyFromCPU(stream, ref countValue, 1);
ulong cycleValue = cycle;
cycleView.CopyFromCPU(ref cycleValue, 1);
cycleView.CopyFromCPU(stream, ref cycleValue, 1);
Span<ulong> hostSpan = _hostBuffer.AsSpan(0, length);
primes.CopyTo(hostSpan);
ref ulong hostRef = ref MemoryMarshal.GetReference(hostSpan);
exponentView.CopyFromCPU(ref hostRef, length);
exponentView.CopyFromCPU(stream, ref hostRef, length);
int sentinel = int.MaxValue;
firstHitView.CopyFromCPU(ref sentinel, 1);
firstHitView.CopyFromCPU(stream, ref sentinel, 1);
var kernel = _kernel!;
kernel(1, divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView);
kernel(stream, new Index1D(1), divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView);
Span<byte> hitSlice = hits.Slice(0, length);
ref byte hitRef = ref MemoryMarshal.GetReference(hitSlice);
hitView.CopyToCPU(ref hitRef, length);
hitView.CopyToCPU(stream, ref hitRef, length);
stream.Synchronize();
// Monitor.Exit(_lease.ExecutionLock);
}
@@ -651,17 +661,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
}
}
private GpuContextLease RentAccelerator()
{
if (_acceleratorPool.TryTake(out var lease))
{
return lease;
}
return GpuContextPool.Rent();
}
private void ReturnAccelerator(GpuContextLease lease) => _acceleratorPool.Add(lease);
private BatchResources RentBatchResources(Accelerator accelerator, int capacity)
{
PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs
+12 -9
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -9,11 +10,16 @@ namespace PerfectNumbers.Core.Gpu;
/// </summary>
public sealed class MersenneNumberDivisorGpuTester
{
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new();
private readonly ConcurrentDictionary<Accelerator, MemoryBuffer1D<byte, Stride1D.Dense>> _resultBuffers = new();
private Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel));
private Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>>();
});
public static void BuildDivisorCandidates()
{
@@ -43,15 +49,12 @@ public sealed class MersenneNumberDivisorGpuTester
var kernel = GetKernel(accelerator);
var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D<byte>(1));
// There is no point in clearing this buffer. We always override item [0] and never use it beyond item [0]
// resultBuffer.MemSetToZero();
kernel(1, exponent, divisor, resultBuffer.View);
stream.Synchronize();
// resultBuffer.MemSetToZero(stream);
kernel(stream, 1, exponent, divisor, resultBuffer.View);
Span<byte> result = stackalloc byte[1];
resultBuffer.View.CopyToCPU(stream, ref result[0], 1);
stream.Synchronize();
bool divisible = result[0] != 0;
result[0] = 0;
resultBuffer.View.CopyFromCPU(stream, ref result[0], 1);
stream.Synchronize();
stream.Dispose();
gpu.Dispose();
return divisible;
PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs
+3 -3
View File
@@ -31,11 +31,11 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u
ulong step3 = ((exponent % 3UL) << 1) % 3UL;
ulong step5 = ((exponent % 5UL) << 1) % 5UL;
GpuUInt128 twoPGpu = (GpuUInt128)twoP;
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator);
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream);
ResiduePrimeViews primeViews = default;
if (_kernelType == GpuKernelType.Pow2Mod)
{
primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator);
primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator, stream);
}
var orderBuffer = accelerator.Allocate1D<ulong>(batchSize);
@@ -92,8 +92,8 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u
smallCyclesView);
}
orderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(orders), currentSize);
stream.Synchronize();
orderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(orders), currentSize);
if (_kernelType == GpuKernelType.Pow2Mod)
{
for (i = 0; i < currentSize; i++)
PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs
+72 -38
View File
@@ -1,6 +1,7 @@
using System.Buffers;
using System.Collections.Concurrent;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
using UInt128 = System.UInt128;
@@ -8,32 +9,60 @@ namespace PerfectNumbers.Core.Gpu;
public class MersenneNumberLucasLehmerGpuTester
{
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>> KernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> AddSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> SubSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> ReduceKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>> IsZeroKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>> BatchKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>> KernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> AddSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> SubSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> ReduceKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>> IsZeroKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>> BatchKernelCache = new();
private Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator)
{
return KernelCache.GetOrAdd(accelerator, accel => accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel));
}
private Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator) =>
KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) =>
AddSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel));
private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) =>
AddSmallKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) =>
SubSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel));
private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) =>
SubSmallKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) =>
ReduceKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel));
private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) =>
ReduceKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) =>
IsZeroKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel));
private Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) =>
IsZeroKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>>();
});
private Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) =>
BatchKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch));
private Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) =>
BatchKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>>();
});
// Configurable LL slice size to keep kernels short. Default 32.
public int SliceSize = 32;
@@ -70,13 +99,16 @@ public class MersenneNumberLucasLehmerGpuTester
{
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetKernel(accelerator);
var modulus = new GpuUInt128(((UInt128)1 << (int)exponent) - 1UL); // TODO: Cache these Mersenne moduli per exponent so LL GPU runs skip rebuilding them every launch.
var buffer = accelerator.Allocate1D<GpuUInt128>(1);
kernel(1, exponent, modulus, buffer.View);
accelerator.Synchronize();
kernel(stream, 1, exponent, modulus, buffer.View);
stream.Synchronize();
// TODO: Replace this with .CopyToCPU with pre-allocated buffer using stream. Synchronize on the stream after copy.
result = buffer.GetAsArray1D()[0].IsZero;
buffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -102,12 +134,13 @@ public class MersenneNumberLucasLehmerGpuTester
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetBatchKernel(accelerator);
var expBuffer = accelerator.Allocate1D<ulong>(count);
ulong[] expArray = ArrayPool<ulong>.Shared.Rent(count);
exponents.CopyTo(expArray);
expBuffer.View.CopyFromCPU(ref expArray[0], count);
expBuffer.View.CopyFromCPU(stream, ref expArray[0], count);
GpuUInt128[] modulusArray = ArrayPool<GpuUInt128>.Shared.Rent(count);
for (int i = 0; i < count; i++)
@@ -118,18 +151,19 @@ public class MersenneNumberLucasLehmerGpuTester
}
var modBuffer = accelerator.Allocate1D<GpuUInt128>(count);
modBuffer.View.CopyFromCPU(ref modulusArray[0], count);
modBuffer.View.CopyFromCPU(stream, ref modulusArray[0], count);
var stateBuffer = accelerator.Allocate1D<GpuUInt128>(count);
kernel(count, expBuffer.View, modBuffer.View, stateBuffer.View);
accelerator.Synchronize();
stateBuffer.View.CopyToCPU(ref residues[0], count);
kernel(stream, count, expBuffer.View, modBuffer.View, stateBuffer.View);
stateBuffer.View.CopyToCPU(stream, ref residues[0], count);
stream.Synchronize();
stateBuffer.Dispose();
modBuffer.Dispose();
expBuffer.Dispose();
ArrayPool<GpuUInt128>.Shared.Return(modulusArray);
ArrayPool<ulong>.Shared.Return(expArray);
stream.Dispose();
gpu.Dispose();
}
@@ -154,6 +188,7 @@ public class MersenneNumberLucasLehmerGpuTester
{
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var addKernel = GetAddSmallKernel(accelerator);
var subKernel = GetSubSmallKernel(accelerator);
var reduceKernel = GetReduceKernel(accelerator);
@@ -167,8 +202,8 @@ public class MersenneNumberLucasLehmerGpuTester
transformLength <<= 1;
}
var stateBuffer = accelerator.Allocate1D<GpuUInt128>(transformLength);
stateBuffer.MemSetToZero();
addKernel(1, 4UL, stateBuffer.View);
stateBuffer.MemSetToZero(stream);
addKernel(stream, 1, 4UL, stateBuffer.View);
// TODO(LL-SLICE): Slice Lucas–Lehmer iterations into short batches to
// avoid long-running kernels and TDR. Example: process 8–64 iterations
@@ -183,26 +218,25 @@ public class MersenneNumberLucasLehmerGpuTester
for (int s = 0; s < iter; s++)
{
// Square in NTT domain with staged kernels (short runtime per kernel).
NttGpuMath.SquareDevice(accelerator, stateBuffer.View, nttMod, primitiveRoot);
subKernel(1, 2UL, stateBuffer.View);
reduceKernel(1, exponent, stateBuffer.View);
NttGpuMath.SquareDevice(accelerator, stream, stateBuffer.View, nttMod, primitiveRoot);
subKernel(stream, 1, 2UL, stateBuffer.View);
reduceKernel(stream, 1, exponent, stateBuffer.View);
}
// Synchronize between slices to yield to scheduler and avoid TDR.
accelerator.Synchronize();
i += (ulong)iter;
}
reduceKernel(1, exponent, stateBuffer.View);
reduceKernel(stream, 1, exponent, stateBuffer.View);
var resultBuffer = accelerator.Allocate1D<byte>(1);
zeroKernel(1, stateBuffer.View, resultBuffer.View);
accelerator.Synchronize();
zeroKernel(stream, 1, stateBuffer.View, resultBuffer.View);
byte[] result = new byte[1];
resultBuffer.View.CopyToCPU(result);
resultBuffer.View.CopyToCPU(stream, result);
stream.Synchronize();
bool isPrime = result[0] != 0;
resultBuffer.Dispose();
stateBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
return isPrime;
}
PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs
+3 -3
View File
@@ -30,13 +30,13 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO
int currentSize;
int found = 0;
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator);
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream);
while (kStart <= maxK && Volatile.Read(ref isPrime))
{
remaining = maxK - kStart + 1UL;
currentSize = remaining > (UInt128)batchSize ? batchSize : (int)remaining;
// This clear is required because the result may not always be set. Remove it after the kernels are modified to always set the result.
foundBuffer.MemSetToZero();
foundBuffer.MemSetToZero(stream);
// Precompute residue automaton bases for this batch
UInt128 q0 = twoP * kStart + 1UL;
// TODO: Replace these modulo recomputations with the divisor-cycle stepping deltas captured in
@@ -49,8 +49,8 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO
kernel(stream, currentSize, exponent, (GpuUInt128)twoP, (GpuUInt128)kStart, last, divMul,
kernelArgs, foundBuffer.View, smallCyclesView);
foundBuffer.View.CopyToCPU(stream, ref found, 1);
stream.Synchronize();
foundBuffer.View.CopyToCPU(ref found, 1);
if (found != 0)
{
Volatile.Write(ref isPrime, false);
PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs
+7 -6
View File
@@ -36,15 +36,16 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder)
var stream = accelerator.CreateStream();
var kernel = GetPow2ModKernel(accelerator);
var resources = RentResources(accelerator, GpuConstants.ScanBatchSize);
var gpuKernels = GpuKernelPool.GetKernels(accelerator);
var gpuKernels = GpuKernelPool.GetKernels(accelerator, stream);
var orderBuffer = resources.OrderBuffer;
ulong[] orderArray = resources.OrderArray;
int batchSize = resources.Capacity;
int batchSize = resources.Capacity;
// Ensure device has small cycles and primes tables for in-kernel lookup
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator);
ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator);
// TODO: Pass-in the stream down the path
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator, stream);
ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator, stream);
UInt128 kStart = 1UL;
UInt128 limit = maxK + UInt128.One;
@@ -70,9 +71,9 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder)
kernel(stream, currentSize, exponent, twoPGpu, (GpuUInt128)kStart, last, 0UL,
kernelArgs, orderBuffer.View, smallCyclesView, primeViews.LastOne, primeViews.LastSeven, primeViews.LastOnePow2, primeViews.LastSevenPow2);
stream.Synchronize();
orderBuffer.View.CopyToCPU(ref ordersRef, currentSize);
orderBuffer.View.CopyToCPU(stream, ref ordersRef, currentSize);
stream.Synchronize();
if (!Volatile.Read(ref isPrime))
{
break;
PerfectNumbers.Core/Gpu/NttGpuMath.cs
+185 -126
View File
@@ -27,25 +27,41 @@ public static class NttGpuMath
// Global backend setting for GPU transforms. Used by ForwardGpu/InverseGpu.
// TODO(NTT-OPT): Consider passing backend explicitly to avoid global state.
public static NttBackend GpuTransformBackend { get; set; } = NttBackend.Reference;
// Controls modular reduction strategy inside staged NTT paths.
public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto;
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> StageKernelCache = new(); // TODO: Same as above – materialize staged kernels during startup so we can drop ConcurrentDictionary usage entirely per the benchmark guidance.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Promote to the static kernel table initialized alongside the GpuModularArithmeticBenchmarks fast path.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>> StageMontKernelCache = new(); // TODO: Inline the Montgomery kernels into the startup table instead of using a concurrent cache now that we no longer mutate state at runtime.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Same plan – reuse the precomputed kernels measured fastest in MontgomeryMultiplyBenchmarks rather than looking them up dynamically.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Collapse into the startup kernel array once Barrett128 constants are preloaded.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> SquareBarrett128KernelCache = new(); // TODO: Fold these kernels into the same startup table to avoid concurrent access overhead highlighted in GpuModularArithmeticBenchmarks.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Prebind and reuse the Montgomery conversions from MontgomeryMultiplyBenchmarks instead of storing them in a concurrent cache.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong>> FromMont64KernelCache = new(); // TODO: As above – drop ConcurrentDictionary once startup prewarms every kernel.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong>> SquareMont64KernelCache = new(); // TODO: Inline into the static kernel table established during initialization.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Move to the startup kernel table so Montgomery-scaled launches skip concurrent lookups.
// Controls modular reduction strategy inside staged NTT paths.
public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto;
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream,Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> StageKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Warm these forward kernels during initialization and store them in a plain array so the LucasLehmerGpuBenchmarks launch costs disappear.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Same as above – replace with startup-prepared tables per LucasLehmerGpuBenchmarks guidance.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int>> BitReverseKernelCache = new(); // TODO: Precompute bit-reversal kernels during boot so we can remove the concurrent lookup overhead flagged in the benchmarks.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>> StageMontKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong>> FromMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> SquareMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int>> BitReverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) =>
StageKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>>();
});
private sealed class SquareCacheEntry
{
@@ -72,7 +88,7 @@ public static class NttGpuMath
public ulong BarrettMuHigh { get; }
public ulong BarrettMuLow { get; }
public SquareCacheEntry(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
public SquareCacheEntry(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{
Buffer = accelerator.Allocate1D<GpuUInt128>(length);
Root = new GpuUInt128(primitiveRoot);
@@ -136,9 +152,9 @@ public static class NttGpuMath
}
Twiddles = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
Twiddles.View.CopyFromCPU(ref forward[0], twiddleCount);
Twiddles.View.CopyFromCPU(stream, ref forward[0], twiddleCount);
TwiddlesInv = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesInv.View.CopyFromCPU(ref inverse[0], twiddleCount);
TwiddlesInv.View.CopyFromCPU(stream, ref inverse[0], twiddleCount);
// Prepare Montgomery constants for 64-bit moduli and twiddles in Montgomery domain.
if (modulus.High == 0UL && (ReductionMode == ModReductionMode.Auto || ReductionMode == ModReductionMode.Mont64))
@@ -166,9 +182,9 @@ public static class NttGpuMath
}
TwiddlesMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesMont.View.CopyFromCPU(ref forwardMont[0], twiddleCount);
TwiddlesMont.View.CopyFromCPU(stream, ref forwardMont[0], twiddleCount);
TwiddlesInvMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesInvMont.View.CopyFromCPU(ref inverseMont[0], twiddleCount);
TwiddlesInvMont.View.CopyFromCPU(stream, ref inverseMont[0], twiddleCount);
pool.Return(forwardMont, clearArray: true);
pool.Return(inverseMont, clearArray: true);
@@ -258,11 +274,11 @@ public static class NttGpuMath
// - Store one GPU buffer with all stage twiddles, or per-stage slices.
// - Expose accessors to retrieve per-stage views used by butterfly kernels.
private static SquareCacheEntry GetSquareCache(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
private static SquareCacheEntry GetSquareCache(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{
var perAccel = SquareCache.GetOrAdd(accelerator, _ => new ConcurrentDictionary<SquareCacheKey, SquareCacheEntry>());
var key = new SquareCacheKey(length, modulus, primitiveRoot);
return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, length, modulus, primitiveRoot));
return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, stream, length, modulus, primitiveRoot));
}
private static readonly ulong[] SmallPrimes = GenerateSmallPrimes(1000);
@@ -349,7 +365,8 @@ public static class NttGpuMath
SquareMont64KernelCache.Clear();
ScaleMont64KernelCache.Clear();
ForwardKernelCache.Clear();
InverseKernelCache.Clear();
InverseKernelCache.Clear();
BitReverseKernelCache.Clear();
// TODO(NTT-OPT): Clear TwiddleCache once introduced.
return;
}
@@ -373,56 +390,97 @@ public static class NttGpuMath
ScaleMont64KernelCache.TryRemove(accelerator, out _);
ForwardKernelCache.TryRemove(accelerator, out _);
InverseKernelCache.TryRemove(accelerator, out _);
BitReverseKernelCache.TryRemove(accelerator, out _);
// TODO(NTT-OPT): Remove per-accelerator twiddle buffers once added.
}
private static Action<Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) =>
BitReverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) =>
BitReverseKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) =>
MulKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) =>
ForwardKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) =>
StageKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) =>
FromMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) =>
ScaleKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
InverseKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) =>
StageMontKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) =>
MulKernelCache.GetOrAdd(accelerator, valueFactory: acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) =>
StageBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) =>
ScaleKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) =>
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) =>
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetSquareBarrett128Kernel(Accelerator accelerator) =>
SquareBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.SquareBarrett128Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) =>
ScaleMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) =>
ToMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ToMont64Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) =>
StageBarrett128KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) =>
FromMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) =>
StageMontKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetSquareMont64Kernel(Accelerator accelerator) =>
SquareMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.SquareMont64Kernel));
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) =>
ScaleMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel));
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) =>
ForwardKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel));
// TODO(NTT-OPT): Introduce stage-wise Cooley–Tukey kernels and a
// per-(accelerator,length,modulus,root) twiddle cache. The plan:
// - Precompute twiddle factors for all stages once and cache on GPU.
// - Replace the current O(n^2) ForwardKernel with O(n log n) butterflies.
// - Provide accessors like GetStageKernel(...) and GetTwiddleBuffer(...).
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
InverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) =>
ToMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>>();
});
// TODO(NTT-OPT): Mirror the forward stage-wise design for the inverse
// transform. Precompute inverse twiddles and a normalization factor
@@ -453,13 +511,15 @@ public static class NttGpuMath
var array = pool.Rent(n);
values.CopyTo(array);
var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref array[0], n);
kernel(n, buffer.View, bits);
accelerator.Synchronize();
buffer.View.CopyToCPU(ref array[0], n);
var stream = accelerator.CreateStream();
buffer.View.CopyFromCPU(stream, ref array[0], n);
kernel(stream, n, buffer.View, bits);
buffer.View.CopyToCPU(stream, ref array[0], n);
stream.Synchronize();
array.AsSpan(0, n).CopyTo(values);
pool.Return(array, clearArray: true);
buffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -510,6 +570,7 @@ public static class NttGpuMath
int n = (int)values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetForwardKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared;
var inputArray = pool.Rent(n);
@@ -517,19 +578,20 @@ public static class NttGpuMath
values.CopyTo(inputArray);
var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
inputBuffer.View.CopyFromCPU(ref inputArray[0], n);
inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n);
UInt128 modValue = modulus;
UInt128 expBase = (modValue - 1UL) / (ulong)n;
var root = new GpuUInt128(primitiveRoot);
root.ModPow((ulong)expBase, modulus);
kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, root);
accelerator.Synchronize();
outputBuffer.View.CopyToCPU(ref outputArray[0], n);
kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, root);
outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n);
stream.Synchronize();
outputArray.AsSpan(0, n).CopyTo(values);
pool.Return(inputArray, clearArray: true);
pool.Return(outputArray, clearArray: true);
outputBuffer.Dispose();
inputBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -607,6 +669,7 @@ public static class NttGpuMath
int n = values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetInverseKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared;
var inputArray = pool.Rent(n);
@@ -614,7 +677,7 @@ public static class NttGpuMath
values.CopyTo(inputArray);
var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
inputBuffer.View.CopyFromCPU(ref inputArray[0], n);
inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n);
UInt128 modValue = modulus;
UInt128 expBase = (modValue - 1UL) / (ulong)n;
var root = new GpuUInt128(primitiveRoot);
@@ -638,14 +701,15 @@ public static class NttGpuMath
{
nInv.ModInv(modulus);
}
kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv);
accelerator.Synchronize();
outputBuffer.View.CopyToCPU(ref outputArray[0], n);
kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv);
outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n);
stream.Synchronize();
outputArray.AsSpan(0, n).CopyTo(values);
pool.Return(inputArray, clearArray: true);
pool.Return(outputArray, clearArray: true);
outputBuffer.Dispose();
inputBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -655,11 +719,14 @@ public static class NttGpuMath
int n = values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref values[0], n);
var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot);
ForwardDevice(accelerator, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(ref values[0], n);
buffer.View.CopyFromCPU(stream, ref values[0], n);
var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot);
ForwardDevice(accelerator, stream, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(stream, ref values[0], n);
stream.Synchronize();
stream.Dispose();
buffer.Dispose();
gpu.Dispose();
@@ -670,38 +737,38 @@ public static class NttGpuMath
int n = values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref values[0], n);
var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot);
InverseDevice(accelerator, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(ref values[0], n);
buffer.View.CopyFromCPU(stream, ref values[0], n);
var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot);
InverseDevice(accelerator, stream, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(stream, ref values[0], n);
stream.Synchronize();
stream.Dispose();
buffer.Dispose();
gpu.Dispose();
}
private static void ForwardDevice(Accelerator accelerator, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
private static void ForwardDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
{
var bitKernel = GetBitReverseKernel(accelerator);
bitKernel(length, data, cache.Bits);
accelerator.Synchronize();
bitKernel(stream, length, data, cache.Bits);
if (cache.UseMontgomery64)
{
var toMont = GetToMont64Kernel(accelerator);
toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
accelerator.Synchronize();
toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
var stageKernel = GetStageMontKernel(accelerator);
int butterflies = length >> 1;
for (int s = 0; s < cache.Bits; s++)
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64);
accelerator.Synchronize();
stageKernel( stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64);
}
var fromMont = GetFromMont64Kernel(accelerator);
fromMont(length, data, cache.ModulusLow, cache.MontNPrime64);
fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64);
}
else if (cache.UseBarrett128)
{
@@ -711,8 +778,7 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
}
}
else
@@ -723,39 +789,34 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus);
}
}
accelerator.Synchronize();
stream.Synchronize();
}
private static void InverseDevice(Accelerator accelerator, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
private static void InverseDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
{
var bitKernel = GetBitReverseKernel(accelerator);
bitKernel(length, data, cache.Bits);
accelerator.Synchronize();
bitKernel(stream, length, data, cache.Bits);
if (cache.UseMontgomery64)
{
var toMont = GetToMont64Kernel(accelerator);
toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
accelerator.Synchronize();
toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
var stageKernel = GetStageMontKernel(accelerator);
int butterflies = length >> 1;
for (int s = 0; s < cache.Bits; s++)
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64);
}
var scaleKernel = GetScaleMont64Kernel(accelerator);
scaleKernel(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64);
accelerator.Synchronize();
scaleKernel(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64);
var fromMont = GetFromMont64Kernel(accelerator);
fromMont(length, data, cache.ModulusLow, cache.MontNPrime64);
fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64);
}
else if (cache.UseBarrett128)
{
@@ -765,12 +826,10 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
}
var scaleKernel = GetScaleBarrett128Kernel(accelerator);
scaleKernel(length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
scaleKernel(stream, length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
}
else
{
@@ -780,15 +839,13 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus);
}
var scaleKernel = GetScaleKernel(accelerator);
scaleKernel(length, data, cache.NInv, modulus);
accelerator.Synchronize();
scaleKernel(stream, length, data, cache.NInv, modulus);
}
accelerator.Synchronize();
stream.Synchronize();
}
public static void Convolve(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
@@ -871,19 +928,20 @@ public static class NttGpuMath
public static void SquareGpu(Span<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) =>
ConvolveGpu(values, values, modulus, primitiveRoot);
public static void SquareDevice(Accelerator accelerator, ArrayView<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
public static void SquareDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{
var cache = GetSquareCache(accelerator, (int)values.Length, modulus, primitiveRoot);
ForwardDevice(accelerator, values, (int)values.Length, modulus, cache);
var cache = GetSquareCache(accelerator, stream, (int)values.Length, modulus, primitiveRoot);
ForwardDevice(accelerator, stream, values, (int)values.Length, modulus, cache);
var mul = GetMulKernel(accelerator);
mul((int)values.Length, values, values, modulus);
InverseDevice(accelerator, values, (int)values.Length, modulus, cache);
mul(stream, (int)values.Length, values, values, modulus);
InverseDevice(accelerator, stream, values, (int)values.Length, modulus, cache);
}
public static void PointwiseMultiply(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus)
{
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetMulKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared;
var leftArray = pool.Rent(left.Length);
@@ -892,18 +950,19 @@ public static class NttGpuMath
right.CopyTo(rightArray);
var leftBuffer = accelerator.Allocate1D<GpuUInt128>(left.Length);
var rightBuffer = accelerator.Allocate1D<GpuUInt128>(right.Length);
leftBuffer.View.CopyFromCPU(ref leftArray[0], left.Length);
rightBuffer.View.CopyFromCPU(ref rightArray[0], right.Length);
kernel(left.Length, leftBuffer.View, rightBuffer.View, modulus);
accelerator.Synchronize();
leftBuffer.View.CopyToCPU(ref leftArray[0], left.Length);
rightBuffer.View.CopyToCPU(ref rightArray[0], right.Length);
leftBuffer.View.CopyFromCPU(stream, ref leftArray[0], left.Length);
rightBuffer.View.CopyFromCPU(stream, ref rightArray[0], right.Length);
kernel(stream, left.Length, leftBuffer.View, rightBuffer.View, modulus);
leftBuffer.View.CopyToCPU(stream, ref leftArray[0], left.Length);
rightBuffer.View.CopyToCPU(stream, ref rightArray[0], right.Length);
stream.Synchronize();
leftArray.AsSpan(0, left.Length).CopyTo(left);
rightArray.AsSpan(0, right.Length).CopyTo(right);
pool.Return(leftArray, clearArray: true);
pool.Return(rightArray, clearArray: true);
rightBuffer.Dispose();
leftBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs
+45 -45
View File
@@ -18,9 +18,9 @@ internal enum GpuPow2ModStatus
internal static partial class PrimeOrderGpuHeuristics
{
internal static void PreloadStaticTables(Accelerator accelerator)
internal static void PreloadStaticTables(Accelerator accelerator, AcceleratorStream stream)
{
_ = GetSmallPrimeDeviceCache(accelerator);
_ = GetSmallPrimeDeviceCache(accelerator, stream);
}
private static readonly ConcurrentDictionary<ulong, byte> OverflowedPrimes = new();
@@ -106,16 +106,16 @@ internal static partial class PrimeOrderGpuHeuristics
s_capability = PrimeOrderGpuCapability.Default;
}
private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator)
private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator, AcceleratorStream stream)
{
return SmallPrimeDeviceCaches.GetOrAdd(accelerator, static acc =>
return SmallPrimeDeviceCaches.GetOrAdd(accelerator, acc =>
{
uint[] primes = PrimesGenerator.SmallPrimes;
ulong[] squares = PrimesGenerator.SmallPrimesPow2;
var primeBuffer = acc.Allocate1D<uint>(primes.Length);
primeBuffer.View.CopyFromCPU(primes);
primeBuffer.View.CopyFromCPU(stream, primes);
var squareBuffer = acc.Allocate1D<ulong>(squares.Length);
squareBuffer.View.CopyFromCPU(squares);
squareBuffer.View.CopyFromCPU(stream, squares);
return new SmallPrimeDeviceCache
{
Primes = primeBuffer,
@@ -203,7 +203,7 @@ internal static partial class PrimeOrderGpuHeuristics
AcceleratorStream stream = lease.Stream;
var kernel = GetPartialFactorKernel(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream);
// TODO: We should create / reallocate these buffer only once or if the new required length is bigger than capacity.
var factorBuffer = accelerator.Allocate1D<ulong>(primeTargets.Length);
@@ -213,11 +213,11 @@ internal static partial class PrimeOrderGpuHeuristics
var fullyFactoredBuffer = accelerator.Allocate1D<byte>(1);
// TODO: There is no need to clear these buffers because the kernel will always assign values within the required bounds.
// factorBuffer.MemSetToZero();
// exponentBuffer.MemSetToZero();
// countBuffer.MemSetToZero();
// remainingBuffer.MemSetToZero();
// fullyFactoredBuffer.MemSetToZero();
// factorBuffer.MemSetToZero(stream);
// exponentBuffer.MemSetToZero(stream);
// countBuffer.MemSetToZero(stream);
// remainingBuffer.MemSetToZero(stream);
// fullyFactoredBuffer.MemSetToZero(stream);
kernel(
stream,
@@ -234,17 +234,18 @@ internal static partial class PrimeOrderGpuHeuristics
remainingBuffer.View,
fullyFactoredBuffer.View);
stream.Synchronize();
countBuffer.View.CopyToCPU(ref factorCount, 1);
factorCount = Math.Min(factorCount, primeTargets.Length);
factorBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length);
exponentBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length);
remainingBuffer.View.CopyToCPU(ref remaining, 1);
countBuffer.View.CopyToCPU(stream, ref factorCount, 1);
factorBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length);
exponentBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length);
remainingBuffer.View.CopyToCPU(stream, ref remaining, 1);
byte fullyFactoredFlag = 0;
fullyFactoredBuffer.View.CopyToCPU(ref fullyFactoredFlag, 1);
fullyFactored = fullyFactoredFlag != 0;
fullyFactoredBuffer.View.CopyToCPU(stream, ref fullyFactoredFlag, 1);
stream.Synchronize();
fullyFactored = fullyFactoredFlag != 0;
factorCount = Math.Min(factorCount, primeTargets.Length);
// TODO: These buffers shouldn't be disposed but rather reused accross call with the assigned accelerator.
factorBuffer.Dispose();
@@ -369,7 +370,7 @@ internal static partial class PrimeOrderGpuHeuristics
AcceleratorStream stream = lease.Stream;
var kernel = GetOrderKernel(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream);
// TODO: These buffers should be created reallocated once and assigned to an accelerator. Callers should use their own
// thread static cache to prevent other threads from using taking accelerators with pre-allocated buffers if they don't
@@ -386,16 +387,16 @@ internal static partial class PrimeOrderGpuHeuristics
var statusBuffer = accelerator.Allocate1D<byte>(1);
// TODO: Remove the cleaning after the order kernel is modified to always set the result.
phiFactorBuffer.MemSetToZero();
phiExponentBuffer.MemSetToZero();
workFactorBuffer.MemSetToZero();
workExponentBuffer.MemSetToZero();
candidateBuffer.MemSetToZero();
stackIndexBuffer.MemSetToZero();
stackExponentBuffer.MemSetToZero();
stackProductBuffer.MemSetToZero();
resultBuffer.MemSetToZero();
statusBuffer.MemSetToZero();
phiFactorBuffer.MemSetToZero(stream);
phiExponentBuffer.MemSetToZero(stream);
workFactorBuffer.MemSetToZero(stream);
workExponentBuffer.MemSetToZero(stream);
candidateBuffer.MemSetToZero(stream);
stackIndexBuffer.MemSetToZero(stream);
stackExponentBuffer.MemSetToZero(stream);
stackProductBuffer.MemSetToZero(stream);
resultBuffer.MemSetToZero(stream);
statusBuffer.MemSetToZero(stream);
uint limit = config.SmallFactorLimit == 0 ? uint.MaxValue : config.SmallFactorLimit;
byte hasPrevious = previousOrder.HasValue ? (byte)1 : (byte)0;
@@ -425,10 +426,11 @@ internal static partial class PrimeOrderGpuHeuristics
cache.Count,
buffers);
stream.Synchronize();
byte status = 0;
statusBuffer.View.CopyToCPU(ref status, 1);
statusBuffer.View.CopyToCPU(stream, ref status, 1);
resultBuffer.View.CopyToCPU(stream, ref order, 1);
stream.Synchronize();
PrimeOrderKernelStatus kernelStatus = (PrimeOrderKernelStatus)status;
if (kernelStatus == PrimeOrderKernelStatus.Fallback)
@@ -445,8 +447,6 @@ internal static partial class PrimeOrderGpuHeuristics
throw new InvalidOperationException("GPU Pollard Rho stack overflow; increase HeuristicStackCapacity.");
}
resultBuffer.View.CopyToCPU(ref order, 1);
if (kernelStatus == PrimeOrderKernelStatus.FactoringFailure)
{
order = 0UL;
@@ -536,15 +536,19 @@ internal static partial class PrimeOrderGpuHeuristics
var exponentBuffer = accelerator.Allocate1D<ulong>(exponents.Length);
var remainderBuffer = accelerator.Allocate1D<ulong>(exponents.Length);
exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponents), exponents.Length);
exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponents), exponents.Length);
// Pow2Mod kernel always assigns the required output elements so we don't need to worry about clearing these.
// remainderBuffer.MemSetToZero();
// remainderBuffer.MemSetToZero(stream);
try
{
kernel(stream, exponents.Length, exponentBuffer.View, divisorData, remainderBuffer.View);
remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(results), exponents.Length);
stream.Synchronize();
exponentBuffer.Dispose();
remainderBuffer.Dispose();
lease.Dispose();
}
catch (Exception)
{
@@ -555,10 +559,6 @@ internal static partial class PrimeOrderGpuHeuristics
throw;
}
remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(results), exponents.Length);
exponentBuffer.Dispose();
remainderBuffer.Dispose();
lease.Dispose();
return true;
}
@@ -603,20 +603,20 @@ internal static partial class PrimeOrderGpuHeuristics
exponentSpan[i] = (GpuUInt128)exponents[i];
}
exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponentSpan), length);
exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponentSpan), length);
// Pow2ModWide kernel always assigns the required output elements so we don't need to worry about clearing these.
// remainderBuffer.MemSetToZero();
// remainderBuffer.MemSetToZero(stream);
GpuUInt128 modulus = (GpuUInt128)prime;
kernel(stream, length, exponentBuffer.View, modulus, remainderBuffer.View);
stream.Synchronize();
Span<GpuUInt128> resultSpan = length <= WideStackThreshold
? stackalloc GpuUInt128[length]
: new Span<GpuUInt128>(rentedResults = ArrayPool<GpuUInt128>.Shared.Rent(length), 0, length);
remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(resultSpan), length);
remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length);
stream.Synchronize();
for (int i = 0; i < length; i++)
{
PerfectNumbers.Core/Gpu/SharedGpuContext.cs
+2 -1
View File
@@ -7,11 +7,12 @@ namespace PerfectNumbers.Core.Gpu;
public static class SharedGpuContext
{
// TODO: Remove Lazy<T> in favor for better performance with nullable fields.
private static readonly Lazy<Context> s_context = new(() => ILGPU.Context.CreateDefault(), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy<Device> s_device = new(() => s_context.Value.GetPreferredDevice(false), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy<Accelerator> s_accelerator = new(() => s_device.Value.CreateAccelerator(s_context.Value), LazyThreadSafetyMode.ExecutionAndPublication);
internal static Context Context => s_context.Value;
internal static Context Context => s_context.Value;
internal static Device Device => s_device.Value;