mlemanczyk/even-perfect-numbers-scanner
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Streams added to all kernels and GPU operations to separate threads from each other.

Browse Source
This commit is contained in:
mlemanczyk committed 2025-11-06 18:53:55 +01:00
1 parent 64d615c402
commit de2a78b27b
30 files changed
+530 -435
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PerfectNumbers.Core.Tests/Gpu/NttGpuMathTests.cs
+10 -16
View File
@@ -154,6 +154,7 @@ public class NttGpuMathTests
try try
{ {
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var method = typeof(NttGpuMath).GetMethod("GetSquareCache", BindingFlags.NonPublic | BindingFlags.Static); var method = typeof(NttGpuMath).GetMethod("GetSquareCache", BindingFlags.NonPublic | BindingFlags.Static);
method.Should().NotBeNull(); method.Should().NotBeNull();
@@ -168,8 +169,10 @@ public class NttGpuMathTests
int count = 7; // length 8 -> 7 twiddles int count = 7; // length 8 -> 7 twiddles
var forward = new GpuUInt128[count]; var forward = new GpuUInt128[count];
var inverse = new GpuUInt128[count]; var inverse = new GpuUInt128[count];
twiddlesMont.CopyToCPU(forward); twiddlesMont.CopyToCPU(stream, forward);
twiddlesInvMont.CopyToCPU(inverse); twiddlesInvMont.CopyToCPU(stream, inverse);
stream.Synchronize();
stream.Dispose();
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
{ {
@@ -346,23 +349,14 @@ public class NttGpuMathTests
var modulus = new GpuUInt128(0UL, 17UL); var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL); var primitiveRoot = new GpuUInt128(0UL, 3UL);
GpuContextLease gpu = GpuContextPool.Rent(); GpuContextLease gpu = GpuContextPool.Rent();
try var stream = gpu.Accelerator.CreateStream();
{
var buffer = gpu.Accelerator.Allocate1D<GpuUInt128>(values.Length); var buffer = gpu.Accelerator.Allocate1D<GpuUInt128>(values.Length);
try buffer.View.CopyFromCPU(stream, ref values[0], values.Length);
{ NttGpuMath.SquareDevice(gpu.Accelerator, stream, buffer.View, modulus, primitiveRoot);
buffer.View.CopyFromCPU(ref values[0], values.Length); stream.Synchronize();
NttGpuMath.SquareDevice(gpu.Accelerator, buffer.View, modulus, primitiveRoot);
}
finally
{
buffer.Dispose(); buffer.Dispose();
} stream.Dispose();
}
finally
{
gpu.Dispose(); gpu.Dispose();
}
// Verify that the square cache now holds entries for the accelerator. // Verify that the square cache now holds entries for the accelerator.
var field = typeof(NttGpuMath).GetField("SquareCache", BindingFlags.NonPublic | BindingFlags.Static)!; var field = typeof(NttGpuMath).GetField("SquareCache", BindingFlags.NonPublic | BindingFlags.Static)!;
PerfectNumbers.Core.Tests/Gpu/NttMontgomeryTests.cs
+7 -9
View File
@@ -34,24 +34,22 @@ public class NttMontgomeryTests
// Device execution // Device execution
GpuContextLease context = GpuContextPool.Rent(); GpuContextLease context = GpuContextPool.Rent();
try
{
var accelerator = context.Accelerator; var accelerator = context.Accelerator;
var stream = accelerator.CreateStream();
using var buffer = accelerator.Allocate1D<GpuUInt128>(gpu.Length); using var buffer = accelerator.Allocate1D<GpuUInt128>(gpu.Length);
buffer.View.CopyFromCPU(ref gpu[0], gpu.Length); buffer.View.CopyFromCPU(stream, ref gpu[0], gpu.Length);
NttGpuMath.SquareDevice(accelerator, buffer.View, modulus, primitiveRoot); NttGpuMath.SquareDevice(accelerator, stream, buffer.View, modulus, primitiveRoot);
buffer.View.CopyToCPU(ref gpu[0], gpu.Length); buffer.View.CopyToCPU(stream, ref gpu[0], gpu.Length);
stream.Synchronize();
for (int i = 0; i < cpu.Length; i++) for (int i = 0; i < cpu.Length; i++)
{ {
gpu[i].Should().Be(cpu[i]); gpu[i].Should().Be(cpu[i]);
} }
}
finally stream.Dispose();
{
context.Dispose(); context.Dispose();
} }
}
[Fact] [Fact]
[Trait("Category", "Fast")] [Trait("Category", "Fast")]
PerfectNumbers.Core/DivisorCycleCache.cs
+10 -5
View File
@@ -4,6 +4,7 @@ using System.Collections.Generic;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using ILGPU; using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime; using ILGPU.Runtime;
using PerfectNumbers.Core.Gpu; using PerfectNumbers.Core.Gpu;
@@ -15,7 +16,7 @@ public sealed class DivisorCycleCache
private const byte ByteOne = 1; private const byte ByteOne = 1;
private const int StackBufferThreshold = 128; private const int StackBufferThreshold = 128;
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>> _gpuKernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>> _gpuKernelCache = new(AcceleratorReferenceComparer.Instance);
private volatile ulong[] _snapshot; private volatile ulong[] _snapshot;
private volatile bool _useGpuGeneration = true; private volatile bool _useGpuGeneration = true;
private readonly int _divisorCyclesBatchSize; private readonly int _divisorCyclesBatchSize;
@@ -284,10 +285,11 @@ public sealed class DivisorCycleCache
int pending; int pending;
do do
{ {
kernel(length, _divisorCyclesBatchSize, divisorBuffer.View, powBuffer.View, orderBuffer.View, resultBuffer.View, statusBuffer.View); kernel(stream, length, _divisorCyclesBatchSize, divisorBuffer.View, powBuffer.View, orderBuffer.View, resultBuffer.View, statusBuffer.View);
stream.Synchronize();
statusBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(statusSpan), length); statusBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(statusSpan), length);
stream.Synchronize();
pending = 0; pending = 0;
for (int i = 0; i < length; i++) for (int i = 0; i < length; i++)
{ {
@@ -300,6 +302,7 @@ public sealed class DivisorCycleCache
while (pending > 0); while (pending > 0);
resultBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length); resultBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length);
stream.Synchronize();
resultSpan.CopyTo(destination); resultSpan.CopyTo(destination);
if (rentedPow is not null) if (rentedPow is not null)
{ {
@@ -312,9 +315,11 @@ public sealed class DivisorCycleCache
gpuLease.Dispose(); gpuLease.Dispose();
} }
private static Action<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>> LoadKernel(Accelerator accelerator) private static Action<AcceleratorStream, Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>> LoadKernel(Accelerator accelerator)
{ {
return accelerator.LoadAutoGroupedStreamKernel<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel); var loaded = accelerator.LoadAutoGroupedStreamKernel<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>>();
} }
private sealed class AcceleratorReferenceComparer : IEqualityComparer<Accelerator> private sealed class AcceleratorReferenceComparer : IEqualityComparer<Accelerator>
PerfectNumbers.Core/Gpu/GpuContextPool.cs
-1
View File
@@ -102,7 +102,6 @@ public static class GpuContextPool
internal static void Return(PooledContext ctx) internal static void Return(PooledContext ctx)
{ {
// ctx.Accelerator.Synchronize();
GpuPool.Enqueue(ctx); GpuPool.Enqueue(ctx);
} }
} }
PerfectNumbers.Core/Gpu/GpuKernelLease.cs
+4 -3
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using ILGPU; using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime; using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
@@ -19,7 +20,7 @@ public sealed class GpuKernelLease
Kernels = kernels; Kernels = kernels;
} }
internal static GpuKernelLease Rent(GpuContextLease gpu, KernelContainer kernels) internal static GpuKernelLease Rent(GpuContextLease gpu, AcceleratorStream stream, KernelContainer kernels)
{ {
if (!Pool.TryDequeue(out var lease)) if (!Pool.TryDequeue(out var lease))
{ {
@@ -28,7 +29,7 @@ public sealed class GpuKernelLease
lease._gpu = gpu; lease._gpu = gpu;
lease.Kernels = kernels; lease.Kernels = kernels;
lease.Stream = gpu.Accelerator.CreateStream(); lease.Stream = stream;
return lease; return lease;
} }
@@ -66,7 +67,7 @@ public sealed class GpuKernelLease
{ {
get get
{ {
var accel = Accelerator; var accel = Accelerator; // avoid capturing 'this' in lambda
return GpuKernelPool.InitOnce(ref Kernels!.Incremental, () => return GpuKernelPool.InitOnce(ref Kernels!.Incremental, () =>
{ {
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernels.IncrementalKernelScan); var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernels.IncrementalKernelScan);
PerfectNumbers.Core/Gpu/GpuKernelPool.cs
+20 -19
View File
@@ -111,26 +111,26 @@ public class GpuKernelPool
return Interlocked.CompareExchange(ref target, newValue, null) ?? newValue; return Interlocked.CompareExchange(ref target, newValue, null) ?? newValue;
} }
internal static KernelContainer GetKernels(Accelerator accelerator) internal static KernelContainer GetKernels(Accelerator accelerator, AcceleratorStream stream)
{ {
return _kernels.GetOrAdd(accelerator, static (accelerator) => return _kernels.GetOrAdd(accelerator, (accelerator) =>
{ {
var kernels = new KernelContainer(); var kernels = new KernelContainer();
PreloadStaticTables(kernels, accelerator); PreloadStaticTables(kernels, accelerator, stream);
return kernels; return kernels;
}); });
} }
internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator) internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{ {
EnsureSmallCyclesOnDevice(kernels, accelerator); EnsureSmallCyclesOnDevice(kernels, accelerator, stream);
EnsureSmallPrimesOnDevice(kernels, accelerator); EnsureSmallPrimesOnDevice(kernels, accelerator, stream);
EnsureSmallPrimeFactorTables(kernels, accelerator); EnsureSmallPrimeFactorTables(kernels, accelerator, stream);
} }
// Ensures the small cycles table is uploaded to the device for the given accelerator. // Ensures the small cycles table is uploaded to the device for the given accelerator.
// Returns the ArrayView to pass into kernels (when kernels are extended to accept it). // Returns the ArrayView to pass into kernels (when kernels are extended to accept it).
public static ArrayView1D<ulong, Stride1D.Dense> EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator) public static ArrayView1D<ulong, Stride1D.Dense> EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{ {
if (kernels.SmallCycles is { } buffer) if (kernels.SmallCycles is { } buffer)
{ {
@@ -139,12 +139,12 @@ public class GpuKernelPool
var host = MersenneDivisorCycles.Shared.ExportSmallCyclesSnapshot(); // TODO: Preload this device buffer during startup and keep it immutable so we can delete the lock above in favor of the preloaded snapshot. var host = MersenneDivisorCycles.Shared.ExportSmallCyclesSnapshot(); // TODO: Preload this device buffer during startup and keep it immutable so we can delete the lock above in favor of the preloaded snapshot.
var device = accelerator.Allocate1D<ulong>(host.Length); var device = accelerator.Allocate1D<ulong>(host.Length);
device.View.CopyFromCPU(host); device.View.CopyFromCPU(stream, host);
kernels.SmallCycles = device; kernels.SmallCycles = device;
return device.View; return device.View;
} }
public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator) public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{ {
if (kernels.SmallPrimesLastOne is { } lastOne && if (kernels.SmallPrimesLastOne is { } lastOne &&
kernels.SmallPrimesLastSeven is { } lastSeven && kernels.SmallPrimesLastSeven is { } lastSeven &&
@@ -168,13 +168,13 @@ public class GpuKernelPool
var hostLastSevenPow2 = PrimesGenerator.SmallPrimesPow2LastSeven; var hostLastSevenPow2 = PrimesGenerator.SmallPrimesPow2LastSeven;
var deviceLastOne = accelerator.Allocate1D<uint>(hostLastOne.Length); var deviceLastOne = accelerator.Allocate1D<uint>(hostLastOne.Length);
deviceLastOne.View.CopyFromCPU(hostLastOne); deviceLastOne.View.CopyFromCPU(stream, hostLastOne);
var deviceLastSeven = accelerator.Allocate1D<uint>(hostLastSeven.Length); var deviceLastSeven = accelerator.Allocate1D<uint>(hostLastSeven.Length);
deviceLastSeven.View.CopyFromCPU(hostLastSeven); deviceLastSeven.View.CopyFromCPU(stream, hostLastSeven);
var deviceLastOnePow2 = accelerator.Allocate1D<ulong>(hostLastOnePow2.Length); var deviceLastOnePow2 = accelerator.Allocate1D<ulong>(hostLastOnePow2.Length);
deviceLastOnePow2.View.CopyFromCPU(hostLastOnePow2); deviceLastOnePow2.View.CopyFromCPU(stream, hostLastOnePow2);
var deviceLastSevenPow2 = accelerator.Allocate1D<ulong>(hostLastSevenPow2.Length); var deviceLastSevenPow2 = accelerator.Allocate1D<ulong>(hostLastSevenPow2.Length);
deviceLastSevenPow2.View.CopyFromCPU(hostLastSevenPow2); deviceLastSevenPow2.View.CopyFromCPU(stream, hostLastSevenPow2);
kernels.SmallPrimesLastOne = deviceLastOne; kernels.SmallPrimesLastOne = deviceLastOne;
kernels.SmallPrimesLastSeven = deviceLastSeven; kernels.SmallPrimesLastSeven = deviceLastSeven;
@@ -184,7 +184,7 @@ public class GpuKernelPool
return new ResiduePrimeViews(deviceLastOne.View, deviceLastSeven.View, deviceLastOnePow2.View, deviceLastSevenPow2.View); return new ResiduePrimeViews(deviceLastOne.View, deviceLastSeven.View, deviceLastOnePow2.View, deviceLastSevenPow2.View);
} }
public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator) public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{ {
if (kernels.SmallPrimeFactorsPrimes is { } primeBuffer && kernels.SmallPrimeFactorsSquares is { } squareBuffer) if (kernels.SmallPrimeFactorsPrimes is { } primeBuffer && kernels.SmallPrimeFactorsSquares is { } squareBuffer)
{ {
@@ -200,9 +200,9 @@ public class GpuKernelPool
var hostSquares = PrimesGenerator.SmallPrimesPow2; var hostSquares = PrimesGenerator.SmallPrimesPow2;
var devicePrimes = accelerator.Allocate1D<uint>(hostPrimes.Length); var devicePrimes = accelerator.Allocate1D<uint>(hostPrimes.Length);
devicePrimes.View.CopyFromCPU(hostPrimes); devicePrimes.View.CopyFromCPU(stream, hostPrimes);
var deviceSquares = accelerator.Allocate1D<ulong>(hostSquares.Length); var deviceSquares = accelerator.Allocate1D<ulong>(hostSquares.Length);
deviceSquares.View.CopyFromCPU(hostSquares); deviceSquares.View.CopyFromCPU(stream, hostSquares);
kernels.SmallPrimeFactorsPrimes = devicePrimes; kernels.SmallPrimeFactorsPrimes = devicePrimes;
kernels.SmallPrimeFactorsSquares = deviceSquares; kernels.SmallPrimeFactorsSquares = deviceSquares;
@@ -218,8 +218,9 @@ public class GpuKernelPool
{ {
GpuPrimeWorkLimiter.Acquire(); GpuPrimeWorkLimiter.Acquire();
var gpu = Rent(); var gpu = Rent();
var kernels = GetKernels(gpu.Accelerator); var stream = gpu.Accelerator.CreateStream();
return GpuKernelLease.Rent(gpu, kernels); var kernels = GetKernels(gpu.Accelerator, stream);
return GpuKernelLease.Rent(gpu, stream, kernels);
} }
/// <summary> /// <summary>
PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs
-10
View File
@@ -44,11 +44,6 @@ public sealed class ScratchBuffer
SmallPrimeFactorPrimeSlots = _accelerator.Allocate1D<ulong>(newSize); SmallPrimeFactorPrimeSlots = _accelerator.Allocate1D<ulong>(newSize);
SmallPrimeFactorExponentSlots = _accelerator.Allocate1D<int>(newSize); SmallPrimeFactorExponentSlots = _accelerator.Allocate1D<int>(newSize);
} }
// else
// {
// SmallPrimeFactorPrimeSlots.MemSetToZero();
// SmallPrimeFactorExponentSlots.MemSetToZero();
// }
} }
public void ResizeSpecialMaxFactors(int newSize) public void ResizeSpecialMaxFactors(int newSize)
@@ -61,11 +56,6 @@ public sealed class ScratchBuffer
SpecialMaxFactors = _accelerator.Allocate1D<ulong>(newSize); SpecialMaxFactors = _accelerator.Allocate1D<ulong>(newSize);
SpecialMaxCandidates = _accelerator.Allocate1D<ulong>(newSize); SpecialMaxCandidates = _accelerator.Allocate1D<ulong>(newSize);
} }
// else
// {
// SpecialMaxFactors.MemSetToZero();
// SpecialMaxCandidates.MemSetToZero();
// }
} }
} }
PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs
+4 -4
View File
@@ -11,13 +11,13 @@ internal static class GpuStaticTableInitializer
{ {
private static readonly ConcurrentDictionary<Accelerator, byte> Initialized = new(AcceleratorReferenceComparer.Instance); private static readonly ConcurrentDictionary<Accelerator, byte> Initialized = new(AcceleratorReferenceComparer.Instance);
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator) internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{ {
Initialized.GetOrAdd(accelerator, acc => Initialized.GetOrAdd(accelerator, acc =>
{ {
PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc); PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc, stream);
GpuKernelPool.PreloadStaticTables(kernels, acc); GpuKernelPool.PreloadStaticTables(kernels, acc, stream);
PrimeOrderGpuHeuristics.PreloadStaticTables( acc); PrimeOrderGpuHeuristics.PreloadStaticTables( acc, stream);
return 0; return 0;
}); });
} }
PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs
+1 -1
View File
@@ -305,7 +305,7 @@ internal static class LucasLehmerKernels
states[idx] = s; states[idx] = s;
} }
public static void Kernel(Index1D index, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state) public static void Kernel(Index1D _, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
{ {
// Lucas–Lehmer iteration in the field GF(2^p-1). // Lucas–Lehmer iteration in the field GF(2^p-1).
// For p < 128, use a Mersenne-specific squaring + reduction to avoid // For p < 128, use a Mersenne-specific squaring + reduction to avoid
PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs
+2 -1
View File
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using ILGPU; using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
@@ -32,7 +33,7 @@ internal static class NttBarrettKernels
} }
// Barrett-based squaring: v = v^2 mod n for 128-bit n // Barrett-based squaring: v = v^2 mod n for 128-bit n
public static void SquareBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow) public static void SquareBarrett128Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
{ {
var v = data[index]; var v = data[index];
Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0); Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0);
PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU; using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs
+3 -2
View File
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using ILGPU; using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
@@ -29,7 +30,7 @@ internal static class NttMontgomeryKernels
data[i2] = new GpuUInt128(0UL, diff); data[i2] = new GpuUInt128(0UL, diff);
} }
public static void ToMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2) public static void ToMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
{ {
ulong a = data[index].Low; ulong a = data[index].Low;
ulong v = MontMul64(a, r2, modulus, nPrime); ulong v = MontMul64(a, r2, modulus, nPrime);
@@ -43,7 +44,7 @@ internal static class NttMontgomeryKernels
data[index] = new GpuUInt128(0UL, v); data[index] = new GpuUInt128(0UL, v);
} }
public static void SquareMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime) public static void SquareMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
{ {
ulong aR = data[index].Low; ulong aR = data[index].Low;
ulong v = MontMul64(aR, aR, modulus, nPrime); ulong v = MontMul64(aR, aR, modulus, nPrime);
PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU; using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU; using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs
+37 -37
View File
@@ -24,14 +24,17 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private ulong _divisorLimit; private ulong _divisorLimit;
// private bool _isConfigured; // private bool _isConfigured;
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>> _kernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>> _kernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentBag<BatchResources> _resourcePool = new(); private readonly ConcurrentBag<BatchResources> _resourcePool = new();
private readonly ConcurrentBag<GpuContextLease> _acceleratorPool = new();
private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new(); private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new();
private Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> GetKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc => _kernelCache.GetOrAdd(accelerator, acc =>
acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.CheckKernel)); {
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.CheckKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>>();
});
public int GpuBatchSize public int GpuBatchSize
@@ -92,8 +95,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ulong processedCount; ulong processedCount;
ulong lastProcessed; ulong lastProcessed;
var gpuLease = RentAccelerator(); var gpuLease = GpuKernelPool.GetKernel();
var accelerator = gpuLease.Accelerator; var accelerator = gpuLease.Accelerator;
var stream = gpuLease.Stream;
// Monitor.Enter(gpuLease.ExecutionLock); // Monitor.Enter(gpuLease.ExecutionLock);
@@ -104,6 +108,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
prime, prime,
allowedMax, allowedMax,
kernel, kernel,
stream,
resources.DivisorDataBuffer, resources.DivisorDataBuffer,
resources.OffsetBuffer, resources.OffsetBuffer,
resources.CountBuffer, resources.CountBuffer,
@@ -123,7 +128,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ReturnBatchResources(resources); ReturnBatchResources(resources);
// Monitor.Exit(gpuLease.ExecutionLock); // Monitor.Exit(gpuLease.ExecutionLock);
ReturnAccelerator(gpuLease); gpuLease.Dispose();
if (composite) if (composite)
{ {
@@ -167,7 +172,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private static bool CheckDivisors( private static bool CheckDivisors(
ulong prime, ulong prime,
ulong allowedMax, ulong allowedMax,
Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> kernel, Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> kernel,
AcceleratorStream stream,
MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> divisorDataBuffer, MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> divisorDataBuffer,
MemoryBuffer1D<int, Stride1D.Dense> offsetBuffer, MemoryBuffer1D<int, Stride1D.Dense> offsetBuffer,
MemoryBuffer1D<int, Stride1D.Dense> countBuffer, MemoryBuffer1D<int, Stride1D.Dense> countBuffer,
@@ -329,19 +335,20 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
continue; continue;
} }
divisorDataView.CopyFromCPU(ref divisorDataRef, admissibleCount); divisorDataView.CopyFromCPU(stream, ref divisorDataRef, admissibleCount);
offsetView.CopyFromCPU(ref offsetRef, admissibleCount); offsetView.CopyFromCPU(stream, ref offsetRef, admissibleCount);
countView.CopyFromCPU(ref countRef, admissibleCount); countView.CopyFromCPU(stream, ref countRef, admissibleCount);
cycleView.CopyFromCPU(ref cycleRef, admissibleCount); cycleView.CopyFromCPU(stream, ref cycleRef, admissibleCount);
exponentViewDevice.CopyFromCPU(ref exponentRef, admissibleCount); exponentViewDevice.CopyFromCPU(stream, ref exponentRef, admissibleCount);
int sentinel = int.MaxValue; int sentinel = int.MaxValue;
hitIndexView.CopyFromCPU(ref sentinel, 1); hitIndexView.CopyFromCPU(stream, ref sentinel, 1);
kernel(admissibleCount, divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView); kernel(stream, new Index1D(admissibleCount), divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView);
int firstHit = sentinel; int firstHit = sentinel;
hitIndexView.CopyToCPU(ref firstHit, 1); hitIndexView.CopyToCPU(stream, ref firstHit, 1);
stream.Synchronize();
int hitIndex = firstHit >= admissibleCount ? -1 : firstHit; int hitIndex = firstHit >= admissibleCount ? -1 : firstHit;
bool hitFound = hitIndex >= 0; bool hitFound = hitIndex >= 0;
composite = hitFound; composite = hitFound;
@@ -477,9 +484,10 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
public sealed class DivisorScanSession : IMersenneNumberDivisorByDivisorTester.IDivisorScanSession public sealed class DivisorScanSession : IMersenneNumberDivisorByDivisorTester.IDivisorScanSession
{ {
private readonly MersenneNumberDivisorByDivisorGpuTester _owner; private readonly MersenneNumberDivisorByDivisorGpuTester _owner;
private readonly GpuContextLease _lease; private readonly GpuKernelLease _lease;
private readonly Accelerator _accelerator; private readonly Accelerator _accelerator;
private Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> _kernel = null!; private readonly AcceleratorStream _stream;
private Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> _kernel = null!;
private MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> _divisorBuffer = null!; private MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> _divisorBuffer = null!;
private MemoryBuffer1D<int, Stride1D.Dense> _offsetBuffer = null!; private MemoryBuffer1D<int, Stride1D.Dense> _offsetBuffer = null!;
private MemoryBuffer1D<int, Stride1D.Dense> _countBuffer = null!; private MemoryBuffer1D<int, Stride1D.Dense> _countBuffer = null!;
@@ -493,8 +501,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
internal DivisorScanSession(MersenneNumberDivisorByDivisorGpuTester owner) internal DivisorScanSession(MersenneNumberDivisorByDivisorGpuTester owner)
{ {
_owner = owner; _owner = owner;
_lease = GpuContextPool.Rent(); _lease = GpuKernelPool.GetKernel();
_accelerator = _lease.Accelerator; _accelerator = _lease.Accelerator;
_stream = _lease.Stream;
_capacity = Math.Max(1, owner._gpuBatchSize); _capacity = Math.Max(1, owner._gpuBatchSize);
} }
@@ -593,35 +602,36 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ArrayView1D<ulong, Stride1D.Dense> exponentView = _exponentsBuffer.View; ArrayView1D<ulong, Stride1D.Dense> exponentView = _exponentsBuffer.View;
ArrayView1D<byte, Stride1D.Dense> hitView = _hitBuffer.View; ArrayView1D<byte, Stride1D.Dense> hitView = _hitBuffer.View;
AcceleratorStream stream = _stream;
GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor); GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor);
divisorView.CopyFromCPU(ref partialData, 1); divisorView.CopyFromCPU(stream, ref partialData, 1);
int offsetValue = 0; int offsetValue = 0;
offsetView.CopyFromCPU(ref offsetValue, 1); offsetView.CopyFromCPU(stream, ref offsetValue, 1);
int countValue = length; int countValue = length;
countView.CopyFromCPU(ref countValue, 1); countView.CopyFromCPU(stream, ref countValue, 1);
ulong cycleValue = cycle; ulong cycleValue = cycle;
cycleView.CopyFromCPU(ref cycleValue, 1); cycleView.CopyFromCPU(stream, ref cycleValue, 1);
Span<ulong> hostSpan = _hostBuffer.AsSpan(0, length); Span<ulong> hostSpan = _hostBuffer.AsSpan(0, length);
primes.CopyTo(hostSpan); primes.CopyTo(hostSpan);
ref ulong hostRef = ref MemoryMarshal.GetReference(hostSpan); ref ulong hostRef = ref MemoryMarshal.GetReference(hostSpan);
exponentView.CopyFromCPU(ref hostRef, length); exponentView.CopyFromCPU(stream, ref hostRef, length);
int sentinel = int.MaxValue; int sentinel = int.MaxValue;
firstHitView.CopyFromCPU(ref sentinel, 1); firstHitView.CopyFromCPU(stream, ref sentinel, 1);
var kernel = _kernel!; var kernel = _kernel!;
kernel(1, divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView); kernel(stream, new Index1D(1), divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView);
Span<byte> hitSlice = hits.Slice(0, length); Span<byte> hitSlice = hits.Slice(0, length);
ref byte hitRef = ref MemoryMarshal.GetReference(hitSlice); ref byte hitRef = ref MemoryMarshal.GetReference(hitSlice);
hitView.CopyToCPU(ref hitRef, length); hitView.CopyToCPU(stream, ref hitRef, length);
stream.Synchronize();
// Monitor.Exit(_lease.ExecutionLock); // Monitor.Exit(_lease.ExecutionLock);
} }
@@ -651,17 +661,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
} }
} }
private GpuContextLease RentAccelerator()
{
if (_acceleratorPool.TryTake(out var lease))
{
return lease;
}
return GpuContextPool.Rent();
}
private void ReturnAccelerator(GpuContextLease lease) => _acceleratorPool.Add(lease);
private BatchResources RentBatchResources(Accelerator accelerator, int capacity) private BatchResources RentBatchResources(Accelerator accelerator, int capacity)
{ {
PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs
+12 -9
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using ILGPU; using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime; using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu; namespace PerfectNumbers.Core.Gpu;
@@ -9,11 +10,16 @@ namespace PerfectNumbers.Core.Gpu;
/// </summary> /// </summary>
public sealed class MersenneNumberDivisorGpuTester public sealed class MersenneNumberDivisorGpuTester
{ {
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new();
private readonly ConcurrentDictionary<Accelerator, MemoryBuffer1D<byte, Stride1D.Dense>> _resultBuffers = new(); private readonly ConcurrentDictionary<Accelerator, MemoryBuffer1D<byte, Stride1D.Dense>> _resultBuffers = new();
private Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel)); _kernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>>();
});
public static void BuildDivisorCandidates() public static void BuildDivisorCandidates()
{ {
@@ -43,15 +49,12 @@ public sealed class MersenneNumberDivisorGpuTester
var kernel = GetKernel(accelerator); var kernel = GetKernel(accelerator);
var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D<byte>(1)); var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D<byte>(1));
// There is no point in clearing this buffer. We always override item [0] and never use it beyond item [0] // There is no point in clearing this buffer. We always override item [0] and never use it beyond item [0]
// resultBuffer.MemSetToZero(); // resultBuffer.MemSetToZero(stream);
kernel(1, exponent, divisor, resultBuffer.View); kernel(stream, 1, exponent, divisor, resultBuffer.View);
stream.Synchronize();
Span<byte> result = stackalloc byte[1]; Span<byte> result = stackalloc byte[1];
resultBuffer.View.CopyToCPU(stream, ref result[0], 1); resultBuffer.View.CopyToCPU(stream, ref result[0], 1);
bool divisible = result[0] != 0;
result[0] = 0;
resultBuffer.View.CopyFromCPU(stream, ref result[0], 1);
stream.Synchronize(); stream.Synchronize();
bool divisible = result[0] != 0;
stream.Dispose(); stream.Dispose();
gpu.Dispose(); gpu.Dispose();
return divisible; return divisible;
PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs
+3 -3
View File
@@ -31,11 +31,11 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u
ulong step3 = ((exponent % 3UL) << 1) % 3UL; ulong step3 = ((exponent % 3UL) << 1) % 3UL;
ulong step5 = ((exponent % 5UL) << 1) % 5UL; ulong step5 = ((exponent % 5UL) << 1) % 5UL;
GpuUInt128 twoPGpu = (GpuUInt128)twoP; GpuUInt128 twoPGpu = (GpuUInt128)twoP;
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator); var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream);
ResiduePrimeViews primeViews = default; ResiduePrimeViews primeViews = default;
if (_kernelType == GpuKernelType.Pow2Mod) if (_kernelType == GpuKernelType.Pow2Mod)
{ {
primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator); primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator, stream);
} }
var orderBuffer = accelerator.Allocate1D<ulong>(batchSize); var orderBuffer = accelerator.Allocate1D<ulong>(batchSize);
@@ -92,8 +92,8 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u
smallCyclesView); smallCyclesView);
} }
orderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(orders), currentSize);
stream.Synchronize(); stream.Synchronize();
orderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(orders), currentSize);
if (_kernelType == GpuKernelType.Pow2Mod) if (_kernelType == GpuKernelType.Pow2Mod)
{ {
for (i = 0; i < currentSize; i++) for (i = 0; i < currentSize; i++)
PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs
+71 -37
View File
@@ -1,6 +1,7 @@
using System.Buffers; using System.Buffers;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using ILGPU; using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime; using ILGPU.Runtime;
using UInt128 = System.UInt128; using UInt128 = System.UInt128;
@@ -8,32 +9,60 @@ namespace PerfectNumbers.Core.Gpu;
public class MersenneNumberLucasLehmerGpuTester public class MersenneNumberLucasLehmerGpuTester
{ {
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>> KernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>> KernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> AddSmallKernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> AddSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> SubSmallKernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> SubSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> ReduceKernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> ReduceKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>> IsZeroKernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>> IsZeroKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>> BatchKernelCache = new(); private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>> BatchKernelCache = new();
private Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator) private Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator) =>
KernelCache.GetOrAdd(accelerator, acc =>
{ {
return KernelCache.GetOrAdd(accelerator, accel => accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel)); var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel);
} var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) =>
AddSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel)); AddSmallKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) =>
SubSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel)); SubSmallKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) =>
ReduceKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel)); ReduceKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) =>
IsZeroKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel)); IsZeroKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>>();
});
private Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) => private Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) =>
BatchKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch)); BatchKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>>();
});
// Configurable LL slice size to keep kernels short. Default 32. // Configurable LL slice size to keep kernels short. Default 32.
public int SliceSize = 32; public int SliceSize = 32;
@@ -70,13 +99,16 @@ public class MersenneNumberLucasLehmerGpuTester
{ {
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetKernel(accelerator); var kernel = GetKernel(accelerator);
var modulus = new GpuUInt128(((UInt128)1 << (int)exponent) - 1UL); // TODO: Cache these Mersenne moduli per exponent so LL GPU runs skip rebuilding them every launch. var modulus = new GpuUInt128(((UInt128)1 << (int)exponent) - 1UL); // TODO: Cache these Mersenne moduli per exponent so LL GPU runs skip rebuilding them every launch.
var buffer = accelerator.Allocate1D<GpuUInt128>(1); var buffer = accelerator.Allocate1D<GpuUInt128>(1);
kernel(1, exponent, modulus, buffer.View); kernel(stream, 1, exponent, modulus, buffer.View);
accelerator.Synchronize(); stream.Synchronize();
// TODO: Replace this with .CopyToCPU with pre-allocated buffer using stream. Synchronize on the stream after copy.
result = buffer.GetAsArray1D()[0].IsZero; result = buffer.GetAsArray1D()[0].IsZero;
buffer.Dispose(); buffer.Dispose();
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
@@ -102,12 +134,13 @@ public class MersenneNumberLucasLehmerGpuTester
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetBatchKernel(accelerator); var kernel = GetBatchKernel(accelerator);
var expBuffer = accelerator.Allocate1D<ulong>(count); var expBuffer = accelerator.Allocate1D<ulong>(count);
ulong[] expArray = ArrayPool<ulong>.Shared.Rent(count); ulong[] expArray = ArrayPool<ulong>.Shared.Rent(count);
exponents.CopyTo(expArray); exponents.CopyTo(expArray);
expBuffer.View.CopyFromCPU(ref expArray[0], count); expBuffer.View.CopyFromCPU(stream, ref expArray[0], count);
GpuUInt128[] modulusArray = ArrayPool<GpuUInt128>.Shared.Rent(count); GpuUInt128[] modulusArray = ArrayPool<GpuUInt128>.Shared.Rent(count);
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
@@ -118,18 +151,19 @@ public class MersenneNumberLucasLehmerGpuTester
} }
var modBuffer = accelerator.Allocate1D<GpuUInt128>(count); var modBuffer = accelerator.Allocate1D<GpuUInt128>(count);
modBuffer.View.CopyFromCPU(ref modulusArray[0], count); modBuffer.View.CopyFromCPU(stream, ref modulusArray[0], count);
var stateBuffer = accelerator.Allocate1D<GpuUInt128>(count); var stateBuffer = accelerator.Allocate1D<GpuUInt128>(count);
kernel(count, expBuffer.View, modBuffer.View, stateBuffer.View); kernel(stream, count, expBuffer.View, modBuffer.View, stateBuffer.View);
accelerator.Synchronize(); stateBuffer.View.CopyToCPU(stream, ref residues[0], count);
stateBuffer.View.CopyToCPU(ref residues[0], count); stream.Synchronize();
stateBuffer.Dispose(); stateBuffer.Dispose();
modBuffer.Dispose(); modBuffer.Dispose();
expBuffer.Dispose(); expBuffer.Dispose();
ArrayPool<GpuUInt128>.Shared.Return(modulusArray); ArrayPool<GpuUInt128>.Shared.Return(modulusArray);
ArrayPool<ulong>.Shared.Return(expArray); ArrayPool<ulong>.Shared.Return(expArray);
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
@@ -154,6 +188,7 @@ public class MersenneNumberLucasLehmerGpuTester
{ {
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var addKernel = GetAddSmallKernel(accelerator); var addKernel = GetAddSmallKernel(accelerator);
var subKernel = GetSubSmallKernel(accelerator); var subKernel = GetSubSmallKernel(accelerator);
var reduceKernel = GetReduceKernel(accelerator); var reduceKernel = GetReduceKernel(accelerator);
@@ -167,8 +202,8 @@ public class MersenneNumberLucasLehmerGpuTester
transformLength <<= 1; transformLength <<= 1;
} }
var stateBuffer = accelerator.Allocate1D<GpuUInt128>(transformLength); var stateBuffer = accelerator.Allocate1D<GpuUInt128>(transformLength);
stateBuffer.MemSetToZero(); stateBuffer.MemSetToZero(stream);
addKernel(1, 4UL, stateBuffer.View); addKernel(stream, 1, 4UL, stateBuffer.View);
// TODO(LL-SLICE): Slice Lucas–Lehmer iterations into short batches to // TODO(LL-SLICE): Slice Lucas–Lehmer iterations into short batches to
// avoid long-running kernels and TDR. Example: process 8–64 iterations // avoid long-running kernels and TDR. Example: process 8–64 iterations
@@ -183,26 +218,25 @@ public class MersenneNumberLucasLehmerGpuTester
for (int s = 0; s < iter; s++) for (int s = 0; s < iter; s++)
{ {
// Square in NTT domain with staged kernels (short runtime per kernel). // Square in NTT domain with staged kernels (short runtime per kernel).
NttGpuMath.SquareDevice(accelerator, stateBuffer.View, nttMod, primitiveRoot); NttGpuMath.SquareDevice(accelerator, stream, stateBuffer.View, nttMod, primitiveRoot);
subKernel(1, 2UL, stateBuffer.View); subKernel(stream, 1, 2UL, stateBuffer.View);
reduceKernel(1, exponent, stateBuffer.View); reduceKernel(stream, 1, exponent, stateBuffer.View);
} }
// Synchronize between slices to yield to scheduler and avoid TDR. // Synchronize between slices to yield to scheduler and avoid TDR.
accelerator.Synchronize();
i += (ulong)iter; i += (ulong)iter;
} }
reduceKernel(1, exponent, stateBuffer.View); reduceKernel(stream, 1, exponent, stateBuffer.View);
var resultBuffer = accelerator.Allocate1D<byte>(1); var resultBuffer = accelerator.Allocate1D<byte>(1);
zeroKernel(1, stateBuffer.View, resultBuffer.View); zeroKernel(stream, 1, stateBuffer.View, resultBuffer.View);
accelerator.Synchronize();
byte[] result = new byte[1]; byte[] result = new byte[1];
resultBuffer.View.CopyToCPU(result); resultBuffer.View.CopyToCPU(stream, result);
stream.Synchronize();
bool isPrime = result[0] != 0; bool isPrime = result[0] != 0;
resultBuffer.Dispose(); resultBuffer.Dispose();
stateBuffer.Dispose(); stateBuffer.Dispose();
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
return isPrime; return isPrime;
} }
PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs
+3 -3
View File
@@ -30,13 +30,13 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO
int currentSize; int currentSize;
int found = 0; int found = 0;
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator); var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream);
while (kStart <= maxK && Volatile.Read(ref isPrime)) while (kStart <= maxK && Volatile.Read(ref isPrime))
{ {
remaining = maxK - kStart + 1UL; remaining = maxK - kStart + 1UL;
currentSize = remaining > (UInt128)batchSize ? batchSize : (int)remaining; currentSize = remaining > (UInt128)batchSize ? batchSize : (int)remaining;
// This clear is required because the result may not always be set. Remove it after the kernels are modified to always set the result. // This clear is required because the result may not always be set. Remove it after the kernels are modified to always set the result.
foundBuffer.MemSetToZero(); foundBuffer.MemSetToZero(stream);
// Precompute residue automaton bases for this batch // Precompute residue automaton bases for this batch
UInt128 q0 = twoP * kStart + 1UL; UInt128 q0 = twoP * kStart + 1UL;
// TODO: Replace these modulo recomputations with the divisor-cycle stepping deltas captured in // TODO: Replace these modulo recomputations with the divisor-cycle stepping deltas captured in
@@ -49,8 +49,8 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO
kernel(stream, currentSize, exponent, (GpuUInt128)twoP, (GpuUInt128)kStart, last, divMul, kernel(stream, currentSize, exponent, (GpuUInt128)twoP, (GpuUInt128)kStart, last, divMul,
kernelArgs, foundBuffer.View, smallCyclesView); kernelArgs, foundBuffer.View, smallCyclesView);
foundBuffer.View.CopyToCPU(stream, ref found, 1);
stream.Synchronize(); stream.Synchronize();
foundBuffer.View.CopyToCPU(ref found, 1);
if (found != 0) if (found != 0)
{ {
Volatile.Write(ref isPrime, false); Volatile.Write(ref isPrime, false);
PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs
+6 -5
View File
@@ -36,15 +36,16 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder)
var stream = accelerator.CreateStream(); var stream = accelerator.CreateStream();
var kernel = GetPow2ModKernel(accelerator); var kernel = GetPow2ModKernel(accelerator);
var resources = RentResources(accelerator, GpuConstants.ScanBatchSize); var resources = RentResources(accelerator, GpuConstants.ScanBatchSize);
var gpuKernels = GpuKernelPool.GetKernels(accelerator); var gpuKernels = GpuKernelPool.GetKernels(accelerator, stream);
var orderBuffer = resources.OrderBuffer; var orderBuffer = resources.OrderBuffer;
ulong[] orderArray = resources.OrderArray; ulong[] orderArray = resources.OrderArray;
int batchSize = resources.Capacity; int batchSize = resources.Capacity;
// Ensure device has small cycles and primes tables for in-kernel lookup // Ensure device has small cycles and primes tables for in-kernel lookup
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator); // TODO: Pass-in the stream down the path
ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator); var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator, stream);
ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator, stream);
UInt128 kStart = 1UL; UInt128 kStart = 1UL;
UInt128 limit = maxK + UInt128.One; UInt128 limit = maxK + UInt128.One;
@@ -70,9 +71,9 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder)
kernel(stream, currentSize, exponent, twoPGpu, (GpuUInt128)kStart, last, 0UL, kernel(stream, currentSize, exponent, twoPGpu, (GpuUInt128)kStart, last, 0UL,
kernelArgs, orderBuffer.View, smallCyclesView, primeViews.LastOne, primeViews.LastSeven, primeViews.LastOnePow2, primeViews.LastSevenPow2); kernelArgs, orderBuffer.View, smallCyclesView, primeViews.LastOne, primeViews.LastSeven, primeViews.LastOnePow2, primeViews.LastSevenPow2);
stream.Synchronize();
orderBuffer.View.CopyToCPU(ref ordersRef, currentSize); orderBuffer.View.CopyToCPU(stream, ref ordersRef, currentSize);
stream.Synchronize();
if (!Volatile.Read(ref isPrime)) if (!Volatile.Read(ref isPrime))
{ {
break; break;
PerfectNumbers.Core/Gpu/NttGpuMath.cs
+180 -121
View File
@@ -29,23 +29,39 @@ public static class NttGpuMath
public static NttBackend GpuTransformBackend { get; set; } = NttBackend.Reference; public static NttBackend GpuTransformBackend { get; set; } = NttBackend.Reference;
// Controls modular reduction strategy inside staged NTT paths. // Controls modular reduction strategy inside staged NTT paths.
public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto; public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto;
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> StageKernelCache = new(); // TODO: Same as above – materialize staged kernels during startup so we can drop ConcurrentDictionary usage entirely per the benchmark guidance.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Promote to the static kernel table initialized alongside the GpuModularArithmeticBenchmarks fast path.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>> StageMontKernelCache = new(); // TODO: Inline the Montgomery kernels into the startup table instead of using a concurrent cache now that we no longer mutate state at runtime.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Same plan – reuse the precomputed kernels measured fastest in MontgomeryMultiplyBenchmarks rather than looking them up dynamically.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Collapse into the startup kernel array once Barrett128 constants are preloaded.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> SquareBarrett128KernelCache = new(); // TODO: Fold these kernels into the same startup table to avoid concurrent access overhead highlighted in GpuModularArithmeticBenchmarks.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Prebind and reuse the Montgomery conversions from MontgomeryMultiplyBenchmarks instead of storing them in a concurrent cache.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong>> FromMont64KernelCache = new(); // TODO: As above – drop ConcurrentDictionary once startup prewarms every kernel.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong>> SquareMont64KernelCache = new(); // TODO: Inline into the static kernel table established during initialization.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Move to the startup kernel table so Montgomery-scaled launches skip concurrent lookups.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Warm these forward kernels during initialization and store them in a plain array so the LucasLehmerGpuBenchmarks launch costs disappear. private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream,Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> StageKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Same as above – replace with startup-prepared tables per LucasLehmerGpuBenchmarks guidance. private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int>> BitReverseKernelCache = new(); // TODO: Precompute bit-reversal kernels during boot so we can remove the concurrent lookup overhead flagged in the benchmarks. private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>> StageMontKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong>> FromMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> SquareMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int>> BitReverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) =>
StageKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>>();
});
private sealed class SquareCacheEntry private sealed class SquareCacheEntry
{ {
@@ -72,7 +88,7 @@ public static class NttGpuMath
public ulong BarrettMuHigh { get; } public ulong BarrettMuHigh { get; }
public ulong BarrettMuLow { get; } public ulong BarrettMuLow { get; }
public SquareCacheEntry(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot) public SquareCacheEntry(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{ {
Buffer = accelerator.Allocate1D<GpuUInt128>(length); Buffer = accelerator.Allocate1D<GpuUInt128>(length);
Root = new GpuUInt128(primitiveRoot); Root = new GpuUInt128(primitiveRoot);
@@ -136,9 +152,9 @@ public static class NttGpuMath
} }
Twiddles = accelerator.Allocate1D<GpuUInt128>(twiddleCount); Twiddles = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
Twiddles.View.CopyFromCPU(ref forward[0], twiddleCount); Twiddles.View.CopyFromCPU(stream, ref forward[0], twiddleCount);
TwiddlesInv = accelerator.Allocate1D<GpuUInt128>(twiddleCount); TwiddlesInv = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesInv.View.CopyFromCPU(ref inverse[0], twiddleCount); TwiddlesInv.View.CopyFromCPU(stream, ref inverse[0], twiddleCount);
// Prepare Montgomery constants for 64-bit moduli and twiddles in Montgomery domain. // Prepare Montgomery constants for 64-bit moduli and twiddles in Montgomery domain.
if (modulus.High == 0UL && (ReductionMode == ModReductionMode.Auto || ReductionMode == ModReductionMode.Mont64)) if (modulus.High == 0UL && (ReductionMode == ModReductionMode.Auto || ReductionMode == ModReductionMode.Mont64))
@@ -166,9 +182,9 @@ public static class NttGpuMath
} }
TwiddlesMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount); TwiddlesMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesMont.View.CopyFromCPU(ref forwardMont[0], twiddleCount); TwiddlesMont.View.CopyFromCPU(stream, ref forwardMont[0], twiddleCount);
TwiddlesInvMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount); TwiddlesInvMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesInvMont.View.CopyFromCPU(ref inverseMont[0], twiddleCount); TwiddlesInvMont.View.CopyFromCPU(stream, ref inverseMont[0], twiddleCount);
pool.Return(forwardMont, clearArray: true); pool.Return(forwardMont, clearArray: true);
pool.Return(inverseMont, clearArray: true); pool.Return(inverseMont, clearArray: true);
@@ -258,11 +274,11 @@ public static class NttGpuMath
// - Store one GPU buffer with all stage twiddles, or per-stage slices. // - Store one GPU buffer with all stage twiddles, or per-stage slices.
// - Expose accessors to retrieve per-stage views used by butterfly kernels. // - Expose accessors to retrieve per-stage views used by butterfly kernels.
private static SquareCacheEntry GetSquareCache(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot) private static SquareCacheEntry GetSquareCache(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{ {
var perAccel = SquareCache.GetOrAdd(accelerator, _ => new ConcurrentDictionary<SquareCacheKey, SquareCacheEntry>()); var perAccel = SquareCache.GetOrAdd(accelerator, _ => new ConcurrentDictionary<SquareCacheKey, SquareCacheEntry>());
var key = new SquareCacheKey(length, modulus, primitiveRoot); var key = new SquareCacheKey(length, modulus, primitiveRoot);
return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, length, modulus, primitiveRoot)); return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, stream, length, modulus, primitiveRoot));
} }
private static readonly ulong[] SmallPrimes = GenerateSmallPrimes(1000); private static readonly ulong[] SmallPrimes = GenerateSmallPrimes(1000);
@@ -350,6 +366,7 @@ public static class NttGpuMath
ScaleMont64KernelCache.Clear(); ScaleMont64KernelCache.Clear();
ForwardKernelCache.Clear(); ForwardKernelCache.Clear();
InverseKernelCache.Clear(); InverseKernelCache.Clear();
BitReverseKernelCache.Clear();
// TODO(NTT-OPT): Clear TwiddleCache once introduced. // TODO(NTT-OPT): Clear TwiddleCache once introduced.
return; return;
} }
@@ -373,56 +390,97 @@ public static class NttGpuMath
ScaleMont64KernelCache.TryRemove(accelerator, out _); ScaleMont64KernelCache.TryRemove(accelerator, out _);
ForwardKernelCache.TryRemove(accelerator, out _); ForwardKernelCache.TryRemove(accelerator, out _);
InverseKernelCache.TryRemove(accelerator, out _); InverseKernelCache.TryRemove(accelerator, out _);
BitReverseKernelCache.TryRemove(accelerator, out _);
// TODO(NTT-OPT): Remove per-accelerator twiddle buffers once added. // TODO(NTT-OPT): Remove per-accelerator twiddle buffers once added.
} }
private static Action<Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) =>
BitReverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel)); BitReverseKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) =>
MulKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel)); ForwardKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) =>
StageKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel)); FromMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
ScaleKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel)); InverseKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) =>
StageMontKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel)); MulKernelCache.GetOrAdd(accelerator, valueFactory: acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) =>
StageBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel)); ScaleKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) =>
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel)); ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetSquareBarrett128Kernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) =>
SquareBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.SquareBarrett128Kernel)); ScaleMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) =>
ToMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ToMont64Kernel)); StageBarrett128KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) =>
FromMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel)); StageMontKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetSquareMont64Kernel(Accelerator accelerator) => private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) =>
SquareMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.SquareMont64Kernel)); ToMont64KernelCache.GetOrAdd(accelerator, acc =>
{
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) => var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel);
ScaleMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel)); var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>>();
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) => });
ForwardKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel));
// TODO(NTT-OPT): Introduce stage-wise Cooley–Tukey kernels and a
// per-(accelerator,length,modulus,root) twiddle cache. The plan:
// - Precompute twiddle factors for all stages once and cache on GPU.
// - Replace the current O(n^2) ForwardKernel with O(n log n) butterflies.
// - Provide accessors like GetStageKernel(...) and GetTwiddleBuffer(...).
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
InverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel));
// TODO(NTT-OPT): Mirror the forward stage-wise design for the inverse // TODO(NTT-OPT): Mirror the forward stage-wise design for the inverse
// transform. Precompute inverse twiddles and a normalization factor // transform. Precompute inverse twiddles and a normalization factor
@@ -453,13 +511,15 @@ public static class NttGpuMath
var array = pool.Rent(n); var array = pool.Rent(n);
values.CopyTo(array); values.CopyTo(array);
var buffer = accelerator.Allocate1D<GpuUInt128>(n); var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref array[0], n); var stream = accelerator.CreateStream();
kernel(n, buffer.View, bits); buffer.View.CopyFromCPU(stream, ref array[0], n);
accelerator.Synchronize(); kernel(stream, n, buffer.View, bits);
buffer.View.CopyToCPU(ref array[0], n); buffer.View.CopyToCPU(stream, ref array[0], n);
stream.Synchronize();
array.AsSpan(0, n).CopyTo(values); array.AsSpan(0, n).CopyTo(values);
pool.Return(array, clearArray: true); pool.Return(array, clearArray: true);
buffer.Dispose(); buffer.Dispose();
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
@@ -510,6 +570,7 @@ public static class NttGpuMath
int n = (int)values.Length; int n = (int)values.Length;
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetForwardKernel(accelerator); var kernel = GetForwardKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared; var pool = ArrayPool<GpuUInt128>.Shared;
var inputArray = pool.Rent(n); var inputArray = pool.Rent(n);
@@ -517,19 +578,20 @@ public static class NttGpuMath
values.CopyTo(inputArray); values.CopyTo(inputArray);
var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n); var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n); var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
inputBuffer.View.CopyFromCPU(ref inputArray[0], n); inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n);
UInt128 modValue = modulus; UInt128 modValue = modulus;
UInt128 expBase = (modValue - 1UL) / (ulong)n; UInt128 expBase = (modValue - 1UL) / (ulong)n;
var root = new GpuUInt128(primitiveRoot); var root = new GpuUInt128(primitiveRoot);
root.ModPow((ulong)expBase, modulus); root.ModPow((ulong)expBase, modulus);
kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, root); kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, root);
accelerator.Synchronize(); outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n);
outputBuffer.View.CopyToCPU(ref outputArray[0], n); stream.Synchronize();
outputArray.AsSpan(0, n).CopyTo(values); outputArray.AsSpan(0, n).CopyTo(values);
pool.Return(inputArray, clearArray: true); pool.Return(inputArray, clearArray: true);
pool.Return(outputArray, clearArray: true); pool.Return(outputArray, clearArray: true);
outputBuffer.Dispose(); outputBuffer.Dispose();
inputBuffer.Dispose(); inputBuffer.Dispose();
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
@@ -607,6 +669,7 @@ public static class NttGpuMath
int n = values.Length; int n = values.Length;
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetInverseKernel(accelerator); var kernel = GetInverseKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared; var pool = ArrayPool<GpuUInt128>.Shared;
var inputArray = pool.Rent(n); var inputArray = pool.Rent(n);
@@ -614,7 +677,7 @@ public static class NttGpuMath
values.CopyTo(inputArray); values.CopyTo(inputArray);
var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n); var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n); var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
inputBuffer.View.CopyFromCPU(ref inputArray[0], n); inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n);
UInt128 modValue = modulus; UInt128 modValue = modulus;
UInt128 expBase = (modValue - 1UL) / (ulong)n; UInt128 expBase = (modValue - 1UL) / (ulong)n;
var root = new GpuUInt128(primitiveRoot); var root = new GpuUInt128(primitiveRoot);
@@ -638,14 +701,15 @@ public static class NttGpuMath
{ {
nInv.ModInv(modulus); nInv.ModInv(modulus);
} }
kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv); kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv);
accelerator.Synchronize(); outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n);
outputBuffer.View.CopyToCPU(ref outputArray[0], n); stream.Synchronize();
outputArray.AsSpan(0, n).CopyTo(values); outputArray.AsSpan(0, n).CopyTo(values);
pool.Return(inputArray, clearArray: true); pool.Return(inputArray, clearArray: true);
pool.Return(outputArray, clearArray: true); pool.Return(outputArray, clearArray: true);
outputBuffer.Dispose(); outputBuffer.Dispose();
inputBuffer.Dispose(); inputBuffer.Dispose();
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
@@ -655,11 +719,14 @@ public static class NttGpuMath
int n = values.Length; int n = values.Length;
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var buffer = accelerator.Allocate1D<GpuUInt128>(n); var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref values[0], n); buffer.View.CopyFromCPU(stream, ref values[0], n);
var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot); var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot);
ForwardDevice(accelerator, buffer.View, n, modulus, cache); ForwardDevice(accelerator, stream, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(ref values[0], n); buffer.View.CopyToCPU(stream, ref values[0], n);
stream.Synchronize();
stream.Dispose();
buffer.Dispose(); buffer.Dispose();
gpu.Dispose(); gpu.Dispose();
@@ -670,38 +737,38 @@ public static class NttGpuMath
int n = values.Length; int n = values.Length;
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var buffer = accelerator.Allocate1D<GpuUInt128>(n); var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref values[0], n); buffer.View.CopyFromCPU(stream, ref values[0], n);
var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot); var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot);
InverseDevice(accelerator, buffer.View, n, modulus, cache); InverseDevice(accelerator, stream, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(ref values[0], n); buffer.View.CopyToCPU(stream, ref values[0], n);
stream.Synchronize();
stream.Dispose();
buffer.Dispose(); buffer.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
private static void ForwardDevice(Accelerator accelerator, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache) private static void ForwardDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
{ {
var bitKernel = GetBitReverseKernel(accelerator); var bitKernel = GetBitReverseKernel(accelerator);
bitKernel(length, data, cache.Bits); bitKernel(stream, length, data, cache.Bits);
accelerator.Synchronize();
if (cache.UseMontgomery64) if (cache.UseMontgomery64)
{ {
var toMont = GetToMont64Kernel(accelerator); var toMont = GetToMont64Kernel(accelerator);
toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64); toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
accelerator.Synchronize();
var stageKernel = GetStageMontKernel(accelerator); var stageKernel = GetStageMontKernel(accelerator);
int butterflies = length >> 1; int butterflies = length >> 1;
for (int s = 0; s < cache.Bits; s++) for (int s = 0; s < cache.Bits; s++)
{ {
int len = 1 << (s + 1); int len = 1 << (s + 1);
int half = len >> 1; int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64); stageKernel( stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64);
accelerator.Synchronize();
} }
var fromMont = GetFromMont64Kernel(accelerator); var fromMont = GetFromMont64Kernel(accelerator);
fromMont(length, data, cache.ModulusLow, cache.MontNPrime64); fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64);
} }
else if (cache.UseBarrett128) else if (cache.UseBarrett128)
{ {
@@ -711,8 +778,7 @@ public static class NttGpuMath
{ {
int len = 1 << (s + 1); int len = 1 << (s + 1);
int half = len >> 1; int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
} }
} }
else else
@@ -723,39 +789,34 @@ public static class NttGpuMath
{ {
int len = 1 << (s + 1); int len = 1 << (s + 1);
int half = len >> 1; int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus); stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus);
accelerator.Synchronize();
} }
} }
accelerator.Synchronize(); stream.Synchronize();
} }
private static void InverseDevice(Accelerator accelerator, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache) private static void InverseDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
{ {
var bitKernel = GetBitReverseKernel(accelerator); var bitKernel = GetBitReverseKernel(accelerator);
bitKernel(length, data, cache.Bits); bitKernel(stream, length, data, cache.Bits);
accelerator.Synchronize();
if (cache.UseMontgomery64) if (cache.UseMontgomery64)
{ {
var toMont = GetToMont64Kernel(accelerator); var toMont = GetToMont64Kernel(accelerator);
toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64); toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
accelerator.Synchronize();
var stageKernel = GetStageMontKernel(accelerator); var stageKernel = GetStageMontKernel(accelerator);
int butterflies = length >> 1; int butterflies = length >> 1;
for (int s = 0; s < cache.Bits; s++) for (int s = 0; s < cache.Bits; s++)
{ {
int len = 1 << (s + 1); int len = 1 << (s + 1);
int half = len >> 1; int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64); stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64);
accelerator.Synchronize();
} }
var scaleKernel = GetScaleMont64Kernel(accelerator); var scaleKernel = GetScaleMont64Kernel(accelerator);
scaleKernel(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64); scaleKernel(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64);
accelerator.Synchronize();
var fromMont = GetFromMont64Kernel(accelerator); var fromMont = GetFromMont64Kernel(accelerator);
fromMont(length, data, cache.ModulusLow, cache.MontNPrime64); fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64);
} }
else if (cache.UseBarrett128) else if (cache.UseBarrett128)
{ {
@@ -765,12 +826,10 @@ public static class NttGpuMath
{ {
int len = 1 << (s + 1); int len = 1 << (s + 1);
int half = len >> 1; int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
} }
var scaleKernel = GetScaleBarrett128Kernel(accelerator); var scaleKernel = GetScaleBarrett128Kernel(accelerator);
scaleKernel(length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); scaleKernel(stream, length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
} }
else else
{ {
@@ -780,15 +839,13 @@ public static class NttGpuMath
{ {
int len = 1 << (s + 1); int len = 1 << (s + 1);
int half = len >> 1; int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus); stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus);
accelerator.Synchronize();
} }
var scaleKernel = GetScaleKernel(accelerator); var scaleKernel = GetScaleKernel(accelerator);
scaleKernel(length, data, cache.NInv, modulus); scaleKernel(stream, length, data, cache.NInv, modulus);
accelerator.Synchronize();
} }
accelerator.Synchronize(); stream.Synchronize();
} }
public static void Convolve(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus, GpuUInt128 primitiveRoot) public static void Convolve(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
@@ -871,19 +928,20 @@ public static class NttGpuMath
public static void SquareGpu(Span<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) => public static void SquareGpu(Span<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) =>
ConvolveGpu(values, values, modulus, primitiveRoot); ConvolveGpu(values, values, modulus, primitiveRoot);
public static void SquareDevice(Accelerator accelerator, ArrayView<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) public static void SquareDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{ {
var cache = GetSquareCache(accelerator, (int)values.Length, modulus, primitiveRoot); var cache = GetSquareCache(accelerator, stream, (int)values.Length, modulus, primitiveRoot);
ForwardDevice(accelerator, values, (int)values.Length, modulus, cache); ForwardDevice(accelerator, stream, values, (int)values.Length, modulus, cache);
var mul = GetMulKernel(accelerator); var mul = GetMulKernel(accelerator);
mul((int)values.Length, values, values, modulus); mul(stream, (int)values.Length, values, values, modulus);
InverseDevice(accelerator, values, (int)values.Length, modulus, cache); InverseDevice(accelerator, stream, values, (int)values.Length, modulus, cache);
} }
public static void PointwiseMultiply(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus) public static void PointwiseMultiply(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus)
{ {
var gpu = GpuContextPool.Rent(); var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetMulKernel(accelerator); var kernel = GetMulKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared; var pool = ArrayPool<GpuUInt128>.Shared;
var leftArray = pool.Rent(left.Length); var leftArray = pool.Rent(left.Length);
@@ -892,18 +950,19 @@ public static class NttGpuMath
right.CopyTo(rightArray); right.CopyTo(rightArray);
var leftBuffer = accelerator.Allocate1D<GpuUInt128>(left.Length); var leftBuffer = accelerator.Allocate1D<GpuUInt128>(left.Length);
var rightBuffer = accelerator.Allocate1D<GpuUInt128>(right.Length); var rightBuffer = accelerator.Allocate1D<GpuUInt128>(right.Length);
leftBuffer.View.CopyFromCPU(ref leftArray[0], left.Length); leftBuffer.View.CopyFromCPU(stream, ref leftArray[0], left.Length);
rightBuffer.View.CopyFromCPU(ref rightArray[0], right.Length); rightBuffer.View.CopyFromCPU(stream, ref rightArray[0], right.Length);
kernel(left.Length, leftBuffer.View, rightBuffer.View, modulus); kernel(stream, left.Length, leftBuffer.View, rightBuffer.View, modulus);
accelerator.Synchronize(); leftBuffer.View.CopyToCPU(stream, ref leftArray[0], left.Length);
leftBuffer.View.CopyToCPU(ref leftArray[0], left.Length); rightBuffer.View.CopyToCPU(stream, ref rightArray[0], right.Length);
rightBuffer.View.CopyToCPU(ref rightArray[0], right.Length); stream.Synchronize();
leftArray.AsSpan(0, left.Length).CopyTo(left); leftArray.AsSpan(0, left.Length).CopyTo(left);
rightArray.AsSpan(0, right.Length).CopyTo(right); rightArray.AsSpan(0, right.Length).CopyTo(right);
pool.Return(leftArray, clearArray: true); pool.Return(leftArray, clearArray: true);
pool.Return(rightArray, clearArray: true); pool.Return(rightArray, clearArray: true);
rightBuffer.Dispose(); rightBuffer.Dispose();
leftBuffer.Dispose(); leftBuffer.Dispose();
stream.Dispose();
gpu.Dispose(); gpu.Dispose();
} }
PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs
+44 -44
View File
@@ -18,9 +18,9 @@ internal enum GpuPow2ModStatus
internal static partial class PrimeOrderGpuHeuristics internal static partial class PrimeOrderGpuHeuristics
{ {
internal static void PreloadStaticTables(Accelerator accelerator) internal static void PreloadStaticTables(Accelerator accelerator, AcceleratorStream stream)
{ {
_ = GetSmallPrimeDeviceCache(accelerator); _ = GetSmallPrimeDeviceCache(accelerator, stream);
} }
private static readonly ConcurrentDictionary<ulong, byte> OverflowedPrimes = new(); private static readonly ConcurrentDictionary<ulong, byte> OverflowedPrimes = new();
@@ -106,16 +106,16 @@ internal static partial class PrimeOrderGpuHeuristics
s_capability = PrimeOrderGpuCapability.Default; s_capability = PrimeOrderGpuCapability.Default;
} }
private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator) private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator, AcceleratorStream stream)
{ {
return SmallPrimeDeviceCaches.GetOrAdd(accelerator, static acc => return SmallPrimeDeviceCaches.GetOrAdd(accelerator, acc =>
{ {
uint[] primes = PrimesGenerator.SmallPrimes; uint[] primes = PrimesGenerator.SmallPrimes;
ulong[] squares = PrimesGenerator.SmallPrimesPow2; ulong[] squares = PrimesGenerator.SmallPrimesPow2;
var primeBuffer = acc.Allocate1D<uint>(primes.Length); var primeBuffer = acc.Allocate1D<uint>(primes.Length);
primeBuffer.View.CopyFromCPU(primes); primeBuffer.View.CopyFromCPU(stream, primes);
var squareBuffer = acc.Allocate1D<ulong>(squares.Length); var squareBuffer = acc.Allocate1D<ulong>(squares.Length);
squareBuffer.View.CopyFromCPU(squares); squareBuffer.View.CopyFromCPU(stream, squares);
return new SmallPrimeDeviceCache return new SmallPrimeDeviceCache
{ {
Primes = primeBuffer, Primes = primeBuffer,
@@ -203,7 +203,7 @@ internal static partial class PrimeOrderGpuHeuristics
AcceleratorStream stream = lease.Stream; AcceleratorStream stream = lease.Stream;
var kernel = GetPartialFactorKernel(accelerator); var kernel = GetPartialFactorKernel(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator); SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream);
// TODO: We should create / reallocate these buffer only once or if the new required length is bigger than capacity. // TODO: We should create / reallocate these buffer only once or if the new required length is bigger than capacity.
var factorBuffer = accelerator.Allocate1D<ulong>(primeTargets.Length); var factorBuffer = accelerator.Allocate1D<ulong>(primeTargets.Length);
@@ -213,11 +213,11 @@ internal static partial class PrimeOrderGpuHeuristics
var fullyFactoredBuffer = accelerator.Allocate1D<byte>(1); var fullyFactoredBuffer = accelerator.Allocate1D<byte>(1);
// TODO: There is no need to clear these buffers because the kernel will always assign values within the required bounds. // TODO: There is no need to clear these buffers because the kernel will always assign values within the required bounds.
// factorBuffer.MemSetToZero(); // factorBuffer.MemSetToZero(stream);
// exponentBuffer.MemSetToZero(); // exponentBuffer.MemSetToZero(stream);
// countBuffer.MemSetToZero(); // countBuffer.MemSetToZero(stream);
// remainingBuffer.MemSetToZero(); // remainingBuffer.MemSetToZero(stream);
// fullyFactoredBuffer.MemSetToZero(); // fullyFactoredBuffer.MemSetToZero(stream);
kernel( kernel(
stream, stream,
@@ -234,17 +234,18 @@ internal static partial class PrimeOrderGpuHeuristics
remainingBuffer.View, remainingBuffer.View,
fullyFactoredBuffer.View); fullyFactoredBuffer.View);
stream.Synchronize();
countBuffer.View.CopyToCPU(ref factorCount, 1); countBuffer.View.CopyToCPU(stream, ref factorCount, 1);
factorCount = Math.Min(factorCount, primeTargets.Length); factorBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length);
factorBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length); exponentBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length);
exponentBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length); remainingBuffer.View.CopyToCPU(stream, ref remaining, 1);
remainingBuffer.View.CopyToCPU(ref remaining, 1);
byte fullyFactoredFlag = 0; byte fullyFactoredFlag = 0;
fullyFactoredBuffer.View.CopyToCPU(ref fullyFactoredFlag, 1); fullyFactoredBuffer.View.CopyToCPU(stream, ref fullyFactoredFlag, 1);
stream.Synchronize();
fullyFactored = fullyFactoredFlag != 0; fullyFactored = fullyFactoredFlag != 0;
factorCount = Math.Min(factorCount, primeTargets.Length);
// TODO: These buffers shouldn't be disposed but rather reused accross call with the assigned accelerator. // TODO: These buffers shouldn't be disposed but rather reused accross call with the assigned accelerator.
factorBuffer.Dispose(); factorBuffer.Dispose();
@@ -369,7 +370,7 @@ internal static partial class PrimeOrderGpuHeuristics
AcceleratorStream stream = lease.Stream; AcceleratorStream stream = lease.Stream;
var kernel = GetOrderKernel(accelerator); var kernel = GetOrderKernel(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator); SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream);
// TODO: These buffers should be created reallocated once and assigned to an accelerator. Callers should use their own // TODO: These buffers should be created reallocated once and assigned to an accelerator. Callers should use their own
// thread static cache to prevent other threads from using taking accelerators with pre-allocated buffers if they don't // thread static cache to prevent other threads from using taking accelerators with pre-allocated buffers if they don't
@@ -386,16 +387,16 @@ internal static partial class PrimeOrderGpuHeuristics
var statusBuffer = accelerator.Allocate1D<byte>(1); var statusBuffer = accelerator.Allocate1D<byte>(1);
// TODO: Remove the cleaning after the order kernel is modified to always set the result. // TODO: Remove the cleaning after the order kernel is modified to always set the result.
phiFactorBuffer.MemSetToZero(); phiFactorBuffer.MemSetToZero(stream);
phiExponentBuffer.MemSetToZero(); phiExponentBuffer.MemSetToZero(stream);
workFactorBuffer.MemSetToZero(); workFactorBuffer.MemSetToZero(stream);
workExponentBuffer.MemSetToZero(); workExponentBuffer.MemSetToZero(stream);
candidateBuffer.MemSetToZero(); candidateBuffer.MemSetToZero(stream);
stackIndexBuffer.MemSetToZero(); stackIndexBuffer.MemSetToZero(stream);
stackExponentBuffer.MemSetToZero(); stackExponentBuffer.MemSetToZero(stream);
stackProductBuffer.MemSetToZero(); stackProductBuffer.MemSetToZero(stream);
resultBuffer.MemSetToZero(); resultBuffer.MemSetToZero(stream);
statusBuffer.MemSetToZero(); statusBuffer.MemSetToZero(stream);
uint limit = config.SmallFactorLimit == 0 ? uint.MaxValue : config.SmallFactorLimit; uint limit = config.SmallFactorLimit == 0 ? uint.MaxValue : config.SmallFactorLimit;
byte hasPrevious = previousOrder.HasValue ? (byte)1 : (byte)0; byte hasPrevious = previousOrder.HasValue ? (byte)1 : (byte)0;
@@ -425,10 +426,11 @@ internal static partial class PrimeOrderGpuHeuristics
cache.Count, cache.Count,
buffers); buffers);
stream.Synchronize();
byte status = 0; byte status = 0;
statusBuffer.View.CopyToCPU(ref status, 1); statusBuffer.View.CopyToCPU(stream, ref status, 1);
resultBuffer.View.CopyToCPU(stream, ref order, 1);
stream.Synchronize();
PrimeOrderKernelStatus kernelStatus = (PrimeOrderKernelStatus)status; PrimeOrderKernelStatus kernelStatus = (PrimeOrderKernelStatus)status;
if (kernelStatus == PrimeOrderKernelStatus.Fallback) if (kernelStatus == PrimeOrderKernelStatus.Fallback)
@@ -445,8 +447,6 @@ internal static partial class PrimeOrderGpuHeuristics
throw new InvalidOperationException("GPU Pollard Rho stack overflow; increase HeuristicStackCapacity."); throw new InvalidOperationException("GPU Pollard Rho stack overflow; increase HeuristicStackCapacity.");
} }
resultBuffer.View.CopyToCPU(ref order, 1);
if (kernelStatus == PrimeOrderKernelStatus.FactoringFailure) if (kernelStatus == PrimeOrderKernelStatus.FactoringFailure)
{ {
order = 0UL; order = 0UL;
@@ -536,15 +536,19 @@ internal static partial class PrimeOrderGpuHeuristics
var exponentBuffer = accelerator.Allocate1D<ulong>(exponents.Length); var exponentBuffer = accelerator.Allocate1D<ulong>(exponents.Length);
var remainderBuffer = accelerator.Allocate1D<ulong>(exponents.Length); var remainderBuffer = accelerator.Allocate1D<ulong>(exponents.Length);
exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponents), exponents.Length); exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponents), exponents.Length);
// Pow2Mod kernel always assigns the required output elements so we don't need to worry about clearing these. // Pow2Mod kernel always assigns the required output elements so we don't need to worry about clearing these.
// remainderBuffer.MemSetToZero(); // remainderBuffer.MemSetToZero(stream);
try try
{ {
kernel(stream, exponents.Length, exponentBuffer.View, divisorData, remainderBuffer.View); kernel(stream, exponents.Length, exponentBuffer.View, divisorData, remainderBuffer.View);
remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(results), exponents.Length);
stream.Synchronize(); stream.Synchronize();
exponentBuffer.Dispose();
remainderBuffer.Dispose();
lease.Dispose();
} }
catch (Exception) catch (Exception)
{ {
@@ -555,10 +559,6 @@ internal static partial class PrimeOrderGpuHeuristics
throw; throw;
} }
remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(results), exponents.Length);
exponentBuffer.Dispose();
remainderBuffer.Dispose();
lease.Dispose();
return true; return true;
} }
@@ -603,20 +603,20 @@ internal static partial class PrimeOrderGpuHeuristics
exponentSpan[i] = (GpuUInt128)exponents[i]; exponentSpan[i] = (GpuUInt128)exponents[i];
} }
exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponentSpan), length); exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponentSpan), length);
// Pow2ModWide kernel always assigns the required output elements so we don't need to worry about clearing these. // Pow2ModWide kernel always assigns the required output elements so we don't need to worry about clearing these.
// remainderBuffer.MemSetToZero(); // remainderBuffer.MemSetToZero(stream);
GpuUInt128 modulus = (GpuUInt128)prime; GpuUInt128 modulus = (GpuUInt128)prime;
kernel(stream, length, exponentBuffer.View, modulus, remainderBuffer.View); kernel(stream, length, exponentBuffer.View, modulus, remainderBuffer.View);
stream.Synchronize();
Span<GpuUInt128> resultSpan = length <= WideStackThreshold Span<GpuUInt128> resultSpan = length <= WideStackThreshold
? stackalloc GpuUInt128[length] ? stackalloc GpuUInt128[length]
: new Span<GpuUInt128>(rentedResults = ArrayPool<GpuUInt128>.Shared.Rent(length), 0, length); : new Span<GpuUInt128>(rentedResults = ArrayPool<GpuUInt128>.Shared.Rent(length), 0, length);
remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(resultSpan), length); remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length);
stream.Synchronize();
for (int i = 0; i < length; i++) for (int i = 0; i < length; i++)
{ {
PerfectNumbers.Core/Gpu/SharedGpuContext.cs
+1
View File
@@ -7,6 +7,7 @@ namespace PerfectNumbers.Core.Gpu;
public static class SharedGpuContext public static class SharedGpuContext
{ {
// TODO: Remove Lazy<T> in favor for better performance with nullable fields.
private static readonly Lazy<Context> s_context = new(() => ILGPU.Context.CreateDefault(), LazyThreadSafetyMode.ExecutionAndPublication); private static readonly Lazy<Context> s_context = new(() => ILGPU.Context.CreateDefault(), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy<Device> s_device = new(() => s_context.Value.GetPreferredDevice(false), LazyThreadSafetyMode.ExecutionAndPublication); private static readonly Lazy<Device> s_device = new(() => s_context.Value.GetPreferredDevice(false), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy<Accelerator> s_accelerator = new(() => s_device.Value.CreateAccelerator(s_context.Value), LazyThreadSafetyMode.ExecutionAndPublication); private static readonly Lazy<Accelerator> s_accelerator = new(() => s_device.Value.CreateAccelerator(s_context.Value), LazyThreadSafetyMode.ExecutionAndPublication);
PerfectNumbers.Core/HeuristicCombinedPrimeTester.cs
+3 -2
View File
@@ -237,17 +237,18 @@ public sealed class HeuristicCombinedPrimeTester
int divisorLength = GetCombinedDivisors(nMod10).Length; int divisorLength = GetCombinedDivisors(nMod10).Length;
int compositeFlag = 0; int compositeFlag = 0;
flagView1D.CopyFromCPU(ref compositeFlag, 1); flagView1D.CopyFromCPU(stream, ref compositeFlag, 1);
kernel( kernel(
stream,
divisorLength, divisorLength,
flagView, flagView,
n, n,
maxDivisorSquare, maxDivisorSquare,
HeuristicGpuDivisorTableKind.Combined, HeuristicGpuDivisorTableKind.Combined,
gpu.HeuristicGpuTables); gpu.HeuristicGpuTables);
flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
stream.Synchronize(); stream.Synchronize();
flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
compositeDetected = compositeFlag != 0; compositeDetected = compositeFlag != 0;
gpu.Dispose(); gpu.Dispose();
PerfectNumbers.Core/HeuristicPrimeTester.cs
+8 -5
View File
@@ -253,17 +253,19 @@ public sealed class HeuristicPrimeTester
int groupALength = HeuristicPrimeSieves.GroupADivisors.Length; int groupALength = HeuristicPrimeSieves.GroupADivisors.Length;
flagView1D.CopyFromCPU(ref compositeFlag, 1); flagView1D.CopyFromCPU(stream, ref compositeFlag, 1);
kernel( kernel(
stream,
groupALength, groupALength,
flagView, flagView,
n, n,
maxDivisorSquare, maxDivisorSquare,
HeuristicGpuDivisorTableKind.GroupA, HeuristicGpuDivisorTableKind.GroupA,
gpu.HeuristicGpuTables); gpu.HeuristicGpuTables);
stream.Synchronize();
flagView1D.CopyToCPU(stream, ref compositeFlag, 1); flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
stream.Synchronize();
compositeDetected = compositeFlag != 0; compositeDetected = compositeFlag != 0;
if (!compositeDetected) if (!compositeDetected)
@@ -288,16 +290,17 @@ public sealed class HeuristicPrimeTester
int divisorLength = GetGroupBDivisors(ending).Length; int divisorLength = GetGroupBDivisors(ending).Length;
compositeFlag = 0; compositeFlag = 0;
flagView1D.CopyFromCPU(ref compositeFlag, 1); flagView1D.CopyFromCPU(stream, ref compositeFlag, 1);
kernel( kernel(
stream,
divisorLength, divisorLength,
flagView, flagView,
n, n,
maxDivisorSquare, maxDivisorSquare,
tableKind, tableKind,
gpu.HeuristicGpuTables); gpu.HeuristicGpuTables);
accelerator.Synchronize(); flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
flagView1D.CopyToCPU(ref compositeFlag, 1); stream.Synchronize();
compositeDetected = compositeFlag != 0; compositeDetected = compositeFlag != 0;
} }
} }
PerfectNumbers.Core/MersenneDivisorCycles.cs
+7 -4
View File
@@ -7,6 +7,7 @@ using System.Text;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using PerfectNumbers.Core.Gpu; using PerfectNumbers.Core.Gpu;
using ILGPU; using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime; using ILGPU.Runtime;
namespace PerfectNumbers.Core; namespace PerfectNumbers.Core;
@@ -806,10 +807,12 @@ public class MersenneDivisorCycles
var lease = GpuKernelPool.GetKernel(); var lease = GpuKernelPool.GetKernel();
var accelerator = lease.Accelerator; var accelerator = lease.Accelerator;
var stream = lease.Stream; var stream = lease.Stream;
var kernel = accelerator.LoadAutoGroupedStreamKernel< var loaded = accelerator.LoadAutoGroupedStreamKernel<
Index1D, Index1D,
ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>,
ArrayView1D<ulong, Stride1D.Dense>>(DivisorCycleKernels.GpuDivisorCycleKernel); ArrayView1D<ulong, Stride1D.Dense>>(DivisorCycleKernels.GpuDivisorCycleKernel);
var kernel = KernelUtil.GetKernel(loaded);
var launcher = kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>>();
using Stream outputStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, BufferSize10M, useAsync: true); using Stream outputStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, BufferSize10M, useAsync: true);
@@ -860,14 +863,15 @@ public class MersenneDivisorCycles
var bufferDiv = accelerator.Allocate1D(validDivisors); var bufferDiv = accelerator.Allocate1D(validDivisors);
var bufferCycle = accelerator.Allocate1D<ulong>(idx); var bufferCycle = accelerator.Allocate1D<ulong>(idx);
kernel( launcher(
stream,
idx, idx,
bufferDiv.View, bufferDiv.View,
bufferCycle.View); bufferCycle.View);
bufferCycle.View.CopyToCPU(stream, ref outCycles[0], idx);
stream.Synchronize(); stream.Synchronize();
bufferCycle.View.CopyToCPU(stream, ref outCycles[0], idx);
bufferCycle.Dispose(); bufferCycle.Dispose();
bufferDiv.Dispose(); bufferDiv.Dispose();
@@ -893,7 +897,6 @@ public class MersenneDivisorCycles
pool.Return(divisors, clearArray: false); pool.Return(divisors, clearArray: false);
pool.Return(outCycles, clearArray: false); pool.Return(outCycles, clearArray: false);
stream.Dispose();
lease.Dispose(); lease.Dispose();
} }
PerfectNumbers.Core/MersenneNumberTester.cs
+2 -1
View File
@@ -168,11 +168,12 @@ public sealed class MersenneNumberTester(
tmp[i] = (PerfectNumbers.Core.Gpu.GpuUInt128)qs[offset + i]; tmp[i] = (PerfectNumbers.Core.Gpu.GpuUInt128)qs[offset + i];
} }
// Copy only the populated elements to the device buffer to avoid overrun on pooled arrays. // Copy only the populated elements to the device buffer to avoid overrun on pooled arrays.
qBuffer.View.CopyFromCPU(ref tmp[0], count); qBuffer.View.CopyFromCPU(stream, ref tmp[0], count);
System.Buffers.ArrayPool<PerfectNumbers.Core.Gpu.GpuUInt128>.Shared.Return(tmp); System.Buffers.ArrayPool<PerfectNumbers.Core.Gpu.GpuUInt128>.Shared.Return(tmp);
orderKernel(stream, count, exponent, divMul, qBuffer.View, orderBuffer.View); orderKernel(stream, count, exponent, divMul, qBuffer.View, orderBuffer.View);
stream.Synchronize(); stream.Synchronize();
// TODO: Change this to .CopyToCPU and use pre-allocated buffer with fixed, max capacity, using stream. Synchronize on the stream after copy.
ulong[] orders = orderBuffer.GetAsArray1D(); ulong[] orders = orderBuffer.GetAsArray1D();
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
{ {
PerfectNumbers.Core/PrimeOrderCalculator.Gpu.cs
+5 -3
View File
@@ -26,7 +26,7 @@ internal static partial class PrimeOrderCalculator
ScratchBuffer scratch = GpuScratchBufferPool.Rent(accelerator, GpuSmallPrimeFactorSlots, 0); ScratchBuffer scratch = GpuScratchBufferPool.Rent(accelerator, GpuSmallPrimeFactorSlots, 0);
SmallPrimeFactorTables tables = GpuKernelPool.EnsureSmallPrimeFactorTables(kernels, accelerator); SmallPrimeFactorTables tables = GpuKernelPool.EnsureSmallPrimeFactorTables(kernels, accelerator, stream);
if (kernels.SmallPrimeFactorsPrimes is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsPrimes)} is null"); if (kernels.SmallPrimeFactorsPrimes is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsPrimes)} is null");
if (kernels.SmallPrimeFactorsSquares is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsSquares)} is null"); if (kernels.SmallPrimeFactorsSquares is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsSquares)} is null");
@@ -52,10 +52,11 @@ internal static partial class PrimeOrderCalculator
smallPrimeFactorCountSlotView, smallPrimeFactorCountSlotView,
smallPrimeFactorRemainingSlotView); smallPrimeFactorRemainingSlotView);
stream.Synchronize();
factorCount = 0; factorCount = 0;
smallPrimeFactorCountSlotView.CopyToCPU(stream, ref factorCount, 1); smallPrimeFactorCountSlotView.CopyToCPU(stream, ref factorCount, 1);
stream.Synchronize();
factorCount = Math.Min(factorCount, GpuSmallPrimeFactorSlots); factorCount = Math.Min(factorCount, GpuSmallPrimeFactorSlots);
if (factorCount > 0) if (factorCount > 0)
@@ -66,6 +67,7 @@ internal static partial class PrimeOrderCalculator
} }
smallPrimeFactorRemainingSlotView.CopyToCPU(stream, ref remaining, 1); smallPrimeFactorRemainingSlotView.CopyToCPU(stream, ref remaining, 1);
stream.Synchronize();
for (int i = 0; i < factorCount; i++) for (int i = 0; i < factorCount; i++)
{ {
@@ -121,11 +123,11 @@ internal static partial class PrimeOrderCalculator
divisorData, divisorData,
scratch.SpecialMaxCandidates.View, scratch.SpecialMaxCandidates.View,
specialMaxResultView); specialMaxResultView);
stream.Synchronize();
Span<ulong> result = stackalloc ulong[1]; Span<ulong> result = stackalloc ulong[1];
// specialMaxResultView.CopyToCPU(stream, ref result, 1); // specialMaxResultView.CopyToCPU(stream, ref result, 1);
specialMaxResultView.CopyToCPU(stream, in result); specialMaxResultView.CopyToCPU(stream, in result);
stream.Synchronize();
// stream.Synchronize(); // stream.Synchronize();
// return result != 0; // return result != 0;
PerfectNumbers.Core/PrimeTester.cs
+80 -85
View File
@@ -111,6 +111,7 @@ public sealed class PrimeTester
inputView.CopyFromCPU(stream, ref value, 1); inputView.CopyFromCPU(stream, ref value, 1);
kernel( kernel(
stream,
1, 1,
inputView, inputView,
gpu.DevicePrimesDefault.View, gpu.DevicePrimesDefault.View,
@@ -124,8 +125,8 @@ public sealed class PrimeTester
gpu.DevicePrimesPow2LastThree.View, gpu.DevicePrimesPow2LastThree.View,
gpu.DevicePrimesPow2LastNine.View, gpu.DevicePrimesPow2LastNine.View,
outputView); outputView);
stream.Synchronize();
outputView.CopyToCPU(stream, ref flag, 1); outputView.CopyToCPU(stream, ref flag, 1);
stream.Synchronize();
gpu.Dispose(); gpu.Dispose();
GpuPrimeWorkLimiter.Release(); GpuPrimeWorkLimiter.Release();
@@ -182,9 +183,10 @@ public sealed class PrimeTester
var valueSlice = values.Slice(pos, count); var valueSlice = values.Slice(pos, count);
ref ulong valueRef = ref MemoryMarshal.GetReference(valueSlice); ref ulong valueRef = ref MemoryMarshal.GetReference(valueSlice);
inputView.CopyFromCPU(ref valueRef, count); inputView.CopyFromCPU(stream, ref valueRef, count);
kernel( kernel(
stream,
count, count,
inputView, inputView,
gpu.DevicePrimesDefault.View, gpu.DevicePrimesDefault.View,
@@ -198,7 +200,6 @@ public sealed class PrimeTester
gpu.DevicePrimesPow2LastThree.View, gpu.DevicePrimesPow2LastThree.View,
gpu.DevicePrimesPow2LastNine.View, gpu.DevicePrimesPow2LastNine.View,
outputView); outputView);
stream.Synchronize();
var resultSlice = results.Slice(pos, count); var resultSlice = results.Slice(pos, count);
ref byte resultRef = ref MemoryMarshal.GetReference(resultSlice); ref byte resultRef = ref MemoryMarshal.GetReference(resultSlice);
@@ -207,6 +208,7 @@ public sealed class PrimeTester
pos += count; pos += count;
} }
stream.Synchronize();
gpu.Dispose(); gpu.Dispose();
GpuPrimeWorkLimiter.Release(); GpuPrimeWorkLimiter.Release();
} }
@@ -219,13 +221,13 @@ public sealed class PrimeTester
private static readonly object SharedTablesLock = new(); private static readonly object SharedTablesLock = new();
private static readonly Dictionary<Accelerator, SharedHeuristicGpuTables> SharedTables = new(AcceleratorReferenceComparer.Instance); private static readonly Dictionary<Accelerator, SharedHeuristicGpuTables> SharedTables = new(AcceleratorReferenceComparer.Instance);
private static SharedHeuristicGpuTables RentSharedTables(Accelerator accelerator) private static SharedHeuristicGpuTables RentSharedTables(Accelerator accelerator, AcceleratorStream stream)
{ {
lock (SharedTablesLock) lock (SharedTablesLock)
{ {
if (!SharedTables.TryGetValue(accelerator, out var tables)) if (!SharedTables.TryGetValue(accelerator, out var tables))
{ {
tables = new SharedHeuristicGpuTables(accelerator); tables = new SharedHeuristicGpuTables(accelerator, stream);
SharedTables.Add(accelerator, tables); SharedTables.Add(accelerator, tables);
} }
@@ -247,9 +249,9 @@ public sealed class PrimeTester
} }
} }
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator) internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{ {
var tables = RentSharedTables(accelerator); var tables = RentSharedTables(accelerator, stream);
var heuristicTables = tables.CreateHeuristicDivisorTables(); var heuristicTables = tables.CreateHeuristicDivisorTables();
HeuristicGpuDivisorTables.InitializeShared(in heuristicTables); HeuristicGpuDivisorTables.InitializeShared(in heuristicTables);
ReleaseSharedTables(tables); ReleaseSharedTables(tables);
@@ -270,15 +272,19 @@ public sealed class PrimeTester
} }
Accelerator accelerator = SharedGpuContext.Accelerator; Accelerator accelerator = SharedGpuContext.Accelerator;
KernelContainer kernels = GpuKernelPool.GetKernels(accelerator); AcceleratorStream stream = accelerator.CreateStream();
GpuStaticTableInitializer.EnsureStaticTables(kernels, accelerator); KernelContainer kernels = GpuKernelPool.GetKernels(accelerator, stream);
GpuStaticTableInitializer.EnsureStaticTables(kernels, accelerator, stream);
int toCreate = target - WarmedLeaseCount; int toCreate = target - WarmedLeaseCount;
for (int i = 0; i < toCreate; i++) for (int i = 0; i < toCreate; i++)
{ {
Pool.Enqueue(new PrimeTesterGpuContextLease(minBufferCapacity)); Pool.Enqueue(new PrimeTesterGpuContextLease(minBufferCapacity, stream));
} }
stream.Synchronize();
stream.Dispose();
WarmedLeaseCount = target; WarmedLeaseCount = target;
} }
} }
@@ -294,14 +300,12 @@ public sealed class PrimeTester
lease.EnsureCapacity(minBufferCapacity); lease.EnsureCapacity(minBufferCapacity);
} }
lease.ResetReturnFlag();
return lease; return lease;
} }
internal static void Return(PrimeTesterGpuContextLease lease) internal static void Return(PrimeTesterGpuContextLease lease)
{ {
lease.Accelerator.Synchronize();
Pool.Enqueue(lease); Pool.Enqueue(lease);
} }
@@ -373,34 +377,22 @@ public sealed class PrimeTester
} }
} }
internal sealed class PrimeTesterGpuContextLease : IDisposable internal sealed class PrimeTesterGpuContextLease
{ {
private bool _returned;
private readonly SharedHeuristicGpuTables _sharedTables; private readonly SharedHeuristicGpuTables _sharedTables;
private readonly HeuristicGpuDivisorTables _heuristicDivisorTables; private readonly HeuristicGpuDivisorTables _heuristicDivisorTables;
internal readonly Context AcceleratorContext; internal readonly Context AcceleratorContext;
public readonly Accelerator Accelerator; public readonly Accelerator Accelerator;
private AcceleratorStream? _stream; private AcceleratorStream? _stream;
public AcceleratorStream Stream public AcceleratorStream Stream
{ {
get [MethodImpl(MethodImplOptions.AggressiveInlining)]
{ get => _stream ?? Accelerator.CreateStream();
if (_stream is { } stream)
{
return stream;
} }
stream = Accelerator.CreateStream(); public static readonly Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>> Kernel = KernelUtil.GetKernel(SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel)).CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>>();
_stream = stream;
return stream;
}
}
public static readonly Action<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>> Kernel = SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel); public static readonly Action<AcceleratorStream, Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables> HeuristicTrialDivisionKernel = KernelUtil.GetKernel(SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel)).CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>>();
public static readonly Action<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables> HeuristicTrialDivisionKernel = SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel);
public MemoryBuffer1D<ulong, Stride1D.Dense> Input = null!; public MemoryBuffer1D<ulong, Stride1D.Dense> Input = null!;
public MemoryBuffer1D<byte, Stride1D.Dense> Output = null!; public MemoryBuffer1D<byte, Stride1D.Dense> Output = null!;
@@ -440,17 +432,34 @@ public sealed class PrimeTester
{ {
AcceleratorContext = SharedGpuContext.Context; AcceleratorContext = SharedGpuContext.Context;
Accelerator = SharedGpuContext.Accelerator; Accelerator = SharedGpuContext.Accelerator;
// GpuStaticTableInitializer.EnsureStaticTables(Accelerator); AcceleratorStream stream = SharedGpuContext.Accelerator.CreateStream();
// Kernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel); var kernels = GpuKernelPool.GetKernels(Accelerator, stream);
// HeuristicTrialDivisionKernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel); GpuStaticTableInitializer.EnsureStaticTables(kernels, Accelerator, stream);
_sharedTables = RentSharedTables(Accelerator); _sharedTables = RentSharedTables(Accelerator, stream);
_heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables();
HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables);
EnsureCapacity(minBufferCapacity);
stream.Synchronize();
stream.Dispose();
}
internal PrimeTesterGpuContextLease(int minBufferCapacity, AcceleratorStream stream)
{
AcceleratorContext = SharedGpuContext.Context;
Accelerator = SharedGpuContext.Accelerator;
// GpuStaticTableInitializer.EnsureStaticTables(Accelerator);
// Kernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel);
// HeuristicTrialDivisionKernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel);
_sharedTables = RentSharedTables(Accelerator, stream);
_heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables(); _heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables();
HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables); HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables);
EnsureCapacity(minBufferCapacity); EnsureCapacity(minBufferCapacity);
_returned = false;
} }
public void EnsureCapacity(int minCapacity) public void EnsureCapacity(int minCapacity)
@@ -470,25 +479,8 @@ public sealed class PrimeTester
BufferCapacity = minCapacity; BufferCapacity = minCapacity;
} }
internal void ResetReturnFlag()
{
_returned = false;
}
public void Dispose() public void Dispose()
{ {
if (_returned)
{
return;
}
_returned = true;
if (_stream is { } stream)
{
stream.Dispose();
_stream = null;
}
Return(this); Return(this);
} }
@@ -497,6 +489,8 @@ public sealed class PrimeTester
Input?.Dispose(); Input?.Dispose();
Output?.Dispose(); Output?.Dispose();
HeuristicFlag?.Dispose(); HeuristicFlag?.Dispose();
_stream?.Dispose();
_stream = null;
ReleaseSharedTables(_sharedTables); ReleaseSharedTables(_sharedTables);
// These resources are shared between GPU leases // These resources are shared between GPU leases
// Accelerator.Dispose(); // Accelerator.Dispose();
@@ -523,86 +517,86 @@ public sealed class PrimeTester
{ {
private int _referenceCount; private int _referenceCount;
internal SharedHeuristicGpuTables(Accelerator accelerator) internal SharedHeuristicGpuTables(Accelerator accelerator, AcceleratorStream stream)
{ {
var primesDefault = DivisorGenerator.SmallPrimes; var primesDefault = DivisorGenerator.SmallPrimes;
DevicePrimesDefault = accelerator.Allocate1D<uint>(primesDefault.Length); DevicePrimesDefault = accelerator.Allocate1D<uint>(primesDefault.Length);
DevicePrimesDefault.View.CopyFromCPU(primesDefault); DevicePrimesDefault.View.CopyFromCPU(stream, primesDefault);
var primesLastOne = DivisorGenerator.SmallPrimesLastOne; var primesLastOne = DivisorGenerator.SmallPrimesLastOne;
DevicePrimesLastOne = accelerator.Allocate1D<uint>(primesLastOne.Length); DevicePrimesLastOne = accelerator.Allocate1D<uint>(primesLastOne.Length);
DevicePrimesLastOne.View.CopyFromCPU(primesLastOne); DevicePrimesLastOne.View.CopyFromCPU(stream, primesLastOne);
var primesLastSeven = DivisorGenerator.SmallPrimesLastSeven; var primesLastSeven = DivisorGenerator.SmallPrimesLastSeven;
DevicePrimesLastSeven = accelerator.Allocate1D<uint>(primesLastSeven.Length); DevicePrimesLastSeven = accelerator.Allocate1D<uint>(primesLastSeven.Length);
DevicePrimesLastSeven.View.CopyFromCPU(primesLastSeven); DevicePrimesLastSeven.View.CopyFromCPU(stream, primesLastSeven);
var primesLastThree = DivisorGenerator.SmallPrimesLastThree; var primesLastThree = DivisorGenerator.SmallPrimesLastThree;
DevicePrimesLastThree = accelerator.Allocate1D<uint>(primesLastThree.Length); DevicePrimesLastThree = accelerator.Allocate1D<uint>(primesLastThree.Length);
DevicePrimesLastThree.View.CopyFromCPU(primesLastThree); DevicePrimesLastThree.View.CopyFromCPU(stream, primesLastThree);
var primesLastNine = DivisorGenerator.SmallPrimesLastNine; var primesLastNine = DivisorGenerator.SmallPrimesLastNine;
DevicePrimesLastNine = accelerator.Allocate1D<uint>(primesLastNine.Length); DevicePrimesLastNine = accelerator.Allocate1D<uint>(primesLastNine.Length);
DevicePrimesLastNine.View.CopyFromCPU(primesLastNine); DevicePrimesLastNine.View.CopyFromCPU(stream, primesLastNine);
var primesPow2Default = DivisorGenerator.SmallPrimesPow2; var primesPow2Default = DivisorGenerator.SmallPrimesPow2;
DevicePrimesPow2Default = accelerator.Allocate1D<ulong>(primesPow2Default.Length); DevicePrimesPow2Default = accelerator.Allocate1D<ulong>(primesPow2Default.Length);
DevicePrimesPow2Default.View.CopyFromCPU(primesPow2Default); DevicePrimesPow2Default.View.CopyFromCPU(stream, primesPow2Default);
var primesPow2LastOne = DivisorGenerator.SmallPrimesPow2LastOne; var primesPow2LastOne = DivisorGenerator.SmallPrimesPow2LastOne;
DevicePrimesPow2LastOne = accelerator.Allocate1D<ulong>(primesPow2LastOne.Length); DevicePrimesPow2LastOne = accelerator.Allocate1D<ulong>(primesPow2LastOne.Length);
DevicePrimesPow2LastOne.View.CopyFromCPU(primesPow2LastOne); DevicePrimesPow2LastOne.View.CopyFromCPU(stream, primesPow2LastOne);
var primesPow2LastSeven = DivisorGenerator.SmallPrimesPow2LastSeven; var primesPow2LastSeven = DivisorGenerator.SmallPrimesPow2LastSeven;
DevicePrimesPow2LastSeven = accelerator.Allocate1D<ulong>(primesPow2LastSeven.Length); DevicePrimesPow2LastSeven = accelerator.Allocate1D<ulong>(primesPow2LastSeven.Length);
DevicePrimesPow2LastSeven.View.CopyFromCPU(primesPow2LastSeven); DevicePrimesPow2LastSeven.View.CopyFromCPU(stream, primesPow2LastSeven);
var primesPow2LastThree = DivisorGenerator.SmallPrimesPow2LastThree; var primesPow2LastThree = DivisorGenerator.SmallPrimesPow2LastThree;
DevicePrimesPow2LastThree = accelerator.Allocate1D<ulong>(primesPow2LastThree.Length); DevicePrimesPow2LastThree = accelerator.Allocate1D<ulong>(primesPow2LastThree.Length);
DevicePrimesPow2LastThree.View.CopyFromCPU(primesPow2LastThree); DevicePrimesPow2LastThree.View.CopyFromCPU(stream, primesPow2LastThree);
var primesPow2LastNine = DivisorGenerator.SmallPrimesPow2LastNine; var primesPow2LastNine = DivisorGenerator.SmallPrimesPow2LastNine;
DevicePrimesPow2LastNine = accelerator.Allocate1D<ulong>(primesPow2LastNine.Length); DevicePrimesPow2LastNine = accelerator.Allocate1D<ulong>(primesPow2LastNine.Length);
DevicePrimesPow2LastNine.View.CopyFromCPU(primesPow2LastNine); DevicePrimesPow2LastNine.View.CopyFromCPU(stream, primesPow2LastNine);
HeuristicPrimeSieves.EnsureInitialized(); HeuristicPrimeSieves.EnsureInitialized();
HeuristicCombinedPrimeTester.EnsureInitialized(); HeuristicCombinedPrimeTester.EnsureInitialized();
var heuristicGroupA = HeuristicPrimeSieves.GroupADivisors; var heuristicGroupA = HeuristicPrimeSieves.GroupADivisors;
HeuristicGroupADivisors = CopySpanToDevice(accelerator, heuristicGroupA); HeuristicGroupADivisors = CopySpanToDevice(accelerator, stream, heuristicGroupA);
var heuristicGroupASquares = HeuristicPrimeSieves.GroupADivisorSquares; var heuristicGroupASquares = HeuristicPrimeSieves.GroupADivisorSquares;
HeuristicGroupADivisorSquares = CopySpanToDevice(accelerator, heuristicGroupASquares); HeuristicGroupADivisorSquares = CopySpanToDevice(accelerator, stream, heuristicGroupASquares);
HeuristicGroupBDivisorsEnding1 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastOneWithoutLastThree); HeuristicGroupBDivisorsEnding1 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastOneWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding1 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastOneWithoutLastThree); HeuristicGroupBDivisorSquaresEnding1 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastOneWithoutLastThree);
HeuristicGroupBDivisorsEnding7 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastSevenWithoutLastThree); HeuristicGroupBDivisorsEnding7 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastSevenWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding7 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastSevenWithoutLastThree); HeuristicGroupBDivisorSquaresEnding7 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastSevenWithoutLastThree);
HeuristicGroupBDivisorsEnding9 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastNineWithoutLastThree); HeuristicGroupBDivisorsEnding9 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastNineWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding9 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastNineWithoutLastThree); HeuristicGroupBDivisorSquaresEnding9 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastNineWithoutLastThree);
var combinedEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1Span; var combinedEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1Span;
HeuristicCombinedDivisorsEnding1 = CopySpanToDevice(accelerator, combinedEnding1); HeuristicCombinedDivisorsEnding1 = CopySpanToDevice(accelerator, stream, combinedEnding1);
var combinedSquaresEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1SquaresSpan; var combinedSquaresEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1SquaresSpan;
HeuristicCombinedDivisorSquaresEnding1 = CopySpanToDevice(accelerator, combinedSquaresEnding1); HeuristicCombinedDivisorSquaresEnding1 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding1);
var combinedEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3Span; var combinedEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3Span;
HeuristicCombinedDivisorsEnding3 = CopySpanToDevice(accelerator, combinedEnding3); HeuristicCombinedDivisorsEnding3 = CopySpanToDevice(accelerator, stream, combinedEnding3);
var combinedSquaresEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3SquaresSpan; var combinedSquaresEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3SquaresSpan;
HeuristicCombinedDivisorSquaresEnding3 = CopySpanToDevice(accelerator, combinedSquaresEnding3); HeuristicCombinedDivisorSquaresEnding3 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding3);
var combinedEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7Span; var combinedEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7Span;
HeuristicCombinedDivisorsEnding7 = CopySpanToDevice(accelerator, combinedEnding7); HeuristicCombinedDivisorsEnding7 = CopySpanToDevice(accelerator, stream, combinedEnding7);
var combinedSquaresEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7SquaresSpan; var combinedSquaresEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7SquaresSpan;
HeuristicCombinedDivisorSquaresEnding7 = CopySpanToDevice(accelerator, combinedSquaresEnding7); HeuristicCombinedDivisorSquaresEnding7 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding7);
var combinedEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9Span; var combinedEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9Span;
HeuristicCombinedDivisorsEnding9 = CopySpanToDevice(accelerator, combinedEnding9); HeuristicCombinedDivisorsEnding9 = CopySpanToDevice(accelerator, stream, combinedEnding9);
var combinedSquaresEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9SquaresSpan; var combinedSquaresEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9SquaresSpan;
HeuristicCombinedDivisorSquaresEnding9 = CopySpanToDevice(accelerator, combinedSquaresEnding9); HeuristicCombinedDivisorSquaresEnding9 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding9);
} }
internal MemoryBuffer1D<uint, Stride1D.Dense> DevicePrimesDefault { get; } internal MemoryBuffer1D<uint, Stride1D.Dense> DevicePrimesDefault { get; }
@@ -666,19 +660,19 @@ public sealed class PrimeTester
} }
} }
private static MemoryBuffer1D<ulong, Stride1D.Dense> CopySpanToDevice(Accelerator accelerator, ReadOnlySpan<ulong> span) private static MemoryBuffer1D<ulong, Stride1D.Dense> CopySpanToDevice(Accelerator accelerator, AcceleratorStream stream, ReadOnlySpan<ulong> span)
{ {
var buffer = accelerator.Allocate1D<ulong>(span.Length); var buffer = accelerator.Allocate1D<ulong>(span.Length);
if (!span.IsEmpty) if (!span.IsEmpty)
{ {
ref ulong sourceRef = ref MemoryMarshal.GetReference(span); ref ulong sourceRef = ref MemoryMarshal.GetReference(span);
buffer.View.CopyFromCPU(ref sourceRef, span.Length); buffer.View.CopyFromCPU(stream, ref sourceRef, span.Length);
} }
return buffer; return buffer;
} }
private static MemoryBuffer1D<ulong, Stride1D.Dense> CopyUintSpanToDevice(Accelerator accelerator, ReadOnlySpan<uint> span) private static MemoryBuffer1D<ulong, Stride1D.Dense> CopyUintSpanToDevice(Accelerator accelerator, AcceleratorStream stream, ReadOnlySpan<uint> span)
{ {
var buffer = accelerator.Allocate1D<ulong>(span.Length); var buffer = accelerator.Allocate1D<ulong>(span.Length);
if (!span.IsEmpty) if (!span.IsEmpty)
@@ -689,7 +683,7 @@ public sealed class PrimeTester
converted[i] = span[i]; converted[i] = span[i];
} }
buffer.View.CopyFromCPU(converted); buffer.View.CopyFromCPU(stream, converted);
} }
return buffer; return buffer;
@@ -760,7 +754,8 @@ public sealed class PrimeTester
var gpu = PrimeTesterGpuContextPool.Rent(); var gpu = PrimeTesterGpuContextPool.Rent();
var accelerator = gpu.Accelerator; var accelerator = gpu.Accelerator;
var stream = gpu.Stream; var stream = gpu.Stream;
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SharesFactorKernel);
var kernel = KernelUtil.GetKernel(accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SharesFactorKernel)).CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<byte>>>();
int length = values.Length; int length = values.Length;
var inputBuffer = accelerator.Allocate1D<ulong>(length); var inputBuffer = accelerator.Allocate1D<ulong>(length);
@@ -770,9 +765,9 @@ public sealed class PrimeTester
ulong[] temp = pool.Rent(length); ulong[] temp = pool.Rent(length);
values.CopyTo(temp); values.CopyTo(temp);
inputBuffer.View.CopyFromCPU(stream, ref temp[0], length); inputBuffer.View.CopyFromCPU(stream, ref temp[0], length);
kernel(length, inputBuffer.View, resultBuffer.View); kernel(stream, length, inputBuffer.View, resultBuffer.View);
stream.Synchronize();
resultBuffer.View.CopyToCPU(stream, ref results[0], length); resultBuffer.View.CopyToCPU(stream, ref results[0], length);
stream.Synchronize();
pool.Return(temp, clearArray: false); pool.Return(temp, clearArray: false);
resultBuffer.Dispose(); resultBuffer.Dispose();
inputBuffer.Dispose(); inputBuffer.Dispose();
PerfectNumbers.Core/ULongExtensions.Gpu.cs
+4 -5
View File
@@ -459,6 +459,7 @@ public static partial class ULongExtensions
GpuKernelLease lease = GpuKernelPool.GetKernel(); GpuKernelLease lease = GpuKernelPool.GetKernel();
Accelerator accelerator = lease.Accelerator; Accelerator accelerator = lease.Accelerator;
AcceleratorStream stream = lease.Stream;
// Keep this commented out. It should never happen in production code. // Keep this commented out. It should never happen in production code.
// if (accelerator.AcceleratorType == AcceleratorType.CPU) // if (accelerator.AcceleratorType == AcceleratorType.CPU)
// { // {
@@ -470,16 +471,14 @@ public static partial class ULongExtensions
var exponentBuffer = accelerator.Allocate1D<ulong>(1); var exponentBuffer = accelerator.Allocate1D<ulong>(1);
var resultBuffer = accelerator.Allocate1D<ulong>(1); var resultBuffer = accelerator.Allocate1D<ulong>(1);
exponentBuffer.View.CopyFromCPU(ref exponent, 1); exponentBuffer.View.CopyFromCPU(stream, ref exponent, 1);
// We don't need to worry about any left-overs here. // We don't need to worry about any left-overs here.
// resultBuffer.MemSetToZero(); // resultBuffer.MemSetToZero(stream);
var kernel = useKeepKernel ? kernelGroup.KeepMontgomery : kernelGroup.ConvertToStandard; var kernel = useKeepKernel ? kernelGroup.KeepMontgomery : kernelGroup.ConvertToStandard;
AcceleratorStream stream = lease.Stream;
kernel(stream, 1, exponentBuffer.View, divisor, resultBuffer.View); kernel(stream, 1, exponentBuffer.View, divisor, resultBuffer.View);
stream.Synchronize();
resultBuffer.View.CopyToCPU(stream, ref result, 1); resultBuffer.View.CopyToCPU(stream, ref result, 1);
stream.Synchronize();
resultBuffer.Dispose(); resultBuffer.Dispose();
exponentBuffer.Dispose(); exponentBuffer.Dispose();