mlemanczyk/even-perfect-numbers-scanner
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Streams added to all kernels and GPU operations to separate threads from each other.

Browse Source
This commit is contained in:
mlemanczyk committed 2025-11-06 18:53:55 +01:00
1 parent 64d615c402
commit de2a78b27b
30 files changed
+1097 -1002
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PerfectNumbers.Core.Tests/Gpu/NttGpuMathTests.cs
+336 -342
View File
@@ -13,367 +13,361 @@ namespace PerfectNumbers.Core.Tests.Gpu;
[Collection("GpuNtt")]
public class NttGpuMathTests
{
[Fact]
[Trait("Category", "Fast")]
public void ForwardGpu_reference_matches_cpu_forward()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Forward(cpu, modulus, primitiveRoot);
NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void ForwardGpu_reference_matches_cpu_forward()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Forward(cpu, modulus, primitiveRoot);
NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void ForwardGpu_staged_matches_cpu_forward()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Forward(cpu, modulus, primitiveRoot);
NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void ForwardGpu_staged_matches_cpu_forward()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Forward(cpu, modulus, primitiveRoot);
NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void InverseGpu_reference_matches_cpu_inverse()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> input = stackalloc GpuUInt128[4];
input[0] = new GpuUInt128(0UL, 5UL);
input[1] = new GpuUInt128(0UL, 7UL);
input[2] = new GpuUInt128(0UL, 9UL);
input[3] = new GpuUInt128(0UL, 11UL);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
Span<GpuUInt128> transformed = stackalloc GpuUInt128[4];
input.CopyTo(transformed);
NttGpuMath.Forward(transformed, modulus, primitiveRoot);
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
transformed.CopyTo(cpu);
NttGpuMath.Inverse(cpu, modulus, primitiveRoot);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
transformed.CopyTo(gpu);
NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < input.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void InverseGpu_reference_matches_cpu_inverse()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> input = stackalloc GpuUInt128[4];
input[0] = new GpuUInt128(0UL, 5UL);
input[1] = new GpuUInt128(0UL, 7UL);
input[2] = new GpuUInt128(0UL, 9UL);
input[3] = new GpuUInt128(0UL, 11UL);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
Span<GpuUInt128> transformed = stackalloc GpuUInt128[4];
input.CopyTo(transformed);
NttGpuMath.Forward(transformed, modulus, primitiveRoot);
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
transformed.CopyTo(cpu);
NttGpuMath.Inverse(cpu, modulus, primitiveRoot);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
transformed.CopyTo(gpu);
NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < input.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void InverseGpu_staged_matches_cpu_inverse()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> input = stackalloc GpuUInt128[4];
input[0] = new GpuUInt128(0UL, 5UL);
input[1] = new GpuUInt128(0UL, 7UL);
input[2] = new GpuUInt128(0UL, 9UL);
input[3] = new GpuUInt128(0UL, 11UL);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
Span<GpuUInt128> transformed = stackalloc GpuUInt128[4];
input.CopyTo(transformed);
NttGpuMath.Forward(transformed, modulus, primitiveRoot);
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
transformed.CopyTo(cpu);
NttGpuMath.Inverse(cpu, modulus, primitiveRoot);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
transformed.CopyTo(gpu);
NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < input.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void InverseGpu_staged_matches_cpu_inverse()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> input = stackalloc GpuUInt128[4];
input[0] = new GpuUInt128(0UL, 5UL);
input[1] = new GpuUInt128(0UL, 7UL);
input[2] = new GpuUInt128(0UL, 9UL);
input[3] = new GpuUInt128(0UL, 11UL);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
Span<GpuUInt128> transformed = stackalloc GpuUInt128[4];
input.CopyTo(transformed);
NttGpuMath.Forward(transformed, modulus, primitiveRoot);
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
transformed.CopyTo(cpu);
NttGpuMath.Inverse(cpu, modulus, primitiveRoot);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
transformed.CopyTo(gpu);
NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < input.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void Montgomery_twiddle_inverses_multiply_to_r2()
{
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
[Fact]
[Trait("Category", "Fast")]
public void Montgomery_twiddle_inverses_multiply_to_r2()
{
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
GpuContextLease gpu = GpuContextPool.Rent();
try
{
var accelerator = gpu.Accelerator;
GpuContextLease gpu = GpuContextPool.Rent();
try
{
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var method = typeof(NttGpuMath).GetMethod("GetSquareCache", BindingFlags.NonPublic | BindingFlags.Static);
method.Should().NotBeNull();
var cache = method!.Invoke(null, new object[] { accelerator, 8, modulus, primitiveRoot })!;
var cacheType = cache.GetType();
var method = typeof(NttGpuMath).GetMethod("GetSquareCache", BindingFlags.NonPublic | BindingFlags.Static);
method.Should().NotBeNull();
var cache = method!.Invoke(null, new object[] { accelerator, 8, modulus, primitiveRoot })!;
var cacheType = cache.GetType();
var twiddlesMont = (MemoryBuffer1D<GpuUInt128, Stride1D.Dense>)cacheType.GetProperty("TwiddlesMont")!.GetValue(cache)!;
var twiddlesInvMont = (MemoryBuffer1D<GpuUInt128, Stride1D.Dense>)cacheType.GetProperty("TwiddlesInvMont")!.GetValue(cache)!;
var r2 = (ulong)cacheType.GetProperty("MontR2Mod64")!.GetValue(cache)!;
var mod = (ulong)cacheType.GetProperty("ModulusLow")!.GetValue(cache)!;
var twiddlesMont = (MemoryBuffer1D<GpuUInt128, Stride1D.Dense>)cacheType.GetProperty("TwiddlesMont")!.GetValue(cache)!;
var twiddlesInvMont = (MemoryBuffer1D<GpuUInt128, Stride1D.Dense>)cacheType.GetProperty("TwiddlesInvMont")!.GetValue(cache)!;
var r2 = (ulong)cacheType.GetProperty("MontR2Mod64")!.GetValue(cache)!;
var mod = (ulong)cacheType.GetProperty("ModulusLow")!.GetValue(cache)!;
int count = 7; // length 8 -> 7 twiddles
var forward = new GpuUInt128[count];
var inverse = new GpuUInt128[count];
twiddlesMont.CopyToCPU(forward);
twiddlesInvMont.CopyToCPU(inverse);
int count = 7; // length 8 -> 7 twiddles
var forward = new GpuUInt128[count];
var inverse = new GpuUInt128[count];
twiddlesMont.CopyToCPU(stream, forward);
twiddlesInvMont.CopyToCPU(stream, inverse);
stream.Synchronize();
stream.Dispose();
for (int i = 0; i < count; i++)
{
ulong prod = (ulong)((((BigInteger)forward[i].Low) * inverse[i].Low) % mod);
prod.Should().Be(r2);
}
}
finally
{
gpu.Dispose();
}
}
for (int i = 0; i < count; i++)
{
ulong prod = (ulong)((((BigInteger)forward[i].Low) * inverse[i].Low) % mod);
prod.Should().Be(r2);
}
}
finally
{
gpu.Dispose();
}
}
[Fact]
[Trait("Category", "Fast")]
public void ConvolveGpu_reference_matches_cpu_convolve()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> leftCpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightCpu = stackalloc GpuUInt128[4];
leftCpu[0] = new GpuUInt128(0UL, 1UL);
leftCpu[1] = new GpuUInt128(0UL, 2UL);
leftCpu[2] = new GpuUInt128(0UL, 3UL);
leftCpu[3] = new GpuUInt128(0UL, 4UL);
rightCpu[0] = new GpuUInt128(0UL, 5UL);
rightCpu[1] = new GpuUInt128(0UL, 6UL);
rightCpu[2] = new GpuUInt128(0UL, 7UL);
rightCpu[3] = new GpuUInt128(0UL, 8UL);
Span<GpuUInt128> leftGpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightGpu = stackalloc GpuUInt128[4];
leftCpu.CopyTo(leftGpu);
rightCpu.CopyTo(rightGpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot);
NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot);
for (int i = 0; i < leftCpu.Length; i++)
{
leftGpu[i].Should().Be(leftCpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void ConvolveGpu_reference_matches_cpu_convolve()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> leftCpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightCpu = stackalloc GpuUInt128[4];
leftCpu[0] = new GpuUInt128(0UL, 1UL);
leftCpu[1] = new GpuUInt128(0UL, 2UL);
leftCpu[2] = new GpuUInt128(0UL, 3UL);
leftCpu[3] = new GpuUInt128(0UL, 4UL);
rightCpu[0] = new GpuUInt128(0UL, 5UL);
rightCpu[1] = new GpuUInt128(0UL, 6UL);
rightCpu[2] = new GpuUInt128(0UL, 7UL);
rightCpu[3] = new GpuUInt128(0UL, 8UL);
Span<GpuUInt128> leftGpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightGpu = stackalloc GpuUInt128[4];
leftCpu.CopyTo(leftGpu);
rightCpu.CopyTo(rightGpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot);
NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot);
for (int i = 0; i < leftCpu.Length; i++)
{
leftGpu[i].Should().Be(leftCpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void ConvolveGpu_staged_matches_cpu_convolve()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> leftCpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightCpu = stackalloc GpuUInt128[4];
leftCpu[0] = new GpuUInt128(0UL, 1UL);
leftCpu[1] = new GpuUInt128(0UL, 2UL);
leftCpu[2] = new GpuUInt128(0UL, 3UL);
leftCpu[3] = new GpuUInt128(0UL, 4UL);
rightCpu[0] = new GpuUInt128(0UL, 5UL);
rightCpu[1] = new GpuUInt128(0UL, 6UL);
rightCpu[2] = new GpuUInt128(0UL, 7UL);
rightCpu[3] = new GpuUInt128(0UL, 8UL);
Span<GpuUInt128> leftGpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightGpu = stackalloc GpuUInt128[4];
leftCpu.CopyTo(leftGpu);
rightCpu.CopyTo(rightGpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot);
NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot);
for (int i = 0; i < leftCpu.Length; i++)
{
leftGpu[i].Should().Be(leftCpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void ConvolveGpu_staged_matches_cpu_convolve()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> leftCpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightCpu = stackalloc GpuUInt128[4];
leftCpu[0] = new GpuUInt128(0UL, 1UL);
leftCpu[1] = new GpuUInt128(0UL, 2UL);
leftCpu[2] = new GpuUInt128(0UL, 3UL);
leftCpu[3] = new GpuUInt128(0UL, 4UL);
rightCpu[0] = new GpuUInt128(0UL, 5UL);
rightCpu[1] = new GpuUInt128(0UL, 6UL);
rightCpu[2] = new GpuUInt128(0UL, 7UL);
rightCpu[3] = new GpuUInt128(0UL, 8UL);
Span<GpuUInt128> leftGpu = stackalloc GpuUInt128[4];
Span<GpuUInt128> rightGpu = stackalloc GpuUInt128[4];
leftCpu.CopyTo(leftGpu);
rightCpu.CopyTo(rightGpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot);
NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot);
for (int i = 0; i < leftCpu.Length; i++)
{
leftGpu[i].Should().Be(leftCpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void SquareGpu_reference_matches_cpu_square()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Square(cpu, modulus, primitiveRoot);
NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void SquareGpu_reference_matches_cpu_square()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Reference;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Square(cpu, modulus, primitiveRoot);
NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void SquareGpu_staged_matches_cpu_square()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Square(cpu, modulus, primitiveRoot);
NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void SquareGpu_staged_matches_cpu_square()
{
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> cpu = stackalloc GpuUInt128[4];
cpu[0] = new GpuUInt128(0UL, 1UL);
cpu[1] = new GpuUInt128(0UL, 2UL);
cpu[2] = new GpuUInt128(0UL, 3UL);
cpu[3] = new GpuUInt128(0UL, 4UL);
Span<GpuUInt128> gpu = stackalloc GpuUInt128[4];
cpu.CopyTo(gpu);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
NttGpuMath.Square(cpu, modulus, primitiveRoot);
NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
[Fact]
[Trait("Category", "Fast")]
public void BitReverseGpu_matches_cpu()
{
GpuUInt128[] cpu = new GpuUInt128[8];
for (ulong i = 0; i < (ulong)cpu.Length; i++)
{
cpu[i] = new GpuUInt128(0UL, i);
}
[Fact]
[Trait("Category", "Fast")]
public void BitReverseGpu_matches_cpu()
{
GpuUInt128[] cpu = new GpuUInt128[8];
for (ulong i = 0; i < (ulong)cpu.Length; i++)
{
cpu[i] = new GpuUInt128(0UL, i);
}
GpuUInt128[] gpu = (GpuUInt128[])cpu.Clone();
NttGpuMath.BitReverse(cpu);
NttGpuMath.BitReverseGpu(gpu);
gpu.Should().Equal(cpu);
}
GpuUInt128[] gpu = (GpuUInt128[])cpu.Clone();
NttGpuMath.BitReverse(cpu);
NttGpuMath.BitReverseGpu(gpu);
gpu.Should().Equal(cpu);
}
[Fact]
[Trait("Category", "Fast")]
public void DisposeAll_clears_ntt_caches()
{
// Ensure the cache starts empty.
NttGpuMath.ClearCaches();
[Fact]
[Trait("Category", "Fast")]
public void DisposeAll_clears_ntt_caches()
{
// Ensure the cache starts empty.
NttGpuMath.ClearCaches();
Span<GpuUInt128> values = stackalloc GpuUInt128[2];
values[0] = new GpuUInt128(0UL, 1UL);
values[1] = new GpuUInt128(0UL, 2UL);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
GpuContextLease gpu = GpuContextPool.Rent();
try
{
var buffer = gpu.Accelerator.Allocate1D<GpuUInt128>(values.Length);
try
{
buffer.View.CopyFromCPU(ref values[0], values.Length);
NttGpuMath.SquareDevice(gpu.Accelerator, buffer.View, modulus, primitiveRoot);
}
finally
{
buffer.Dispose();
}
}
finally
{
gpu.Dispose();
}
Span<GpuUInt128> values = stackalloc GpuUInt128[2];
values[0] = new GpuUInt128(0UL, 1UL);
values[1] = new GpuUInt128(0UL, 2UL);
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
GpuContextLease gpu = GpuContextPool.Rent();
var stream = gpu.Accelerator.CreateStream();
var buffer = gpu.Accelerator.Allocate1D<GpuUInt128>(values.Length);
buffer.View.CopyFromCPU(stream, ref values[0], values.Length);
NttGpuMath.SquareDevice(gpu.Accelerator, stream, buffer.View, modulus, primitiveRoot);
stream.Synchronize();
buffer.Dispose();
stream.Dispose();
gpu.Dispose();
// Verify that the square cache now holds entries for the accelerator.
var field = typeof(NttGpuMath).GetField("SquareCache", BindingFlags.NonPublic | BindingFlags.Static)!;
var cacheObj = field.GetValue(null)!;
int count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!;
count.Should().BeGreaterThan(0);
// Verify that the square cache now holds entries for the accelerator.
var field = typeof(NttGpuMath).GetField("SquareCache", BindingFlags.NonPublic | BindingFlags.Static)!;
var cacheObj = field.GetValue(null)!;
int count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!;
count.Should().BeGreaterThan(0);
GpuContextPool.DisposeAll();
GpuContextPool.DisposeAll();
count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!;
count.Should().Be(0);
}
count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!;
count.Should().Be(0);
}
}
PerfectNumbers.Core.Tests/Gpu/NttMontgomeryTests.cs
+71 -73
View File
@@ -9,86 +9,84 @@ namespace PerfectNumbers.Core.Tests.Gpu;
[Trait("Category", "Fast")]
public class NttMontgomeryTests
{
[Fact]
[Trait("Category", "Fast")]
public void SquareDevice_montgomery64_matches_cpu_square_on_small_input()
{
// Small 64-bit modulus and tiny input to keep runtime trivial.
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
[Fact]
[Trait("Category", "Fast")]
public void SquareDevice_montgomery64_matches_cpu_square_on_small_input()
{
// Small 64-bit modulus and tiny input to keep runtime trivial.
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
Span<GpuUInt128> cpu = stackalloc GpuUInt128[8];
for (ulong i = 0; i < (ulong)cpu.Length; i++)
{
cpu[(int)i] = new GpuUInt128(0UL, i + 1UL);
}
Span<GpuUInt128> gpu = stackalloc GpuUInt128[16];
cpu.CopyTo(gpu);
for (int i = cpu.Length; i < gpu.Length; i++)
{
gpu[i] = new GpuUInt128(0UL, 0UL);
}
Span<GpuUInt128> cpu = stackalloc GpuUInt128[8];
for (ulong i = 0; i < (ulong)cpu.Length; i++)
{
cpu[(int)i] = new GpuUInt128(0UL, i + 1UL);
}
Span<GpuUInt128> gpu = stackalloc GpuUInt128[16];
cpu.CopyTo(gpu);
for (int i = cpu.Length; i < gpu.Length; i++)
{
gpu[i] = new GpuUInt128(0UL, 0UL);
}
// CPU reference
NttGpuMath.Square(cpu, modulus, primitiveRoot);
// CPU reference
NttGpuMath.Square(cpu, modulus, primitiveRoot);
// Device execution
GpuContextLease context = GpuContextPool.Rent();
try
{
var accelerator = context.Accelerator;
using var buffer = accelerator.Allocate1D<GpuUInt128>(gpu.Length);
buffer.View.CopyFromCPU(ref gpu[0], gpu.Length);
NttGpuMath.SquareDevice(accelerator, buffer.View, modulus, primitiveRoot);
buffer.View.CopyToCPU(ref gpu[0], gpu.Length);
// Device execution
GpuContextLease context = GpuContextPool.Rent();
var accelerator = context.Accelerator;
var stream = accelerator.CreateStream();
using var buffer = accelerator.Allocate1D<GpuUInt128>(gpu.Length);
buffer.View.CopyFromCPU(stream, ref gpu[0], gpu.Length);
NttGpuMath.SquareDevice(accelerator, stream, buffer.View, modulus, primitiveRoot);
buffer.View.CopyToCPU(stream, ref gpu[0], gpu.Length);
stream.Synchronize();
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
context.Dispose();
}
}
for (int i = 0; i < cpu.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
[Fact]
[Trait("Category", "Fast")]
public void InverseDevice_montgomery64_matches_cpu_inverse_on_small_input()
{
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
stream.Dispose();
context.Dispose();
}
Span<GpuUInt128> input = stackalloc GpuUInt128[8];
for (ulong i = 0; i < (ulong)input.Length; i++)
{
input[(int)i] = new GpuUInt128(0UL, i + 1UL);
}
[Fact]
[Trait("Category", "Fast")]
public void InverseDevice_montgomery64_matches_cpu_inverse_on_small_input()
{
var modulus = new GpuUInt128(0UL, 17UL);
var primitiveRoot = new GpuUInt128(0UL, 3UL);
Span<GpuUInt128> transformed = stackalloc GpuUInt128[input.Length];
input.CopyTo(transformed);
NttGpuMath.Forward(transformed, modulus, primitiveRoot);
Span<GpuUInt128> input = stackalloc GpuUInt128[8];
for (ulong i = 0; i < (ulong)input.Length; i++)
{
input[(int)i] = new GpuUInt128(0UL, i + 1UL);
}
Span<GpuUInt128> cpu = stackalloc GpuUInt128[input.Length];
transformed.CopyTo(cpu);
NttGpuMath.Inverse(cpu, modulus, primitiveRoot);
Span<GpuUInt128> transformed = stackalloc GpuUInt128[input.Length];
input.CopyTo(transformed);
NttGpuMath.Forward(transformed, modulus, primitiveRoot);
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> gpu = stackalloc GpuUInt128[input.Length];
transformed.CopyTo(gpu);
NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < input.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
Span<GpuUInt128> cpu = stackalloc GpuUInt128[input.Length];
transformed.CopyTo(cpu);
NttGpuMath.Inverse(cpu, modulus, primitiveRoot);
var original = NttGpuMath.GpuTransformBackend;
try
{
NttGpuMath.GpuTransformBackend = NttBackend.Staged;
Span<GpuUInt128> gpu = stackalloc GpuUInt128[input.Length];
transformed.CopyTo(gpu);
NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot);
for (int i = 0; i < input.Length; i++)
{
gpu[i].Should().Be(cpu[i]);
}
}
finally
{
NttGpuMath.GpuTransformBackend = original;
}
}
}
PerfectNumbers.Core/DivisorCycleCache.cs
+11 -6
View File
@@ -4,6 +4,7 @@ using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
using PerfectNumbers.Core.Gpu;
@@ -15,7 +16,7 @@ public sealed class DivisorCycleCache
private const byte ByteOne = 1;
private const int StackBufferThreshold = 128;
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>> _gpuKernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>> _gpuKernelCache = new(AcceleratorReferenceComparer.Instance);
private volatile ulong[] _snapshot;
private volatile bool _useGpuGeneration = true;
private readonly int _divisorCyclesBatchSize;
@@ -284,10 +285,11 @@ public sealed class DivisorCycleCache
int pending;
do
{
kernel(length, _divisorCyclesBatchSize, divisorBuffer.View, powBuffer.View, orderBuffer.View, resultBuffer.View, statusBuffer.View);
stream.Synchronize();
kernel(stream, length, _divisorCyclesBatchSize, divisorBuffer.View, powBuffer.View, orderBuffer.View, resultBuffer.View, statusBuffer.View);
statusBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(statusSpan), length);
stream.Synchronize();
pending = 0;
for (int i = 0; i < length; i++)
{
@@ -300,6 +302,7 @@ public sealed class DivisorCycleCache
while (pending > 0);
resultBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length);
stream.Synchronize();
resultSpan.CopyTo(destination);
if (rentedPow is not null)
{
@@ -312,9 +315,11 @@ public sealed class DivisorCycleCache
gpuLease.Dispose();
}
private static Action<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>> LoadKernel(Accelerator accelerator)
private static Action<AcceleratorStream, Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>> LoadKernel(Accelerator accelerator)
{
return accelerator.LoadAutoGroupedStreamKernel<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel);
var loaded = accelerator.LoadAutoGroupedStreamKernel<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>>();
}
private sealed class AcceleratorReferenceComparer : IEqualityComparer<Accelerator>
PerfectNumbers.Core/Gpu/GpuContextPool.cs
-1
View File
@@ -102,7 +102,6 @@ public static class GpuContextPool
internal static void Return(PooledContext ctx)
{
// ctx.Accelerator.Synchronize();
GpuPool.Enqueue(ctx);
}
}
PerfectNumbers.Core/Gpu/GpuKernelLease.cs
+4 -3
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -19,7 +20,7 @@ public sealed class GpuKernelLease
Kernels = kernels;
}
internal static GpuKernelLease Rent(GpuContextLease gpu, KernelContainer kernels)
internal static GpuKernelLease Rent(GpuContextLease gpu, AcceleratorStream stream, KernelContainer kernels)
{
if (!Pool.TryDequeue(out var lease))
{
@@ -28,7 +29,7 @@ public sealed class GpuKernelLease
lease._gpu = gpu;
lease.Kernels = kernels;
lease.Stream = gpu.Accelerator.CreateStream();
lease.Stream = stream;
return lease;
}
@@ -66,7 +67,7 @@ public sealed class GpuKernelLease
{
get
{
var accel = Accelerator;
var accel = Accelerator; // avoid capturing 'this' in lambda
return GpuKernelPool.InitOnce(ref Kernels!.Incremental, () =>
{
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernels.IncrementalKernelScan);
PerfectNumbers.Core/Gpu/GpuKernelPool.cs
+20 -19
View File
@@ -111,26 +111,26 @@ public class GpuKernelPool
return Interlocked.CompareExchange(ref target, newValue, null) ?? newValue;
}
internal static KernelContainer GetKernels(Accelerator accelerator)
internal static KernelContainer GetKernels(Accelerator accelerator, AcceleratorStream stream)
{
return _kernels.GetOrAdd(accelerator, static (accelerator) =>
return _kernels.GetOrAdd(accelerator, (accelerator) =>
{
var kernels = new KernelContainer();
PreloadStaticTables(kernels, accelerator);
PreloadStaticTables(kernels, accelerator, stream);
return kernels;
});
}
internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator)
internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
EnsureSmallCyclesOnDevice(kernels, accelerator);
EnsureSmallPrimesOnDevice(kernels, accelerator);
EnsureSmallPrimeFactorTables(kernels, accelerator);
EnsureSmallCyclesOnDevice(kernels, accelerator, stream);
EnsureSmallPrimesOnDevice(kernels, accelerator, stream);
EnsureSmallPrimeFactorTables(kernels, accelerator, stream);
}
// Ensures the small cycles table is uploaded to the device for the given accelerator.
// Returns the ArrayView to pass into kernels (when kernels are extended to accept it).
public static ArrayView1D<ulong, Stride1D.Dense> EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator)
public static ArrayView1D<ulong, Stride1D.Dense> EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
if (kernels.SmallCycles is { } buffer)
{
@@ -139,12 +139,12 @@ public class GpuKernelPool
var host = MersenneDivisorCycles.Shared.ExportSmallCyclesSnapshot(); // TODO: Preload this device buffer during startup and keep it immutable so we can delete the lock above in favor of the preloaded snapshot.
var device = accelerator.Allocate1D<ulong>(host.Length);
device.View.CopyFromCPU(host);
device.View.CopyFromCPU(stream, host);
kernels.SmallCycles = device;
return device.View;
}
public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator)
public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
if (kernels.SmallPrimesLastOne is { } lastOne &&
kernels.SmallPrimesLastSeven is { } lastSeven &&
@@ -168,13 +168,13 @@ public class GpuKernelPool
var hostLastSevenPow2 = PrimesGenerator.SmallPrimesPow2LastSeven;
var deviceLastOne = accelerator.Allocate1D<uint>(hostLastOne.Length);
deviceLastOne.View.CopyFromCPU(hostLastOne);
deviceLastOne.View.CopyFromCPU(stream, hostLastOne);
var deviceLastSeven = accelerator.Allocate1D<uint>(hostLastSeven.Length);
deviceLastSeven.View.CopyFromCPU(hostLastSeven);
deviceLastSeven.View.CopyFromCPU(stream, hostLastSeven);
var deviceLastOnePow2 = accelerator.Allocate1D<ulong>(hostLastOnePow2.Length);
deviceLastOnePow2.View.CopyFromCPU(hostLastOnePow2);
deviceLastOnePow2.View.CopyFromCPU(stream, hostLastOnePow2);
var deviceLastSevenPow2 = accelerator.Allocate1D<ulong>(hostLastSevenPow2.Length);
deviceLastSevenPow2.View.CopyFromCPU(hostLastSevenPow2);
deviceLastSevenPow2.View.CopyFromCPU(stream, hostLastSevenPow2);
kernels.SmallPrimesLastOne = deviceLastOne;
kernels.SmallPrimesLastSeven = deviceLastSeven;
@@ -184,7 +184,7 @@ public class GpuKernelPool
return new ResiduePrimeViews(deviceLastOne.View, deviceLastSeven.View, deviceLastOnePow2.View, deviceLastSevenPow2.View);
}
public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator)
public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
if (kernels.SmallPrimeFactorsPrimes is { } primeBuffer && kernels.SmallPrimeFactorsSquares is { } squareBuffer)
{
@@ -200,9 +200,9 @@ public class GpuKernelPool
var hostSquares = PrimesGenerator.SmallPrimesPow2;
var devicePrimes = accelerator.Allocate1D<uint>(hostPrimes.Length);
devicePrimes.View.CopyFromCPU(hostPrimes);
devicePrimes.View.CopyFromCPU(stream, hostPrimes);
var deviceSquares = accelerator.Allocate1D<ulong>(hostSquares.Length);
deviceSquares.View.CopyFromCPU(hostSquares);
deviceSquares.View.CopyFromCPU(stream, hostSquares);
kernels.SmallPrimeFactorsPrimes = devicePrimes;
kernels.SmallPrimeFactorsSquares = deviceSquares;
@@ -218,8 +218,9 @@ public class GpuKernelPool
{
GpuPrimeWorkLimiter.Acquire();
var gpu = Rent();
var kernels = GetKernels(gpu.Accelerator);
return GpuKernelLease.Rent(gpu, kernels);
var stream = gpu.Accelerator.CreateStream();
var kernels = GetKernels(gpu.Accelerator, stream);
return GpuKernelLease.Rent(gpu, stream, kernels);
}
/// <summary>
PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs
-10
View File
@@ -44,11 +44,6 @@ public sealed class ScratchBuffer
SmallPrimeFactorPrimeSlots = _accelerator.Allocate1D<ulong>(newSize);
SmallPrimeFactorExponentSlots = _accelerator.Allocate1D<int>(newSize);
}
// else
// {
// SmallPrimeFactorPrimeSlots.MemSetToZero();
// SmallPrimeFactorExponentSlots.MemSetToZero();
// }
}
public void ResizeSpecialMaxFactors(int newSize)
@@ -61,11 +56,6 @@ public sealed class ScratchBuffer
SpecialMaxFactors = _accelerator.Allocate1D<ulong>(newSize);
SpecialMaxCandidates = _accelerator.Allocate1D<ulong>(newSize);
}
// else
// {
// SpecialMaxFactors.MemSetToZero();
// SpecialMaxCandidates.MemSetToZero();
// }
}
}
PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs
+4 -4
View File
@@ -11,13 +11,13 @@ internal static class GpuStaticTableInitializer
{
private static readonly ConcurrentDictionary<Accelerator, byte> Initialized = new(AcceleratorReferenceComparer.Instance);
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator)
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
Initialized.GetOrAdd(accelerator, acc =>
{
PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc);
GpuKernelPool.PreloadStaticTables(kernels, acc);
PrimeOrderGpuHeuristics.PreloadStaticTables( acc);
PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc, stream);
GpuKernelPool.PreloadStaticTables(kernels, acc, stream);
PrimeOrderGpuHeuristics.PreloadStaticTables( acc, stream);
return 0;
});
}
PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs
+1 -1
View File
@@ -305,7 +305,7 @@ internal static class LucasLehmerKernels
states[idx] = s;
}
public static void Kernel(Index1D index, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
public static void Kernel(Index1D _, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
{
// Lucas–Lehmer iteration in the field GF(2^p-1).
// For p < 128, use a Mersenne-specific squaring + reduction to avoid
PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs
+2 -1
View File
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices;
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -32,7 +33,7 @@ internal static class NttBarrettKernels
}
// Barrett-based squaring: v = v^2 mod n for 128-bit n
public static void SquareBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
public static void SquareBarrett128Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
{
var v = data[index];
Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0);
PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs
+3 -2
View File
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices;
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -29,7 +30,7 @@ internal static class NttMontgomeryKernels
data[i2] = new GpuUInt128(0UL, diff);
}
public static void ToMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
public static void ToMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
{
ulong a = data[index].Low;
ulong v = MontMul64(a, r2, modulus, nPrime);
@@ -43,7 +44,7 @@ internal static class NttMontgomeryKernels
data[index] = new GpuUInt128(0UL, v);
}
public static void SquareMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
public static void SquareMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
{
ulong aR = data[index].Low;
ulong v = MontMul64(aR, aR, modulus, nPrime);
PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs
+1
View File
@@ -1,4 +1,5 @@
using ILGPU;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs
+41 -41
View File
@@ -24,14 +24,17 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private ulong _divisorLimit;
// private bool _isConfigured;
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>> _kernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>> _kernelCache = new(AcceleratorReferenceComparer.Instance);
private readonly ConcurrentBag<BatchResources> _resourcePool = new();
private readonly ConcurrentBag<GpuContextLease> _acceleratorPool = new();
private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new();
private Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc =>
acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.CheckKernel));
private Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>(DivisorByDivisorKernels.CheckKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>>>();
});
public int GpuBatchSize
@@ -92,8 +95,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ulong processedCount;
ulong lastProcessed;
var gpuLease = RentAccelerator();
var gpuLease = GpuKernelPool.GetKernel();
var accelerator = gpuLease.Accelerator;
var stream = gpuLease.Stream;
// Monitor.Enter(gpuLease.ExecutionLock);
@@ -104,6 +108,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
prime,
allowedMax,
kernel,
stream,
resources.DivisorDataBuffer,
resources.OffsetBuffer,
resources.CountBuffer,
@@ -122,9 +127,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
out processedCount);
ReturnBatchResources(resources);
// Monitor.Exit(gpuLease.ExecutionLock);
ReturnAccelerator(gpuLease);
// Monitor.Exit(gpuLease.ExecutionLock);
gpuLease.Dispose();
if (composite)
{
divisorsExhausted = true;
@@ -167,7 +172,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
private static bool CheckDivisors(
ulong prime,
ulong allowedMax,
Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> kernel,
Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> kernel,
AcceleratorStream stream,
MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> divisorDataBuffer,
MemoryBuffer1D<int, Stride1D.Dense> offsetBuffer,
MemoryBuffer1D<int, Stride1D.Dense> countBuffer,
@@ -329,19 +335,20 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
continue;
}
divisorDataView.CopyFromCPU(ref divisorDataRef, admissibleCount);
offsetView.CopyFromCPU(ref offsetRef, admissibleCount);
countView.CopyFromCPU(ref countRef, admissibleCount);
cycleView.CopyFromCPU(ref cycleRef, admissibleCount);
exponentViewDevice.CopyFromCPU(ref exponentRef, admissibleCount);
divisorDataView.CopyFromCPU(stream, ref divisorDataRef, admissibleCount);
offsetView.CopyFromCPU(stream, ref offsetRef, admissibleCount);
countView.CopyFromCPU(stream, ref countRef, admissibleCount);
cycleView.CopyFromCPU(stream, ref cycleRef, admissibleCount);
exponentViewDevice.CopyFromCPU(stream, ref exponentRef, admissibleCount);
int sentinel = int.MaxValue;
hitIndexView.CopyFromCPU(ref sentinel, 1);
hitIndexView.CopyFromCPU(stream, ref sentinel, 1);
kernel(admissibleCount, divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView);
kernel(stream, new Index1D(admissibleCount), divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView);
int firstHit = sentinel;
hitIndexView.CopyToCPU(ref firstHit, 1);
hitIndexView.CopyToCPU(stream, ref firstHit, 1);
stream.Synchronize();
int hitIndex = firstHit >= admissibleCount ? -1 : firstHit;
bool hitFound = hitIndex >= 0;
composite = hitFound;
@@ -477,9 +484,10 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
public sealed class DivisorScanSession : IMersenneNumberDivisorByDivisorTester.IDivisorScanSession
{
private readonly MersenneNumberDivisorByDivisorGpuTester _owner;
private readonly GpuContextLease _lease;
private readonly GpuKernelLease _lease;
private readonly Accelerator _accelerator;
private Action<Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> _kernel = null!;
private readonly AcceleratorStream _stream;
private Action<AcceleratorStream, Index1D, ArrayView<GpuDivisorPartialData>, ArrayView<int>, ArrayView<int>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>, ArrayView<int>> _kernel = null!;
private MemoryBuffer1D<GpuDivisorPartialData, Stride1D.Dense> _divisorBuffer = null!;
private MemoryBuffer1D<int, Stride1D.Dense> _offsetBuffer = null!;
private MemoryBuffer1D<int, Stride1D.Dense> _countBuffer = null!;
@@ -493,8 +501,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
internal DivisorScanSession(MersenneNumberDivisorByDivisorGpuTester owner)
{
_owner = owner;
_lease = GpuContextPool.Rent();
_lease = GpuKernelPool.GetKernel();
_accelerator = _lease.Accelerator;
_stream = _lease.Stream;
_capacity = Math.Max(1, owner._gpuBatchSize);
}
@@ -593,35 +602,36 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
ArrayView1D<ulong, Stride1D.Dense> exponentView = _exponentsBuffer.View;
ArrayView1D<byte, Stride1D.Dense> hitView = _hitBuffer.View;
GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor);
divisorView.CopyFromCPU(ref partialData, 1);
AcceleratorStream stream = _stream;
GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor);
divisorView.CopyFromCPU(stream, ref partialData, 1);
int offsetValue = 0;
offsetView.CopyFromCPU(ref offsetValue, 1);
offsetView.CopyFromCPU(stream, ref offsetValue, 1);
int countValue = length;
countView.CopyFromCPU(ref countValue, 1);
countView.CopyFromCPU(stream, ref countValue, 1);
ulong cycleValue = cycle;
cycleView.CopyFromCPU(ref cycleValue, 1);
cycleView.CopyFromCPU(stream, ref cycleValue, 1);
Span<ulong> hostSpan = _hostBuffer.AsSpan(0, length);
primes.CopyTo(hostSpan);
ref ulong hostRef = ref MemoryMarshal.GetReference(hostSpan);
exponentView.CopyFromCPU(ref hostRef, length);
exponentView.CopyFromCPU(stream, ref hostRef, length);
int sentinel = int.MaxValue;
firstHitView.CopyFromCPU(ref sentinel, 1);
firstHitView.CopyFromCPU(stream, ref sentinel, 1);
var kernel = _kernel!;
kernel(1, divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView);
kernel(stream, new Index1D(1), divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView);
Span<byte> hitSlice = hits.Slice(0, length);
ref byte hitRef = ref MemoryMarshal.GetReference(hitSlice);
hitView.CopyToCPU(ref hitRef, length);
hitView.CopyToCPU(stream, ref hitRef, length);
stream.Synchronize();
// Monitor.Exit(_lease.ExecutionLock);
}
@@ -651,17 +661,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
}
}
private GpuContextLease RentAccelerator()
{
if (_acceleratorPool.TryTake(out var lease))
{
return lease;
}
return GpuContextPool.Rent();
}
private void ReturnAccelerator(GpuContextLease lease) => _acceleratorPool.Add(lease);
private BatchResources RentBatchResources(Accelerator accelerator, int capacity)
{
PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs
+12 -9
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
namespace PerfectNumbers.Core.Gpu;
@@ -9,11 +10,16 @@ namespace PerfectNumbers.Core.Gpu;
/// </summary>
public sealed class MersenneNumberDivisorGpuTester
{
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new();
private readonly ConcurrentDictionary<Accelerator, MemoryBuffer1D<byte, Stride1D.Dense>> _resultBuffers = new();
private Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel));
private Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
_kernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>>();
});
public static void BuildDivisorCandidates()
{
@@ -43,15 +49,12 @@ public sealed class MersenneNumberDivisorGpuTester
var kernel = GetKernel(accelerator);
var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D<byte>(1));
// There is no point in clearing this buffer. We always override item [0] and never use it beyond item [0]
// resultBuffer.MemSetToZero();
kernel(1, exponent, divisor, resultBuffer.View);
stream.Synchronize();
// resultBuffer.MemSetToZero(stream);
kernel(stream, 1, exponent, divisor, resultBuffer.View);
Span<byte> result = stackalloc byte[1];
resultBuffer.View.CopyToCPU(stream, ref result[0], 1);
stream.Synchronize();
bool divisible = result[0] != 0;
result[0] = 0;
resultBuffer.View.CopyFromCPU(stream, ref result[0], 1);
stream.Synchronize();
stream.Dispose();
gpu.Dispose();
return divisible;
PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs
+3 -3
View File
@@ -31,11 +31,11 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u
ulong step3 = ((exponent % 3UL) << 1) % 3UL;
ulong step5 = ((exponent % 5UL) << 1) % 5UL;
GpuUInt128 twoPGpu = (GpuUInt128)twoP;
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator);
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream);
ResiduePrimeViews primeViews = default;
if (_kernelType == GpuKernelType.Pow2Mod)
{
primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator);
primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator, stream);
}
var orderBuffer = accelerator.Allocate1D<ulong>(batchSize);
@@ -92,8 +92,8 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u
smallCyclesView);
}
orderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(orders), currentSize);
stream.Synchronize();
orderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(orders), currentSize);
if (_kernelType == GpuKernelType.Pow2Mod)
{
for (i = 0; i < currentSize; i++)
PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs
+72 -38
View File
@@ -1,6 +1,7 @@
using System.Buffers;
using System.Collections.Concurrent;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
using UInt128 = System.UInt128;
@@ -8,32 +9,60 @@ namespace PerfectNumbers.Core.Gpu;
public class MersenneNumberLucasLehmerGpuTester
{
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>> KernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> AddSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> SubSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ArrayView<GpuUInt128>>> ReduceKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>> IsZeroKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>> BatchKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>> KernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> AddSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> SubSmallKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>> ReduceKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>> IsZeroKernelCache = new();
private readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>> BatchKernelCache = new();
private Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator)
{
return KernelCache.GetOrAdd(accelerator, accel => accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel));
}
private Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator) =>
KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) =>
AddSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel));
private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) =>
AddSmallKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) =>
SubSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel));
private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) =>
SubSmallKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) =>
ReduceKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel));
private Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) =>
ReduceKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView<GpuUInt128>>>();
});
private Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) =>
IsZeroKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel));
private Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) =>
IsZeroKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>>();
});
private Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) =>
BatchKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch));
private Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) =>
BatchKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>>();
});
// Configurable LL slice size to keep kernels short. Default 32.
public int SliceSize = 32;
@@ -70,13 +99,16 @@ public class MersenneNumberLucasLehmerGpuTester
{
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetKernel(accelerator);
var modulus = new GpuUInt128(((UInt128)1 << (int)exponent) - 1UL); // TODO: Cache these Mersenne moduli per exponent so LL GPU runs skip rebuilding them every launch.
var buffer = accelerator.Allocate1D<GpuUInt128>(1);
kernel(1, exponent, modulus, buffer.View);
accelerator.Synchronize();
kernel(stream, 1, exponent, modulus, buffer.View);
stream.Synchronize();
// TODO: Replace this with .CopyToCPU with pre-allocated buffer using stream. Synchronize on the stream after copy.
result = buffer.GetAsArray1D()[0].IsZero;
buffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -102,12 +134,13 @@ public class MersenneNumberLucasLehmerGpuTester
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetBatchKernel(accelerator);
var expBuffer = accelerator.Allocate1D<ulong>(count);
ulong[] expArray = ArrayPool<ulong>.Shared.Rent(count);
exponents.CopyTo(expArray);
expBuffer.View.CopyFromCPU(ref expArray[0], count);
expBuffer.View.CopyFromCPU(stream, ref expArray[0], count);
GpuUInt128[] modulusArray = ArrayPool<GpuUInt128>.Shared.Rent(count);
for (int i = 0; i < count; i++)
@@ -118,18 +151,19 @@ public class MersenneNumberLucasLehmerGpuTester
}
var modBuffer = accelerator.Allocate1D<GpuUInt128>(count);
modBuffer.View.CopyFromCPU(ref modulusArray[0], count);
modBuffer.View.CopyFromCPU(stream, ref modulusArray[0], count);
var stateBuffer = accelerator.Allocate1D<GpuUInt128>(count);
kernel(count, expBuffer.View, modBuffer.View, stateBuffer.View);
accelerator.Synchronize();
stateBuffer.View.CopyToCPU(ref residues[0], count);
kernel(stream, count, expBuffer.View, modBuffer.View, stateBuffer.View);
stateBuffer.View.CopyToCPU(stream, ref residues[0], count);
stream.Synchronize();
stateBuffer.Dispose();
modBuffer.Dispose();
expBuffer.Dispose();
ArrayPool<GpuUInt128>.Shared.Return(modulusArray);
ArrayPool<ulong>.Shared.Return(expArray);
stream.Dispose();
gpu.Dispose();
}
@@ -154,6 +188,7 @@ public class MersenneNumberLucasLehmerGpuTester
{
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var addKernel = GetAddSmallKernel(accelerator);
var subKernel = GetSubSmallKernel(accelerator);
var reduceKernel = GetReduceKernel(accelerator);
@@ -167,8 +202,8 @@ public class MersenneNumberLucasLehmerGpuTester
transformLength <<= 1;
}
var stateBuffer = accelerator.Allocate1D<GpuUInt128>(transformLength);
stateBuffer.MemSetToZero();
addKernel(1, 4UL, stateBuffer.View);
stateBuffer.MemSetToZero(stream);
addKernel(stream, 1, 4UL, stateBuffer.View);
// TODO(LL-SLICE): Slice Lucas–Lehmer iterations into short batches to
// avoid long-running kernels and TDR. Example: process 8–64 iterations
@@ -183,26 +218,25 @@ public class MersenneNumberLucasLehmerGpuTester
for (int s = 0; s < iter; s++)
{
// Square in NTT domain with staged kernels (short runtime per kernel).
NttGpuMath.SquareDevice(accelerator, stateBuffer.View, nttMod, primitiveRoot);
subKernel(1, 2UL, stateBuffer.View);
reduceKernel(1, exponent, stateBuffer.View);
NttGpuMath.SquareDevice(accelerator, stream, stateBuffer.View, nttMod, primitiveRoot);
subKernel(stream, 1, 2UL, stateBuffer.View);
reduceKernel(stream, 1, exponent, stateBuffer.View);
}
// Synchronize between slices to yield to scheduler and avoid TDR.
accelerator.Synchronize();
i += (ulong)iter;
}
reduceKernel(1, exponent, stateBuffer.View);
reduceKernel(stream, 1, exponent, stateBuffer.View);
var resultBuffer = accelerator.Allocate1D<byte>(1);
zeroKernel(1, stateBuffer.View, resultBuffer.View);
accelerator.Synchronize();
zeroKernel(stream, 1, stateBuffer.View, resultBuffer.View);
byte[] result = new byte[1];
resultBuffer.View.CopyToCPU(result);
resultBuffer.View.CopyToCPU(stream, result);
stream.Synchronize();
bool isPrime = result[0] != 0;
resultBuffer.Dispose();
stateBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
return isPrime;
}
PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs
+3 -3
View File
@@ -30,13 +30,13 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO
int currentSize;
int found = 0;
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator);
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream);
while (kStart <= maxK && Volatile.Read(ref isPrime))
{
remaining = maxK - kStart + 1UL;
currentSize = remaining > (UInt128)batchSize ? batchSize : (int)remaining;
// This clear is required because the result may not always be set. Remove it after the kernels are modified to always set the result.
foundBuffer.MemSetToZero();
foundBuffer.MemSetToZero(stream);
// Precompute residue automaton bases for this batch
UInt128 q0 = twoP * kStart + 1UL;
// TODO: Replace these modulo recomputations with the divisor-cycle stepping deltas captured in
@@ -49,8 +49,8 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO
kernel(stream, currentSize, exponent, (GpuUInt128)twoP, (GpuUInt128)kStart, last, divMul,
kernelArgs, foundBuffer.View, smallCyclesView);
foundBuffer.View.CopyToCPU(stream, ref found, 1);
stream.Synchronize();
foundBuffer.View.CopyToCPU(ref found, 1);
if (found != 0)
{
Volatile.Write(ref isPrime, false);
PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs
+7 -6
View File
@@ -36,15 +36,16 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder)
var stream = accelerator.CreateStream();
var kernel = GetPow2ModKernel(accelerator);
var resources = RentResources(accelerator, GpuConstants.ScanBatchSize);
var gpuKernels = GpuKernelPool.GetKernels(accelerator);
var gpuKernels = GpuKernelPool.GetKernels(accelerator, stream);
var orderBuffer = resources.OrderBuffer;
ulong[] orderArray = resources.OrderArray;
int batchSize = resources.Capacity;
int batchSize = resources.Capacity;
// Ensure device has small cycles and primes tables for in-kernel lookup
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator);
ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator);
// TODO: Pass-in the stream down the path
var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator, stream);
ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator, stream);
UInt128 kStart = 1UL;
UInt128 limit = maxK + UInt128.One;
@@ -70,9 +71,9 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder)
kernel(stream, currentSize, exponent, twoPGpu, (GpuUInt128)kStart, last, 0UL,
kernelArgs, orderBuffer.View, smallCyclesView, primeViews.LastOne, primeViews.LastSeven, primeViews.LastOnePow2, primeViews.LastSevenPow2);
stream.Synchronize();
orderBuffer.View.CopyToCPU(ref ordersRef, currentSize);
orderBuffer.View.CopyToCPU(stream, ref ordersRef, currentSize);
stream.Synchronize();
if (!Volatile.Read(ref isPrime))
{
break;
PerfectNumbers.Core/Gpu/NttGpuMath.cs
+185 -126
View File
@@ -27,25 +27,41 @@ public static class NttGpuMath
// Global backend setting for GPU transforms. Used by ForwardGpu/InverseGpu.
// TODO(NTT-OPT): Consider passing backend explicitly to avoid global state.
public static NttBackend GpuTransformBackend { get; set; } = NttBackend.Reference;
// Controls modular reduction strategy inside staged NTT paths.
public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto;
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> StageKernelCache = new(); // TODO: Same as above – materialize staged kernels during startup so we can drop ConcurrentDictionary usage entirely per the benchmark guidance.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Promote to the static kernel table initialized alongside the GpuModularArithmeticBenchmarks fast path.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>> StageMontKernelCache = new(); // TODO: Inline the Montgomery kernels into the startup table instead of using a concurrent cache now that we no longer mutate state at runtime.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Same plan – reuse the precomputed kernels measured fastest in MontgomeryMultiplyBenchmarks rather than looking them up dynamically.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Collapse into the startup kernel array once Barrett128 constants are preloaded.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> SquareBarrett128KernelCache = new(); // TODO: Fold these kernels into the same startup table to avoid concurrent access overhead highlighted in GpuModularArithmeticBenchmarks.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Prebind and reuse the Montgomery conversions from MontgomeryMultiplyBenchmarks instead of storing them in a concurrent cache.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong>> FromMont64KernelCache = new(); // TODO: As above – drop ConcurrentDictionary once startup prewarms every kernel.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong>> SquareMont64KernelCache = new(); // TODO: Inline into the static kernel table established during initialization.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Move to the startup kernel table so Montgomery-scaled launches skip concurrent lookups.
// Controls modular reduction strategy inside staged NTT paths.
public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto;
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream,Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> StageKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Warm these forward kernels during initialization and store them in a plain array so the LucasLehmerGpuBenchmarks launch costs disappear.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Same as above – replace with startup-prepared tables per LucasLehmerGpuBenchmarks guidance.
private static readonly ConcurrentDictionary<Accelerator, Action<Index1D, ArrayView<GpuUInt128>, int>> BitReverseKernelCache = new(); // TODO: Precompute bit-reversal kernels during boot so we can remove the concurrent lookup overhead flagged in the benchmarks.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>> StageMontKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong>> FromMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>> SquareMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static readonly ConcurrentDictionary<Accelerator, Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int>> BitReverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup.
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) =>
StageKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>>();
});
private sealed class SquareCacheEntry
{
@@ -72,7 +88,7 @@ public static class NttGpuMath
public ulong BarrettMuHigh { get; }
public ulong BarrettMuLow { get; }
public SquareCacheEntry(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
public SquareCacheEntry(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{
Buffer = accelerator.Allocate1D<GpuUInt128>(length);
Root = new GpuUInt128(primitiveRoot);
@@ -136,9 +152,9 @@ public static class NttGpuMath
}
Twiddles = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
Twiddles.View.CopyFromCPU(ref forward[0], twiddleCount);
Twiddles.View.CopyFromCPU(stream, ref forward[0], twiddleCount);
TwiddlesInv = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesInv.View.CopyFromCPU(ref inverse[0], twiddleCount);
TwiddlesInv.View.CopyFromCPU(stream, ref inverse[0], twiddleCount);
// Prepare Montgomery constants for 64-bit moduli and twiddles in Montgomery domain.
if (modulus.High == 0UL && (ReductionMode == ModReductionMode.Auto || ReductionMode == ModReductionMode.Mont64))
@@ -166,9 +182,9 @@ public static class NttGpuMath
}
TwiddlesMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesMont.View.CopyFromCPU(ref forwardMont[0], twiddleCount);
TwiddlesMont.View.CopyFromCPU(stream, ref forwardMont[0], twiddleCount);
TwiddlesInvMont = accelerator.Allocate1D<GpuUInt128>(twiddleCount);
TwiddlesInvMont.View.CopyFromCPU(ref inverseMont[0], twiddleCount);
TwiddlesInvMont.View.CopyFromCPU(stream, ref inverseMont[0], twiddleCount);
pool.Return(forwardMont, clearArray: true);
pool.Return(inverseMont, clearArray: true);
@@ -258,11 +274,11 @@ public static class NttGpuMath
// - Store one GPU buffer with all stage twiddles, or per-stage slices.
// - Expose accessors to retrieve per-stage views used by butterfly kernels.
private static SquareCacheEntry GetSquareCache(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
private static SquareCacheEntry GetSquareCache(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{
var perAccel = SquareCache.GetOrAdd(accelerator, _ => new ConcurrentDictionary<SquareCacheKey, SquareCacheEntry>());
var key = new SquareCacheKey(length, modulus, primitiveRoot);
return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, length, modulus, primitiveRoot));
return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, stream, length, modulus, primitiveRoot));
}
private static readonly ulong[] SmallPrimes = GenerateSmallPrimes(1000);
@@ -349,7 +365,8 @@ public static class NttGpuMath
SquareMont64KernelCache.Clear();
ScaleMont64KernelCache.Clear();
ForwardKernelCache.Clear();
InverseKernelCache.Clear();
InverseKernelCache.Clear();
BitReverseKernelCache.Clear();
// TODO(NTT-OPT): Clear TwiddleCache once introduced.
return;
}
@@ -373,56 +390,97 @@ public static class NttGpuMath
ScaleMont64KernelCache.TryRemove(accelerator, out _);
ForwardKernelCache.TryRemove(accelerator, out _);
InverseKernelCache.TryRemove(accelerator, out _);
BitReverseKernelCache.TryRemove(accelerator, out _);
// TODO(NTT-OPT): Remove per-accelerator twiddle buffers once added.
}
private static Action<Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) =>
BitReverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) =>
BitReverseKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) =>
MulKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) =>
ForwardKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) =>
StageKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) =>
FromMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) =>
ScaleKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
InverseKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) =>
StageMontKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) =>
MulKernelCache.GetOrAdd(accelerator, valueFactory: acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) =>
StageBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) =>
ScaleKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) =>
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) =>
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetSquareBarrett128Kernel(Accelerator accelerator) =>
SquareBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.SquareBarrett128Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) =>
ScaleMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) =>
ToMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ToMont64Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) =>
StageBarrett128KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) =>
FromMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) =>
StageMontKernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>>();
});
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetSquareMont64Kernel(Accelerator accelerator) =>
SquareMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.SquareMont64Kernel));
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) =>
ScaleMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel));
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) =>
ForwardKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel));
// TODO(NTT-OPT): Introduce stage-wise Cooley–Tukey kernels and a
// per-(accelerator,length,modulus,root) twiddle cache. The plan:
// - Precompute twiddle factors for all stages once and cache on GPU.
// - Replace the current O(n^2) ForwardKernel with O(n log n) butterflies.
// - Provide accessors like GetStageKernel(...) and GetTwiddleBuffer(...).
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
InverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel));
private static Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) =>
ToMont64KernelCache.GetOrAdd(accelerator, acc =>
{
var loaded = acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel);
var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>>();
});
// TODO(NTT-OPT): Mirror the forward stage-wise design for the inverse
// transform. Precompute inverse twiddles and a normalization factor
@@ -453,13 +511,15 @@ public static class NttGpuMath
var array = pool.Rent(n);
values.CopyTo(array);
var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref array[0], n);
kernel(n, buffer.View, bits);
accelerator.Synchronize();
buffer.View.CopyToCPU(ref array[0], n);
var stream = accelerator.CreateStream();
buffer.View.CopyFromCPU(stream, ref array[0], n);
kernel(stream, n, buffer.View, bits);
buffer.View.CopyToCPU(stream, ref array[0], n);
stream.Synchronize();
array.AsSpan(0, n).CopyTo(values);
pool.Return(array, clearArray: true);
buffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -510,6 +570,7 @@ public static class NttGpuMath
int n = (int)values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetForwardKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared;
var inputArray = pool.Rent(n);
@@ -517,19 +578,20 @@ public static class NttGpuMath
values.CopyTo(inputArray);
var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
inputBuffer.View.CopyFromCPU(ref inputArray[0], n);
inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n);
UInt128 modValue = modulus;
UInt128 expBase = (modValue - 1UL) / (ulong)n;
var root = new GpuUInt128(primitiveRoot);
root.ModPow((ulong)expBase, modulus);
kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, root);
accelerator.Synchronize();
outputBuffer.View.CopyToCPU(ref outputArray[0], n);
kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, root);
outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n);
stream.Synchronize();
outputArray.AsSpan(0, n).CopyTo(values);
pool.Return(inputArray, clearArray: true);
pool.Return(outputArray, clearArray: true);
outputBuffer.Dispose();
inputBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -607,6 +669,7 @@ public static class NttGpuMath
int n = values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetInverseKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared;
var inputArray = pool.Rent(n);
@@ -614,7 +677,7 @@ public static class NttGpuMath
values.CopyTo(inputArray);
var inputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
var outputBuffer = accelerator.Allocate1D<GpuUInt128>(n);
inputBuffer.View.CopyFromCPU(ref inputArray[0], n);
inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n);
UInt128 modValue = modulus;
UInt128 expBase = (modValue - 1UL) / (ulong)n;
var root = new GpuUInt128(primitiveRoot);
@@ -638,14 +701,15 @@ public static class NttGpuMath
{
nInv.ModInv(modulus);
}
kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv);
accelerator.Synchronize();
outputBuffer.View.CopyToCPU(ref outputArray[0], n);
kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv);
outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n);
stream.Synchronize();
outputArray.AsSpan(0, n).CopyTo(values);
pool.Return(inputArray, clearArray: true);
pool.Return(outputArray, clearArray: true);
outputBuffer.Dispose();
inputBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
@@ -655,11 +719,14 @@ public static class NttGpuMath
int n = values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref values[0], n);
var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot);
ForwardDevice(accelerator, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(ref values[0], n);
buffer.View.CopyFromCPU(stream, ref values[0], n);
var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot);
ForwardDevice(accelerator, stream, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(stream, ref values[0], n);
stream.Synchronize();
stream.Dispose();
buffer.Dispose();
gpu.Dispose();
@@ -670,38 +737,38 @@ public static class NttGpuMath
int n = values.Length;
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var buffer = accelerator.Allocate1D<GpuUInt128>(n);
buffer.View.CopyFromCPU(ref values[0], n);
var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot);
InverseDevice(accelerator, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(ref values[0], n);
buffer.View.CopyFromCPU(stream, ref values[0], n);
var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot);
InverseDevice(accelerator, stream, buffer.View, n, modulus, cache);
buffer.View.CopyToCPU(stream, ref values[0], n);
stream.Synchronize();
stream.Dispose();
buffer.Dispose();
gpu.Dispose();
}
private static void ForwardDevice(Accelerator accelerator, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
private static void ForwardDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
{
var bitKernel = GetBitReverseKernel(accelerator);
bitKernel(length, data, cache.Bits);
accelerator.Synchronize();
bitKernel(stream, length, data, cache.Bits);
if (cache.UseMontgomery64)
{
var toMont = GetToMont64Kernel(accelerator);
toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
accelerator.Synchronize();
toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
var stageKernel = GetStageMontKernel(accelerator);
int butterflies = length >> 1;
for (int s = 0; s < cache.Bits; s++)
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64);
accelerator.Synchronize();
stageKernel( stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64);
}
var fromMont = GetFromMont64Kernel(accelerator);
fromMont(length, data, cache.ModulusLow, cache.MontNPrime64);
fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64);
}
else if (cache.UseBarrett128)
{
@@ -711,8 +778,7 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
}
}
else
@@ -723,39 +789,34 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus);
}
}
accelerator.Synchronize();
stream.Synchronize();
}
private static void InverseDevice(Accelerator accelerator, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
private static void InverseDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> data, int length, GpuUInt128 modulus, SquareCacheEntry cache)
{
var bitKernel = GetBitReverseKernel(accelerator);
bitKernel(length, data, cache.Bits);
accelerator.Synchronize();
bitKernel(stream, length, data, cache.Bits);
if (cache.UseMontgomery64)
{
var toMont = GetToMont64Kernel(accelerator);
toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
accelerator.Synchronize();
toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64);
var stageKernel = GetStageMontKernel(accelerator);
int butterflies = length >> 1;
for (int s = 0; s < cache.Bits; s++)
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64);
}
var scaleKernel = GetScaleMont64Kernel(accelerator);
scaleKernel(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64);
accelerator.Synchronize();
scaleKernel(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64);
var fromMont = GetFromMont64Kernel(accelerator);
fromMont(length, data, cache.ModulusLow, cache.MontNPrime64);
fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64);
}
else if (cache.UseBarrett128)
{
@@ -765,12 +826,10 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
}
var scaleKernel = GetScaleBarrett128Kernel(accelerator);
scaleKernel(length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
accelerator.Synchronize();
scaleKernel(stream, length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow);
}
else
{
@@ -780,15 +839,13 @@ public static class NttGpuMath
{
int len = 1 << (s + 1);
int half = len >> 1;
stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus);
accelerator.Synchronize();
stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus);
}
var scaleKernel = GetScaleKernel(accelerator);
scaleKernel(length, data, cache.NInv, modulus);
accelerator.Synchronize();
scaleKernel(stream, length, data, cache.NInv, modulus);
}
accelerator.Synchronize();
stream.Synchronize();
}
public static void Convolve(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
@@ -871,19 +928,20 @@ public static class NttGpuMath
public static void SquareGpu(Span<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) =>
ConvolveGpu(values, values, modulus, primitiveRoot);
public static void SquareDevice(Accelerator accelerator, ArrayView<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
public static void SquareDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView<GpuUInt128> values, GpuUInt128 modulus, GpuUInt128 primitiveRoot)
{
var cache = GetSquareCache(accelerator, (int)values.Length, modulus, primitiveRoot);
ForwardDevice(accelerator, values, (int)values.Length, modulus, cache);
var cache = GetSquareCache(accelerator, stream, (int)values.Length, modulus, primitiveRoot);
ForwardDevice(accelerator, stream, values, (int)values.Length, modulus, cache);
var mul = GetMulKernel(accelerator);
mul((int)values.Length, values, values, modulus);
InverseDevice(accelerator, values, (int)values.Length, modulus, cache);
mul(stream, (int)values.Length, values, values, modulus);
InverseDevice(accelerator, stream, values, (int)values.Length, modulus, cache);
}
public static void PointwiseMultiply(Span<GpuUInt128> left, Span<GpuUInt128> right, GpuUInt128 modulus)
{
var gpu = GpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = accelerator.CreateStream();
var kernel = GetMulKernel(accelerator);
var pool = ArrayPool<GpuUInt128>.Shared;
var leftArray = pool.Rent(left.Length);
@@ -892,18 +950,19 @@ public static class NttGpuMath
right.CopyTo(rightArray);
var leftBuffer = accelerator.Allocate1D<GpuUInt128>(left.Length);
var rightBuffer = accelerator.Allocate1D<GpuUInt128>(right.Length);
leftBuffer.View.CopyFromCPU(ref leftArray[0], left.Length);
rightBuffer.View.CopyFromCPU(ref rightArray[0], right.Length);
kernel(left.Length, leftBuffer.View, rightBuffer.View, modulus);
accelerator.Synchronize();
leftBuffer.View.CopyToCPU(ref leftArray[0], left.Length);
rightBuffer.View.CopyToCPU(ref rightArray[0], right.Length);
leftBuffer.View.CopyFromCPU(stream, ref leftArray[0], left.Length);
rightBuffer.View.CopyFromCPU(stream, ref rightArray[0], right.Length);
kernel(stream, left.Length, leftBuffer.View, rightBuffer.View, modulus);
leftBuffer.View.CopyToCPU(stream, ref leftArray[0], left.Length);
rightBuffer.View.CopyToCPU(stream, ref rightArray[0], right.Length);
stream.Synchronize();
leftArray.AsSpan(0, left.Length).CopyTo(left);
rightArray.AsSpan(0, right.Length).CopyTo(right);
pool.Return(leftArray, clearArray: true);
pool.Return(rightArray, clearArray: true);
rightBuffer.Dispose();
leftBuffer.Dispose();
stream.Dispose();
gpu.Dispose();
}
PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs
+45 -45
View File
@@ -18,9 +18,9 @@ internal enum GpuPow2ModStatus
internal static partial class PrimeOrderGpuHeuristics
{
internal static void PreloadStaticTables(Accelerator accelerator)
internal static void PreloadStaticTables(Accelerator accelerator, AcceleratorStream stream)
{
_ = GetSmallPrimeDeviceCache(accelerator);
_ = GetSmallPrimeDeviceCache(accelerator, stream);
}
private static readonly ConcurrentDictionary<ulong, byte> OverflowedPrimes = new();
@@ -106,16 +106,16 @@ internal static partial class PrimeOrderGpuHeuristics
s_capability = PrimeOrderGpuCapability.Default;
}
private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator)
private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator, AcceleratorStream stream)
{
return SmallPrimeDeviceCaches.GetOrAdd(accelerator, static acc =>
return SmallPrimeDeviceCaches.GetOrAdd(accelerator, acc =>
{
uint[] primes = PrimesGenerator.SmallPrimes;
ulong[] squares = PrimesGenerator.SmallPrimesPow2;
var primeBuffer = acc.Allocate1D<uint>(primes.Length);
primeBuffer.View.CopyFromCPU(primes);
primeBuffer.View.CopyFromCPU(stream, primes);
var squareBuffer = acc.Allocate1D<ulong>(squares.Length);
squareBuffer.View.CopyFromCPU(squares);
squareBuffer.View.CopyFromCPU(stream, squares);
return new SmallPrimeDeviceCache
{
Primes = primeBuffer,
@@ -203,7 +203,7 @@ internal static partial class PrimeOrderGpuHeuristics
AcceleratorStream stream = lease.Stream;
var kernel = GetPartialFactorKernel(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream);
// TODO: We should create / reallocate these buffer only once or if the new required length is bigger than capacity.
var factorBuffer = accelerator.Allocate1D<ulong>(primeTargets.Length);
@@ -213,11 +213,11 @@ internal static partial class PrimeOrderGpuHeuristics
var fullyFactoredBuffer = accelerator.Allocate1D<byte>(1);
// TODO: There is no need to clear these buffers because the kernel will always assign values within the required bounds.
// factorBuffer.MemSetToZero();
// exponentBuffer.MemSetToZero();
// countBuffer.MemSetToZero();
// remainingBuffer.MemSetToZero();
// fullyFactoredBuffer.MemSetToZero();
// factorBuffer.MemSetToZero(stream);
// exponentBuffer.MemSetToZero(stream);
// countBuffer.MemSetToZero(stream);
// remainingBuffer.MemSetToZero(stream);
// fullyFactoredBuffer.MemSetToZero(stream);
kernel(
stream,
@@ -234,17 +234,18 @@ internal static partial class PrimeOrderGpuHeuristics
remainingBuffer.View,
fullyFactoredBuffer.View);
stream.Synchronize();
countBuffer.View.CopyToCPU(ref factorCount, 1);
factorCount = Math.Min(factorCount, primeTargets.Length);
factorBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length);
exponentBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length);
remainingBuffer.View.CopyToCPU(ref remaining, 1);
countBuffer.View.CopyToCPU(stream, ref factorCount, 1);
factorBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length);
exponentBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length);
remainingBuffer.View.CopyToCPU(stream, ref remaining, 1);
byte fullyFactoredFlag = 0;
fullyFactoredBuffer.View.CopyToCPU(ref fullyFactoredFlag, 1);
fullyFactored = fullyFactoredFlag != 0;
fullyFactoredBuffer.View.CopyToCPU(stream, ref fullyFactoredFlag, 1);
stream.Synchronize();
fullyFactored = fullyFactoredFlag != 0;
factorCount = Math.Min(factorCount, primeTargets.Length);
// TODO: These buffers shouldn't be disposed but rather reused accross call with the assigned accelerator.
factorBuffer.Dispose();
@@ -369,7 +370,7 @@ internal static partial class PrimeOrderGpuHeuristics
AcceleratorStream stream = lease.Stream;
var kernel = GetOrderKernel(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator);
SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream);
// TODO: These buffers should be created reallocated once and assigned to an accelerator. Callers should use their own
// thread static cache to prevent other threads from using taking accelerators with pre-allocated buffers if they don't
@@ -386,16 +387,16 @@ internal static partial class PrimeOrderGpuHeuristics
var statusBuffer = accelerator.Allocate1D<byte>(1);
// TODO: Remove the cleaning after the order kernel is modified to always set the result.
phiFactorBuffer.MemSetToZero();
phiExponentBuffer.MemSetToZero();
workFactorBuffer.MemSetToZero();
workExponentBuffer.MemSetToZero();
candidateBuffer.MemSetToZero();
stackIndexBuffer.MemSetToZero();
stackExponentBuffer.MemSetToZero();
stackProductBuffer.MemSetToZero();
resultBuffer.MemSetToZero();
statusBuffer.MemSetToZero();
phiFactorBuffer.MemSetToZero(stream);
phiExponentBuffer.MemSetToZero(stream);
workFactorBuffer.MemSetToZero(stream);
workExponentBuffer.MemSetToZero(stream);
candidateBuffer.MemSetToZero(stream);
stackIndexBuffer.MemSetToZero(stream);
stackExponentBuffer.MemSetToZero(stream);
stackProductBuffer.MemSetToZero(stream);
resultBuffer.MemSetToZero(stream);
statusBuffer.MemSetToZero(stream);
uint limit = config.SmallFactorLimit == 0 ? uint.MaxValue : config.SmallFactorLimit;
byte hasPrevious = previousOrder.HasValue ? (byte)1 : (byte)0;
@@ -425,10 +426,11 @@ internal static partial class PrimeOrderGpuHeuristics
cache.Count,
buffers);
stream.Synchronize();
byte status = 0;
statusBuffer.View.CopyToCPU(ref status, 1);
statusBuffer.View.CopyToCPU(stream, ref status, 1);
resultBuffer.View.CopyToCPU(stream, ref order, 1);
stream.Synchronize();
PrimeOrderKernelStatus kernelStatus = (PrimeOrderKernelStatus)status;
if (kernelStatus == PrimeOrderKernelStatus.Fallback)
@@ -445,8 +447,6 @@ internal static partial class PrimeOrderGpuHeuristics
throw new InvalidOperationException("GPU Pollard Rho stack overflow; increase HeuristicStackCapacity.");
}
resultBuffer.View.CopyToCPU(ref order, 1);
if (kernelStatus == PrimeOrderKernelStatus.FactoringFailure)
{
order = 0UL;
@@ -536,15 +536,19 @@ internal static partial class PrimeOrderGpuHeuristics
var exponentBuffer = accelerator.Allocate1D<ulong>(exponents.Length);
var remainderBuffer = accelerator.Allocate1D<ulong>(exponents.Length);
exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponents), exponents.Length);
exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponents), exponents.Length);
// Pow2Mod kernel always assigns the required output elements so we don't need to worry about clearing these.
// remainderBuffer.MemSetToZero();
// remainderBuffer.MemSetToZero(stream);
try
{
kernel(stream, exponents.Length, exponentBuffer.View, divisorData, remainderBuffer.View);
remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(results), exponents.Length);
stream.Synchronize();
exponentBuffer.Dispose();
remainderBuffer.Dispose();
lease.Dispose();
}
catch (Exception)
{
@@ -555,10 +559,6 @@ internal static partial class PrimeOrderGpuHeuristics
throw;
}
remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(results), exponents.Length);
exponentBuffer.Dispose();
remainderBuffer.Dispose();
lease.Dispose();
return true;
}
@@ -603,20 +603,20 @@ internal static partial class PrimeOrderGpuHeuristics
exponentSpan[i] = (GpuUInt128)exponents[i];
}
exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponentSpan), length);
exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponentSpan), length);
// Pow2ModWide kernel always assigns the required output elements so we don't need to worry about clearing these.
// remainderBuffer.MemSetToZero();
// remainderBuffer.MemSetToZero(stream);
GpuUInt128 modulus = (GpuUInt128)prime;
kernel(stream, length, exponentBuffer.View, modulus, remainderBuffer.View);
stream.Synchronize();
Span<GpuUInt128> resultSpan = length <= WideStackThreshold
? stackalloc GpuUInt128[length]
: new Span<GpuUInt128>(rentedResults = ArrayPool<GpuUInt128>.Shared.Rent(length), 0, length);
remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(resultSpan), length);
remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length);
stream.Synchronize();
for (int i = 0; i < length; i++)
{
PerfectNumbers.Core/Gpu/SharedGpuContext.cs
+2 -1
View File
@@ -7,11 +7,12 @@ namespace PerfectNumbers.Core.Gpu;
public static class SharedGpuContext
{
// TODO: Remove Lazy<T> in favor for better performance with nullable fields.
private static readonly Lazy<Context> s_context = new(() => ILGPU.Context.CreateDefault(), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy<Device> s_device = new(() => s_context.Value.GetPreferredDevice(false), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy<Accelerator> s_accelerator = new(() => s_device.Value.CreateAccelerator(s_context.Value), LazyThreadSafetyMode.ExecutionAndPublication);
internal static Context Context => s_context.Value;
internal static Context Context => s_context.Value;
internal static Device Device => s_device.Value;
PerfectNumbers.Core/HeuristicCombinedPrimeTester.cs
+3 -2
View File
@@ -237,17 +237,18 @@ public sealed class HeuristicCombinedPrimeTester
int divisorLength = GetCombinedDivisors(nMod10).Length;
int compositeFlag = 0;
flagView1D.CopyFromCPU(ref compositeFlag, 1);
flagView1D.CopyFromCPU(stream, ref compositeFlag, 1);
kernel(
stream,
divisorLength,
flagView,
n,
maxDivisorSquare,
HeuristicGpuDivisorTableKind.Combined,
gpu.HeuristicGpuTables);
flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
stream.Synchronize();
flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
compositeDetected = compositeFlag != 0;
gpu.Dispose();
PerfectNumbers.Core/HeuristicPrimeTester.cs
+8 -5
View File
@@ -253,17 +253,19 @@ public sealed class HeuristicPrimeTester
int groupALength = HeuristicPrimeSieves.GroupADivisors.Length;
flagView1D.CopyFromCPU(ref compositeFlag, 1);
flagView1D.CopyFromCPU(stream, ref compositeFlag, 1);
kernel(
stream,
groupALength,
flagView,
n,
maxDivisorSquare,
HeuristicGpuDivisorTableKind.GroupA,
gpu.HeuristicGpuTables);
stream.Synchronize();
flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
stream.Synchronize();
compositeDetected = compositeFlag != 0;
if (!compositeDetected)
@@ -288,16 +290,17 @@ public sealed class HeuristicPrimeTester
int divisorLength = GetGroupBDivisors(ending).Length;
compositeFlag = 0;
flagView1D.CopyFromCPU(ref compositeFlag, 1);
flagView1D.CopyFromCPU(stream, ref compositeFlag, 1);
kernel(
stream,
divisorLength,
flagView,
n,
maxDivisorSquare,
tableKind,
gpu.HeuristicGpuTables);
accelerator.Synchronize();
flagView1D.CopyToCPU(ref compositeFlag, 1);
flagView1D.CopyToCPU(stream, ref compositeFlag, 1);
stream.Synchronize();
compositeDetected = compositeFlag != 0;
}
}
PerfectNumbers.Core/MersenneDivisorCycles.cs
+7 -4
View File
@@ -7,6 +7,7 @@ using System.Text;
using System.Runtime.InteropServices;
using PerfectNumbers.Core.Gpu;
using ILGPU;
using ILGPU.Algorithms;
using ILGPU.Runtime;
namespace PerfectNumbers.Core;
@@ -806,10 +807,12 @@ public class MersenneDivisorCycles
var lease = GpuKernelPool.GetKernel();
var accelerator = lease.Accelerator;
var stream = lease.Stream;
var kernel = accelerator.LoadAutoGroupedStreamKernel<
var loaded = accelerator.LoadAutoGroupedStreamKernel<
Index1D,
ArrayView1D<ulong, Stride1D.Dense>,
ArrayView1D<ulong, Stride1D.Dense>>(DivisorCycleKernels.GpuDivisorCycleKernel);
var kernel = KernelUtil.GetKernel(loaded);
var launcher = kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>>();
using Stream outputStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, BufferSize10M, useAsync: true);
@@ -860,14 +863,15 @@ public class MersenneDivisorCycles
var bufferDiv = accelerator.Allocate1D(validDivisors);
var bufferCycle = accelerator.Allocate1D<ulong>(idx);
kernel(
launcher(
stream,
idx,
bufferDiv.View,
bufferCycle.View);
bufferCycle.View.CopyToCPU(stream, ref outCycles[0], idx);
stream.Synchronize();
bufferCycle.View.CopyToCPU(stream, ref outCycles[0], idx);
bufferCycle.Dispose();
bufferDiv.Dispose();
@@ -893,7 +897,6 @@ public class MersenneDivisorCycles
pool.Return(divisors, clearArray: false);
pool.Return(outCycles, clearArray: false);
stream.Dispose();
lease.Dispose();
}
PerfectNumbers.Core/MersenneNumberTester.cs
+149 -148
View File
@@ -16,27 +16,27 @@ public enum GpuKernelType
}
public sealed class MersenneNumberTester(
bool useIncremental = true,
bool useOrderCache = false,
GpuKernelType kernelType = GpuKernelType.Incremental,
bool useOrder = false,
bool useGpuLucas = true,
bool useGpuScan = true,
bool useGpuOrder = true,
bool useResidue = true,
ulong maxK = 5_000_000UL)
bool useIncremental = true,
bool useOrderCache = false,
GpuKernelType kernelType = GpuKernelType.Incremental,
bool useOrder = false,
bool useGpuLucas = true,
bool useGpuScan = true,
bool useGpuOrder = true,
bool useResidue = true,
ulong maxK = 5_000_000UL)
{
private readonly bool _useResidue = useResidue;
private readonly bool _useIncremental = useIncremental && !useResidue;
private readonly ulong _maxK = maxK;
private readonly bool _useResidue = useResidue;
private readonly bool _useIncremental = useIncremental && !useResidue;
private readonly ulong _maxK = maxK;
private readonly GpuKernelType _kernelType = kernelType;
private readonly bool _useGpuLucas = useGpuLucas;
private readonly bool _useGpuScan = useGpuScan; // device for pow2mod/incremental scanning
private readonly bool _useGpuOrder = useGpuOrder; // device for order computations
private static readonly ConcurrentDictionary<UInt128, ulong> OrderCache = new();
// TODO: Swap this ConcurrentDictionary for the pooled dictionary variant highlighted in
// Pow2MontgomeryModBenchmarks once order warmups reuse deterministic divisor-cycle snapshots; the pooled approach removed
// the locking overhead when scanning p >= 138M in those benchmarks.
private readonly bool _useGpuLucas = useGpuLucas;
private readonly bool _useGpuScan = useGpuScan; // device for pow2mod/incremental scanning
private readonly bool _useGpuOrder = useGpuOrder; // device for order computations
private static readonly ConcurrentDictionary<UInt128, ulong> OrderCache = new();
// TODO: Swap this ConcurrentDictionary for the pooled dictionary variant highlighted in
// Pow2MontgomeryModBenchmarks once order warmups reuse deterministic divisor-cycle snapshots; the pooled approach removed
// the locking overhead when scanning p >= 138M in those benchmarks.
private readonly MersenneNumberIncrementalGpuTester _incrementalGpuTester = new(kernelType, useGpuOrder);
private readonly MersenneNumberIncrementalCpuTester _incrementalCpuTester = new(kernelType);
private readonly MersenneNumberOrderGpuTester _orderGpuTester = new(kernelType, useGpuOrder);
@@ -49,7 +49,7 @@ public sealed class MersenneNumberTester(
// TODO: Ensure Program passes useResidue correctly and that residue-vs-incremental-vs-LL selection respects CLI flags.
// Also consider injecting a shared cycles cache (MersenneDivisorCycles.Shared) to both CPU and GPU testers.
public void WarmUpOrders(ulong exponent, ulong limit = 5_000_000UL)
public void WarmUpOrders(ulong exponent, ulong limit = 5_000_000UL)
{
bool cacheEnabled = useOrderCache;
if (!cacheEnabled)
@@ -57,34 +57,34 @@ public sealed class MersenneNumberTester(
return;
}
UInt128 twoP = (UInt128)exponent << 1; // 2 * p
LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One;
// CPU path: compute orders directly and cache
if (!_useGpuOrder)
{
UInt128 k = UInt128.One;
UInt128 twoP = (UInt128)exponent << 1; // 2 * p
LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One;
// CPU path: compute orders directly and cache
if (!_useGpuOrder)
{
UInt128 k = UInt128.One;
UInt128 q;
ulong remainder;
for (; k <= limit; k++)
{
q = twoP * k + UInt128.One;
remainder = q.Mod10();
// TODO: Replace these repeated Mod10/Mod8/Mod3/Mod5 calls with a residue automaton walk so
// order warmups reuse the optimized cycle stepping validated in the residue benchmarks.
bool shouldCheck = lastDigit == LastDigit.Seven
? remainder == 7UL || remainder == 9UL || remainder == 3UL
: remainder == 1UL || remainder == 3UL || remainder == 9UL;
{
q = twoP * k + UInt128.One;
remainder = q.Mod10();
// TODO: Replace these repeated Mod10/Mod8/Mod3/Mod5 calls with a residue automaton walk so
// order warmups reuse the optimized cycle stepping validated in the residue benchmarks.
bool shouldCheck = lastDigit == LastDigit.Seven
? remainder == 7UL || remainder == 9UL || remainder == 3UL
: remainder == 1UL || remainder == 3UL || remainder == 9UL;
// Early modular filters for odd order and small-prime rejections
// Early modular filters for odd order and small-prime rejections
if (shouldCheck)
{
ulong mod8 = q.Mod8();
if (mod8 != 1UL && mod8 != 7UL)
{
continue;
}
else if (q.Mod3() == 0UL || q.Mod5() == 0UL)
{
ulong mod8 = q.Mod8();
if (mod8 != 1UL && mod8 != 7UL)
{
continue;
}
else if (q.Mod3() == 0UL || q.Mod5() == 0UL)
{
continue;
}
}
@@ -94,9 +94,9 @@ public sealed class MersenneNumberTester(
continue;
}
ulong ord = q.CalculateOrder();
OrderCache[q] = ord;
}
ulong ord = q.CalculateOrder();
OrderCache[q] = ord;
}
return;
}
@@ -108,23 +108,23 @@ public sealed class MersenneNumberTester(
ulong remainder2;
for (; k2 <= limit; k2++)
{
q2 = twoP * k2 + 1UL;
remainder2 = q2.Mod10();
// TODO: Reuse the same residue-automaton fast path here so the GPU warmup staging avoids `%` and
// branches the benchmarks showed slower than the tracked stepping helper.
bool shouldCheck2 = lastDigit == LastDigit.Seven
? remainder2 == 7UL || remainder2 == 9UL || remainder2 == 3UL
: remainder2 == 1UL || remainder2 == 3UL || remainder2 == 9UL;
q2 = twoP * k2 + 1UL;
remainder2 = q2.Mod10();
// TODO: Reuse the same residue-automaton fast path here so the GPU warmup staging avoids `%` and
// branches the benchmarks showed slower than the tracked stepping helper.
bool shouldCheck2 = lastDigit == LastDigit.Seven
? remainder2 == 7UL || remainder2 == 9UL || remainder2 == 3UL
: remainder2 == 1UL || remainder2 == 3UL || remainder2 == 9UL;
// Early modular filters for odd order and small-prime rejections
// Early modular filters for odd order and small-prime rejections
if (shouldCheck2)
{
ulong mod8 = q2.Mod8();
ulong mod8 = q2.Mod8();
if (mod8 != 1UL && mod8 != 7UL)
{
shouldCheck2 = false;
}
else if (q2.Mod3() == 0UL || q2.Mod5() == 0UL)
else if (q2.Mod3() == 0UL || q2.Mod5() == 0UL)
{
shouldCheck2 = false;
}
@@ -146,104 +146,105 @@ public sealed class MersenneNumberTester(
UInt128[] qs = qsBuffer;
var gpuLease = GpuKernelPool.GetKernel();
var accelerator = gpuLease.Accelerator;
var stream = gpuLease.Stream;
var orderKernel = gpuLease.OrderKernel!;
var gpuLease = GpuKernelPool.GetKernel();
var accelerator = gpuLease.Accelerator;
var stream = gpuLease.Stream;
var orderKernel = gpuLease.OrderKernel!;
// Guard long-running kernels by chunking the warm-up across batches.
// Reuse the same ScanBatchSize knob used for scanning.
int batchSize = GpuConstants.ScanBatchSize;
ulong divMul = (ulong)((((UInt128)1 << 64) - 1UL) / exponent) + 1UL;
int offset = 0;
while (offset < idx)
{
int count = Math.Min(batchSize, idx - offset);
var qBuffer = accelerator.Allocate1D<PerfectNumbers.Core.Gpu.GpuUInt128>(count);
var orderBuffer = accelerator.Allocate1D<ulong>(count);
// Convert to device-friendly GpuUInt128 on the fly
var tmp = System.Buffers.ArrayPool<PerfectNumbers.Core.Gpu.GpuUInt128>.Shared.Rent(count);
for (int i = 0; i < count; i++)
{
tmp[i] = (PerfectNumbers.Core.Gpu.GpuUInt128)qs[offset + i];
}
// Copy only the populated elements to the device buffer to avoid overrun on pooled arrays.
qBuffer.View.CopyFromCPU(ref tmp[0], count);
System.Buffers.ArrayPool<PerfectNumbers.Core.Gpu.GpuUInt128>.Shared.Return(tmp);
orderKernel(stream, count, exponent, divMul, qBuffer.View, orderBuffer.View);
stream.Synchronize();
// Guard long-running kernels by chunking the warm-up across batches.
// Reuse the same ScanBatchSize knob used for scanning.
int batchSize = GpuConstants.ScanBatchSize;
ulong divMul = (ulong)((((UInt128)1 << 64) - 1UL) / exponent) + 1UL;
int offset = 0;
while (offset < idx)
{
int count = Math.Min(batchSize, idx - offset);
var qBuffer = accelerator.Allocate1D<PerfectNumbers.Core.Gpu.GpuUInt128>(count);
var orderBuffer = accelerator.Allocate1D<ulong>(count);
// Convert to device-friendly GpuUInt128 on the fly
var tmp = System.Buffers.ArrayPool<PerfectNumbers.Core.Gpu.GpuUInt128>.Shared.Rent(count);
for (int i = 0; i < count; i++)
{
tmp[i] = (PerfectNumbers.Core.Gpu.GpuUInt128)qs[offset + i];
}
// Copy only the populated elements to the device buffer to avoid overrun on pooled arrays.
qBuffer.View.CopyFromCPU(stream, ref tmp[0], count);
System.Buffers.ArrayPool<PerfectNumbers.Core.Gpu.GpuUInt128>.Shared.Return(tmp);
orderKernel(stream, count, exponent, divMul, qBuffer.View, orderBuffer.View);
stream.Synchronize();
ulong[] orders = orderBuffer.GetAsArray1D();
for (int i = 0; i < count; i++)
{
ulong order = orders[i];
if (order == 0UL)
{
order = qs[offset + i].CalculateOrder();
}
// TODO: Change this to .CopyToCPU and use pre-allocated buffer with fixed, max capacity, using stream. Synchronize on the stream after copy.
ulong[] orders = orderBuffer.GetAsArray1D();
for (int i = 0; i < count; i++)
{
ulong order = orders[i];
if (order == 0UL)
{
order = qs[offset + i].CalculateOrder();
}
if (useOrderCache)
{
OrderCache[qs[offset + i]] = order;
}
}
if (useOrderCache)
{
OrderCache[qs[offset + i]] = order;
}
}
offset += count;
orderBuffer.Dispose();
qBuffer.Dispose();
}
offset += count;
orderBuffer.Dispose();
qBuffer.Dispose();
}
gpuLease.Dispose();
ArrayPool<UInt128>.Shared.Return(qs);
}
gpuLease.Dispose();
ArrayPool<UInt128>.Shared.Return(qs);
}
public bool IsMersennePrime(ulong exponent)
{
// Safe early rejections using known orders for tiny primes:
// 7 | M_p iff 3 | p. Avoid rejecting p == 3 where M_3 == 7 is prime.
if ((exponent % 3UL) == 0UL && exponent != 3UL)
{
// TODO: Replace this `% 3` check with ULongExtensions.Mod3 to align the early rejection with
// the benchmarked bitmask helper instead of generic modulo for CPU workloads.
return false;
}
public bool IsMersennePrime(ulong exponent)
{
// Safe early rejections using known orders for tiny primes:
// 7 | M_p iff 3 | p. Avoid rejecting p == 3 where M_3 == 7 is prime.
if ((exponent % 3UL) == 0UL && exponent != 3UL)
{
// TODO: Replace this `% 3` check with ULongExtensions.Mod3 to align the early rejection with
// the benchmarked bitmask helper instead of generic modulo for CPU workloads.
return false;
}
if ((exponent & 3UL) == 1UL && exponent.SharesFactorWithExponentMinusOne())
{
return false;
}
if ((exponent & 3UL) == 1UL && exponent.SharesFactorWithExponentMinusOne())
{
return false;
}
// Skip even exponents: for even p > 2, M_p is composite; here p is large.
if ((exponent & 1UL) == 0UL)
{
return false;
}
// Skip even exponents: for even p > 2, M_p is composite; here p is large.
if ((exponent & 1UL) == 0UL)
{
return false;
}
bool prePrime = true;
UInt128 twoP = (UInt128)exponent << 1; // 2 * p
// UInt128 maxDivisor = new(ulong.MaxValue, ulong.MaxValue);
UInt128 maxK = UInt128Numbers.Two.Pow(exponent) / 2 + 1;
LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One;
UInt128 twoP = (UInt128)exponent << 1; // 2 * p
// UInt128 maxDivisor = new(ulong.MaxValue, ulong.MaxValue);
UInt128 maxK = UInt128Numbers.Two.Pow(exponent) / 2 + 1;
LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One;
// In residue mode the residue scan is the final primality check.
if (_useResidue)
{
if (_maxK < maxK)
{
maxK = _maxK;
}
// In residue mode the residue scan is the final primality check.
if (_useResidue)
{
if (_maxK < maxK)
{
maxK = _maxK;
}
if (_useGpuScan)
{
_residueGpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime);
}
else
{
_residueCpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime);
}
if (_useGpuScan)
{
_residueGpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime);
}
else
{
_residueCpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime);
}
return prePrime;
}
return prePrime;
}
if (!_useIncremental)
{
@@ -296,8 +297,8 @@ public sealed class MersenneNumberTester(
}
}
return isPrime;
}
return isPrime;
}
// Limit of k to scan in the pre-Lucas prescan
public const int PreLucasPreScanK = 5_000_000;
@@ -309,12 +310,12 @@ public sealed class MersenneNumberTester(
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LegacyOrderKernel(Index1D index, ulong exponent, ArrayView<UInt128> qs, ArrayView<ulong> orders)
{
UInt128 q = qs[index];
UInt128 pow = exponent.PowMod(q);
orders[index] = pow == 1UL ? exponent : 0UL;
}
private static void LegacyOrderKernel(Index1D index, ulong exponent, ArrayView<UInt128> qs, ArrayView<ulong> orders)
{
UInt128 q = qs[index];
UInt128 pow = exponent.PowMod(q);
orders[index] = pow == 1UL ? exponent : 0UL;
}
}
PerfectNumbers.Core/PrimeOrderCalculator.Gpu.cs
+5 -3
View File
@@ -26,7 +26,7 @@ internal static partial class PrimeOrderCalculator
ScratchBuffer scratch = GpuScratchBufferPool.Rent(accelerator, GpuSmallPrimeFactorSlots, 0);
SmallPrimeFactorTables tables = GpuKernelPool.EnsureSmallPrimeFactorTables(kernels, accelerator);
SmallPrimeFactorTables tables = GpuKernelPool.EnsureSmallPrimeFactorTables(kernels, accelerator, stream);
if (kernels.SmallPrimeFactorsPrimes is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsPrimes)} is null");
if (kernels.SmallPrimeFactorsSquares is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsSquares)} is null");
@@ -52,10 +52,11 @@ internal static partial class PrimeOrderCalculator
smallPrimeFactorCountSlotView,
smallPrimeFactorRemainingSlotView);
stream.Synchronize();
factorCount = 0;
smallPrimeFactorCountSlotView.CopyToCPU(stream, ref factorCount, 1);
stream.Synchronize();
factorCount = Math.Min(factorCount, GpuSmallPrimeFactorSlots);
if (factorCount > 0)
@@ -66,6 +67,7 @@ internal static partial class PrimeOrderCalculator
}
smallPrimeFactorRemainingSlotView.CopyToCPU(stream, ref remaining, 1);
stream.Synchronize();
for (int i = 0; i < factorCount; i++)
{
@@ -121,11 +123,11 @@ internal static partial class PrimeOrderCalculator
divisorData,
scratch.SpecialMaxCandidates.View,
specialMaxResultView);
stream.Synchronize();
Span<ulong> result = stackalloc ulong[1];
// specialMaxResultView.CopyToCPU(stream, ref result, 1);
specialMaxResultView.CopyToCPU(stream, in result);
stream.Synchronize();
// stream.Synchronize();
// return result != 0;
PerfectNumbers.Core/PrimeTester.cs
+96 -101
View File
@@ -111,6 +111,7 @@ public sealed class PrimeTester
inputView.CopyFromCPU(stream, ref value, 1);
kernel(
stream,
1,
inputView,
gpu.DevicePrimesDefault.View,
@@ -124,8 +125,8 @@ public sealed class PrimeTester
gpu.DevicePrimesPow2LastThree.View,
gpu.DevicePrimesPow2LastNine.View,
outputView);
stream.Synchronize();
outputView.CopyToCPU(stream, ref flag, 1);
stream.Synchronize();
gpu.Dispose();
GpuPrimeWorkLimiter.Release();
@@ -182,9 +183,10 @@ public sealed class PrimeTester
var valueSlice = values.Slice(pos, count);
ref ulong valueRef = ref MemoryMarshal.GetReference(valueSlice);
inputView.CopyFromCPU(ref valueRef, count);
inputView.CopyFromCPU(stream, ref valueRef, count);
kernel(
stream,
count,
inputView,
gpu.DevicePrimesDefault.View,
@@ -198,8 +200,7 @@ public sealed class PrimeTester
gpu.DevicePrimesPow2LastThree.View,
gpu.DevicePrimesPow2LastNine.View,
outputView);
stream.Synchronize();
var resultSlice = results.Slice(pos, count);
ref byte resultRef = ref MemoryMarshal.GetReference(resultSlice);
outputView.CopyToCPU(stream, ref resultRef, count);
@@ -207,6 +208,7 @@ public sealed class PrimeTester
pos += count;
}
stream.Synchronize();
gpu.Dispose();
GpuPrimeWorkLimiter.Release();
}
@@ -219,13 +221,13 @@ public sealed class PrimeTester
private static readonly object SharedTablesLock = new();
private static readonly Dictionary<Accelerator, SharedHeuristicGpuTables> SharedTables = new(AcceleratorReferenceComparer.Instance);
private static SharedHeuristicGpuTables RentSharedTables(Accelerator accelerator)
private static SharedHeuristicGpuTables RentSharedTables(Accelerator accelerator, AcceleratorStream stream)
{
lock (SharedTablesLock)
{
if (!SharedTables.TryGetValue(accelerator, out var tables))
{
tables = new SharedHeuristicGpuTables(accelerator);
tables = new SharedHeuristicGpuTables(accelerator, stream);
SharedTables.Add(accelerator, tables);
}
@@ -247,9 +249,9 @@ public sealed class PrimeTester
}
}
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator)
internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream)
{
var tables = RentSharedTables(accelerator);
var tables = RentSharedTables(accelerator, stream);
var heuristicTables = tables.CreateHeuristicDivisorTables();
HeuristicGpuDivisorTables.InitializeShared(in heuristicTables);
ReleaseSharedTables(tables);
@@ -270,38 +272,40 @@ public sealed class PrimeTester
}
Accelerator accelerator = SharedGpuContext.Accelerator;
KernelContainer kernels = GpuKernelPool.GetKernels(accelerator);
GpuStaticTableInitializer.EnsureStaticTables(kernels, accelerator);
AcceleratorStream stream = accelerator.CreateStream();
KernelContainer kernels = GpuKernelPool.GetKernels(accelerator, stream);
GpuStaticTableInitializer.EnsureStaticTables(kernels, accelerator, stream);
int toCreate = target - WarmedLeaseCount;
for (int i = 0; i < toCreate; i++)
{
Pool.Enqueue(new PrimeTesterGpuContextLease(minBufferCapacity));
Pool.Enqueue(new PrimeTesterGpuContextLease(minBufferCapacity, stream));
}
stream.Synchronize();
stream.Dispose();
WarmedLeaseCount = target;
}
}
internal static PrimeTesterGpuContextLease Rent(int minBufferCapacity = 1)
{
if (!Pool.TryDequeue(out var lease))
{
lease = new PrimeTesterGpuContextLease(minBufferCapacity);
}
else
{
lease.EnsureCapacity(minBufferCapacity);
}
internal static PrimeTesterGpuContextLease Rent(int minBufferCapacity = 1)
{
if (!Pool.TryDequeue(out var lease))
{
lease = new PrimeTesterGpuContextLease(minBufferCapacity);
}
else
{
lease.EnsureCapacity(minBufferCapacity);
}
lease.ResetReturnFlag();
return lease;
}
return lease;
}
internal static void Return(PrimeTesterGpuContextLease lease)
{
lease.Accelerator.Synchronize();
Pool.Enqueue(lease);
}
@@ -373,35 +377,23 @@ public sealed class PrimeTester
}
}
internal sealed class PrimeTesterGpuContextLease : IDisposable
internal sealed class PrimeTesterGpuContextLease
{
private bool _returned;
private readonly SharedHeuristicGpuTables _sharedTables;
private readonly HeuristicGpuDivisorTables _heuristicDivisorTables;
internal readonly Context AcceleratorContext;
public readonly Accelerator Accelerator;
private AcceleratorStream? _stream;
private AcceleratorStream? _stream;
public AcceleratorStream Stream
{
get
{
if (_stream is { } stream)
{
return stream;
}
stream = Accelerator.CreateStream();
_stream = stream;
return stream;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _stream ?? Accelerator.CreateStream();
}
public static readonly Action<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>> Kernel = SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel);
public static readonly Action<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables> HeuristicTrialDivisionKernel = SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel);
public static readonly Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>> Kernel = KernelUtil.GetKernel(SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel)).CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>>();
public static readonly Action<AcceleratorStream, Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables> HeuristicTrialDivisionKernel = KernelUtil.GetKernel(SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel)).CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>>();
public MemoryBuffer1D<ulong, Stride1D.Dense> Input = null!;
public MemoryBuffer1D<byte, Stride1D.Dense> Output = null!;
public MemoryBuffer1D<int, Stride1D.Dense> HeuristicFlag = null!;
@@ -440,17 +432,34 @@ public sealed class PrimeTester
{
AcceleratorContext = SharedGpuContext.Context;
Accelerator = SharedGpuContext.Accelerator;
// GpuStaticTableInitializer.EnsureStaticTables(Accelerator);
// Kernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel);
// HeuristicTrialDivisionKernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel);
AcceleratorStream stream = SharedGpuContext.Accelerator.CreateStream();
var kernels = GpuKernelPool.GetKernels(Accelerator, stream);
GpuStaticTableInitializer.EnsureStaticTables(kernels, Accelerator, stream);
_sharedTables = RentSharedTables(Accelerator);
_sharedTables = RentSharedTables(Accelerator, stream);
_heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables();
HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables);
EnsureCapacity(minBufferCapacity);
stream.Synchronize();
stream.Dispose();
}
internal PrimeTesterGpuContextLease(int minBufferCapacity, AcceleratorStream stream)
{
AcceleratorContext = SharedGpuContext.Context;
Accelerator = SharedGpuContext.Accelerator;
// GpuStaticTableInitializer.EnsureStaticTables(Accelerator);
// Kernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<uint>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel);
// HeuristicTrialDivisionKernel = Accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<int>, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel);
_sharedTables = RentSharedTables(Accelerator, stream);
_heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables();
HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables);
EnsureCapacity(minBufferCapacity);
_returned = false;
}
public void EnsureCapacity(int minCapacity)
@@ -470,25 +479,8 @@ public sealed class PrimeTester
BufferCapacity = minCapacity;
}
internal void ResetReturnFlag()
{
_returned = false;
}
public void Dispose()
{
if (_returned)
{
return;
}
_returned = true;
if (_stream is { } stream)
{
stream.Dispose();
_stream = null;
}
Return(this);
}
@@ -497,6 +489,8 @@ public sealed class PrimeTester
Input?.Dispose();
Output?.Dispose();
HeuristicFlag?.Dispose();
_stream?.Dispose();
_stream = null;
ReleaseSharedTables(_sharedTables);
// These resources are shared between GPU leases
// Accelerator.Dispose();
@@ -523,86 +517,86 @@ public sealed class PrimeTester
{
private int _referenceCount;
internal SharedHeuristicGpuTables(Accelerator accelerator)
internal SharedHeuristicGpuTables(Accelerator accelerator, AcceleratorStream stream)
{
var primesDefault = DivisorGenerator.SmallPrimes;
DevicePrimesDefault = accelerator.Allocate1D<uint>(primesDefault.Length);
DevicePrimesDefault.View.CopyFromCPU(primesDefault);
DevicePrimesDefault.View.CopyFromCPU(stream, primesDefault);
var primesLastOne = DivisorGenerator.SmallPrimesLastOne;
DevicePrimesLastOne = accelerator.Allocate1D<uint>(primesLastOne.Length);
DevicePrimesLastOne.View.CopyFromCPU(primesLastOne);
DevicePrimesLastOne.View.CopyFromCPU(stream, primesLastOne);
var primesLastSeven = DivisorGenerator.SmallPrimesLastSeven;
DevicePrimesLastSeven = accelerator.Allocate1D<uint>(primesLastSeven.Length);
DevicePrimesLastSeven.View.CopyFromCPU(primesLastSeven);
DevicePrimesLastSeven.View.CopyFromCPU(stream, primesLastSeven);
var primesLastThree = DivisorGenerator.SmallPrimesLastThree;
DevicePrimesLastThree = accelerator.Allocate1D<uint>(primesLastThree.Length);
DevicePrimesLastThree.View.CopyFromCPU(primesLastThree);
DevicePrimesLastThree.View.CopyFromCPU(stream, primesLastThree);
var primesLastNine = DivisorGenerator.SmallPrimesLastNine;
DevicePrimesLastNine = accelerator.Allocate1D<uint>(primesLastNine.Length);
DevicePrimesLastNine.View.CopyFromCPU(primesLastNine);
DevicePrimesLastNine.View.CopyFromCPU(stream, primesLastNine);
var primesPow2Default = DivisorGenerator.SmallPrimesPow2;
DevicePrimesPow2Default = accelerator.Allocate1D<ulong>(primesPow2Default.Length);
DevicePrimesPow2Default.View.CopyFromCPU(primesPow2Default);
DevicePrimesPow2Default.View.CopyFromCPU(stream, primesPow2Default);
var primesPow2LastOne = DivisorGenerator.SmallPrimesPow2LastOne;
DevicePrimesPow2LastOne = accelerator.Allocate1D<ulong>(primesPow2LastOne.Length);
DevicePrimesPow2LastOne.View.CopyFromCPU(primesPow2LastOne);
DevicePrimesPow2LastOne.View.CopyFromCPU(stream, primesPow2LastOne);
var primesPow2LastSeven = DivisorGenerator.SmallPrimesPow2LastSeven;
DevicePrimesPow2LastSeven = accelerator.Allocate1D<ulong>(primesPow2LastSeven.Length);
DevicePrimesPow2LastSeven.View.CopyFromCPU(primesPow2LastSeven);
DevicePrimesPow2LastSeven.View.CopyFromCPU(stream, primesPow2LastSeven);
var primesPow2LastThree = DivisorGenerator.SmallPrimesPow2LastThree;
DevicePrimesPow2LastThree = accelerator.Allocate1D<ulong>(primesPow2LastThree.Length);
DevicePrimesPow2LastThree.View.CopyFromCPU(primesPow2LastThree);
DevicePrimesPow2LastThree.View.CopyFromCPU(stream, primesPow2LastThree);
var primesPow2LastNine = DivisorGenerator.SmallPrimesPow2LastNine;
DevicePrimesPow2LastNine = accelerator.Allocate1D<ulong>(primesPow2LastNine.Length);
DevicePrimesPow2LastNine.View.CopyFromCPU(primesPow2LastNine);
DevicePrimesPow2LastNine.View.CopyFromCPU(stream, primesPow2LastNine);
HeuristicPrimeSieves.EnsureInitialized();
HeuristicCombinedPrimeTester.EnsureInitialized();
var heuristicGroupA = HeuristicPrimeSieves.GroupADivisors;
HeuristicGroupADivisors = CopySpanToDevice(accelerator, heuristicGroupA);
HeuristicGroupADivisors = CopySpanToDevice(accelerator, stream, heuristicGroupA);
var heuristicGroupASquares = HeuristicPrimeSieves.GroupADivisorSquares;
HeuristicGroupADivisorSquares = CopySpanToDevice(accelerator, heuristicGroupASquares);
HeuristicGroupADivisorSquares = CopySpanToDevice(accelerator, stream, heuristicGroupASquares);
HeuristicGroupBDivisorsEnding1 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastOneWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding1 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastOneWithoutLastThree);
HeuristicGroupBDivisorsEnding1 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastOneWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding1 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastOneWithoutLastThree);
HeuristicGroupBDivisorsEnding7 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastSevenWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding7 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastSevenWithoutLastThree);
HeuristicGroupBDivisorsEnding7 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastSevenWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding7 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastSevenWithoutLastThree);
HeuristicGroupBDivisorsEnding9 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastNineWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding9 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastNineWithoutLastThree);
HeuristicGroupBDivisorsEnding9 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastNineWithoutLastThree);
HeuristicGroupBDivisorSquaresEnding9 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastNineWithoutLastThree);
var combinedEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1Span;
HeuristicCombinedDivisorsEnding1 = CopySpanToDevice(accelerator, combinedEnding1);
HeuristicCombinedDivisorsEnding1 = CopySpanToDevice(accelerator, stream, combinedEnding1);
var combinedSquaresEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1SquaresSpan;
HeuristicCombinedDivisorSquaresEnding1 = CopySpanToDevice(accelerator, combinedSquaresEnding1);
HeuristicCombinedDivisorSquaresEnding1 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding1);
var combinedEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3Span;
HeuristicCombinedDivisorsEnding3 = CopySpanToDevice(accelerator, combinedEnding3);
HeuristicCombinedDivisorsEnding3 = CopySpanToDevice(accelerator, stream, combinedEnding3);
var combinedSquaresEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3SquaresSpan;
HeuristicCombinedDivisorSquaresEnding3 = CopySpanToDevice(accelerator, combinedSquaresEnding3);
HeuristicCombinedDivisorSquaresEnding3 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding3);
var combinedEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7Span;
HeuristicCombinedDivisorsEnding7 = CopySpanToDevice(accelerator, combinedEnding7);
HeuristicCombinedDivisorsEnding7 = CopySpanToDevice(accelerator, stream, combinedEnding7);
var combinedSquaresEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7SquaresSpan;
HeuristicCombinedDivisorSquaresEnding7 = CopySpanToDevice(accelerator, combinedSquaresEnding7);
HeuristicCombinedDivisorSquaresEnding7 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding7);
var combinedEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9Span;
HeuristicCombinedDivisorsEnding9 = CopySpanToDevice(accelerator, combinedEnding9);
HeuristicCombinedDivisorsEnding9 = CopySpanToDevice(accelerator, stream, combinedEnding9);
var combinedSquaresEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9SquaresSpan;
HeuristicCombinedDivisorSquaresEnding9 = CopySpanToDevice(accelerator, combinedSquaresEnding9);
HeuristicCombinedDivisorSquaresEnding9 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding9);
}
internal MemoryBuffer1D<uint, Stride1D.Dense> DevicePrimesDefault { get; }
@@ -666,19 +660,19 @@ public sealed class PrimeTester
}
}
private static MemoryBuffer1D<ulong, Stride1D.Dense> CopySpanToDevice(Accelerator accelerator, ReadOnlySpan<ulong> span)
private static MemoryBuffer1D<ulong, Stride1D.Dense> CopySpanToDevice(Accelerator accelerator, AcceleratorStream stream, ReadOnlySpan<ulong> span)
{
var buffer = accelerator.Allocate1D<ulong>(span.Length);
if (!span.IsEmpty)
{
ref ulong sourceRef = ref MemoryMarshal.GetReference(span);
buffer.View.CopyFromCPU(ref sourceRef, span.Length);
buffer.View.CopyFromCPU(stream, ref sourceRef, span.Length);
}
return buffer;
}
private static MemoryBuffer1D<ulong, Stride1D.Dense> CopyUintSpanToDevice(Accelerator accelerator, ReadOnlySpan<uint> span)
private static MemoryBuffer1D<ulong, Stride1D.Dense> CopyUintSpanToDevice(Accelerator accelerator, AcceleratorStream stream, ReadOnlySpan<uint> span)
{
var buffer = accelerator.Allocate1D<ulong>(span.Length);
if (!span.IsEmpty)
@@ -689,7 +683,7 @@ public sealed class PrimeTester
converted[i] = span[i];
}
buffer.View.CopyFromCPU(converted);
buffer.View.CopyFromCPU(stream, converted);
}
return buffer;
@@ -760,7 +754,8 @@ public sealed class PrimeTester
var gpu = PrimeTesterGpuContextPool.Rent();
var accelerator = gpu.Accelerator;
var stream = gpu.Stream;
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SharesFactorKernel);
var kernel = KernelUtil.GetKernel(accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SharesFactorKernel)).CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView<ulong>, ArrayView<byte>>>();
int length = values.Length;
var inputBuffer = accelerator.Allocate1D<ulong>(length);
@@ -770,9 +765,9 @@ public sealed class PrimeTester
ulong[] temp = pool.Rent(length);
values.CopyTo(temp);
inputBuffer.View.CopyFromCPU(stream, ref temp[0], length);
kernel(length, inputBuffer.View, resultBuffer.View);
stream.Synchronize();
kernel(stream, length, inputBuffer.View, resultBuffer.View);
resultBuffer.View.CopyToCPU(stream, ref results[0], length);
stream.Synchronize();
pool.Return(temp, clearArray: false);
resultBuffer.Dispose();
inputBuffer.Dispose();
PerfectNumbers.Core/ULongExtensions.Gpu.cs
+4 -5
View File
@@ -459,6 +459,7 @@ public static partial class ULongExtensions
GpuKernelLease lease = GpuKernelPool.GetKernel();
Accelerator accelerator = lease.Accelerator;
AcceleratorStream stream = lease.Stream;
// Keep this commented out. It should never happen in production code.
// if (accelerator.AcceleratorType == AcceleratorType.CPU)
// {
@@ -470,16 +471,14 @@ public static partial class ULongExtensions
var exponentBuffer = accelerator.Allocate1D<ulong>(1);
var resultBuffer = accelerator.Allocate1D<ulong>(1);
exponentBuffer.View.CopyFromCPU(ref exponent, 1);
exponentBuffer.View.CopyFromCPU(stream, ref exponent, 1);
// We don't need to worry about any left-overs here.
// resultBuffer.MemSetToZero();
// resultBuffer.MemSetToZero(stream);
var kernel = useKeepKernel ? kernelGroup.KeepMontgomery : kernelGroup.ConvertToStandard;
AcceleratorStream stream = lease.Stream;
kernel(stream, 1, exponentBuffer.View, divisor, resultBuffer.View);
stream.Synchronize();
resultBuffer.View.CopyToCPU(stream, ref result, 1);
stream.Synchronize();
resultBuffer.Dispose();
exponentBuffer.Dispose();