From de2a78b27bd7483916374d8af92c9e01678d800b Mon Sep 17 00:00:00 2001 From: mlemanczyk Date: Thu, 6 Nov 2025 18:53:55 +0100 Subject: [PATCH] Streams added to all kernels and GPU operations to separate threads from each other. --- .../Gpu/NttGpuMathTests.cs | 678 +++++++++--------- .../Gpu/NttMontgomeryTests.cs | 144 ++-- PerfectNumbers.Core/DivisorCycleCache.cs | 17 +- PerfectNumbers.Core/Gpu/GpuContextPool.cs | 1 - PerfectNumbers.Core/Gpu/GpuKernelLease.cs | 7 +- PerfectNumbers.Core/Gpu/GpuKernelPool.cs | 39 +- .../Gpu/GpuScratchBufferPool.cs | 10 - .../Gpu/GpuStaticTableInitializer.cs | 8 +- .../Gpu/Kernels/LucasLehmerKernels.cs | 2 +- .../Gpu/Kernels/NttBarrettKernels.cs | 3 +- .../Gpu/Kernels/NttButterflyKernels.cs | 1 + .../Gpu/Kernels/NttMontgomeryKernels.cs | 5 +- .../Gpu/Kernels/NttPointwiseKernels.cs | 1 + .../Gpu/Kernels/NttTransformKernels.cs | 1 + ...MersenneNumberDivisorByDivisorGpuTester.cs | 82 +-- .../Gpu/MersenneNumberDivisorGpuTester.cs | 21 +- .../Gpu/MersenneNumberIncrementalGpuTester.cs | 6 +- .../Gpu/MersenneNumberLucasLehmerGpuTester.cs | 110 ++- .../Gpu/MersenneNumberOrderGpuTester.cs | 6 +- .../Gpu/MersenneNumberResidueGpuTester.cs | 13 +- PerfectNumbers.Core/Gpu/NttGpuMath.cs | 311 ++++---- .../Gpu/PrimeOrderGpuHeuristics.cs | 90 +-- PerfectNumbers.Core/Gpu/SharedGpuContext.cs | 3 +- .../HeuristicCombinedPrimeTester.cs | 5 +- PerfectNumbers.Core/HeuristicPrimeTester.cs | 13 +- PerfectNumbers.Core/MersenneDivisorCycles.cs | 11 +- PerfectNumbers.Core/MersenneNumberTester.cs | 297 ++++---- .../PrimeOrderCalculator.Gpu.cs | 8 +- PerfectNumbers.Core/PrimeTester.cs | 197 +++-- PerfectNumbers.Core/ULongExtensions.Gpu.cs | 9 +- 30 files changed, 1097 insertions(+), 1002 deletions(-) diff --git a/PerfectNumbers.Core.Tests/Gpu/NttGpuMathTests.cs b/PerfectNumbers.Core.Tests/Gpu/NttGpuMathTests.cs index a250fd5..a8b4ae4 100644 --- a/PerfectNumbers.Core.Tests/Gpu/NttGpuMathTests.cs +++ b/PerfectNumbers.Core.Tests/Gpu/NttGpuMathTests.cs @@ -13,367 +13,361 @@ namespace PerfectNumbers.Core.Tests.Gpu; [Collection("GpuNtt")] public class NttGpuMathTests { - [Fact] - [Trait("Category", "Fast")] - public void ForwardGpu_reference_matches_cpu_forward() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Reference; - Span cpu = stackalloc GpuUInt128[4]; - cpu[0] = new GpuUInt128(0UL, 1UL); - cpu[1] = new GpuUInt128(0UL, 2UL); - cpu[2] = new GpuUInt128(0UL, 3UL); - cpu[3] = new GpuUInt128(0UL, 4UL); - Span gpu = stackalloc GpuUInt128[4]; - cpu.CopyTo(gpu); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - NttGpuMath.Forward(cpu, modulus, primitiveRoot); - NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < cpu.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void ForwardGpu_reference_matches_cpu_forward() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Reference; + Span cpu = stackalloc GpuUInt128[4]; + cpu[0] = new GpuUInt128(0UL, 1UL); + cpu[1] = new GpuUInt128(0UL, 2UL); + cpu[2] = new GpuUInt128(0UL, 3UL); + cpu[3] = new GpuUInt128(0UL, 4UL); + Span gpu = stackalloc GpuUInt128[4]; + cpu.CopyTo(gpu); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + NttGpuMath.Forward(cpu, modulus, primitiveRoot); + NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < cpu.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void ForwardGpu_staged_matches_cpu_forward() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Staged; - Span cpu = stackalloc GpuUInt128[4]; - cpu[0] = new GpuUInt128(0UL, 1UL); - cpu[1] = new GpuUInt128(0UL, 2UL); - cpu[2] = new GpuUInt128(0UL, 3UL); - cpu[3] = new GpuUInt128(0UL, 4UL); - Span gpu = stackalloc GpuUInt128[4]; - cpu.CopyTo(gpu); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - NttGpuMath.Forward(cpu, modulus, primitiveRoot); - NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < cpu.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void ForwardGpu_staged_matches_cpu_forward() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Staged; + Span cpu = stackalloc GpuUInt128[4]; + cpu[0] = new GpuUInt128(0UL, 1UL); + cpu[1] = new GpuUInt128(0UL, 2UL); + cpu[2] = new GpuUInt128(0UL, 3UL); + cpu[3] = new GpuUInt128(0UL, 4UL); + Span gpu = stackalloc GpuUInt128[4]; + cpu.CopyTo(gpu); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + NttGpuMath.Forward(cpu, modulus, primitiveRoot); + NttGpuMath.ForwardGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < cpu.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void InverseGpu_reference_matches_cpu_inverse() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Reference; - Span input = stackalloc GpuUInt128[4]; - input[0] = new GpuUInt128(0UL, 5UL); - input[1] = new GpuUInt128(0UL, 7UL); - input[2] = new GpuUInt128(0UL, 9UL); - input[3] = new GpuUInt128(0UL, 11UL); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - Span transformed = stackalloc GpuUInt128[4]; - input.CopyTo(transformed); - NttGpuMath.Forward(transformed, modulus, primitiveRoot); - Span cpu = stackalloc GpuUInt128[4]; - transformed.CopyTo(cpu); - NttGpuMath.Inverse(cpu, modulus, primitiveRoot); - Span gpu = stackalloc GpuUInt128[4]; - transformed.CopyTo(gpu); - NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < input.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void InverseGpu_reference_matches_cpu_inverse() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Reference; + Span input = stackalloc GpuUInt128[4]; + input[0] = new GpuUInt128(0UL, 5UL); + input[1] = new GpuUInt128(0UL, 7UL); + input[2] = new GpuUInt128(0UL, 9UL); + input[3] = new GpuUInt128(0UL, 11UL); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + Span transformed = stackalloc GpuUInt128[4]; + input.CopyTo(transformed); + NttGpuMath.Forward(transformed, modulus, primitiveRoot); + Span cpu = stackalloc GpuUInt128[4]; + transformed.CopyTo(cpu); + NttGpuMath.Inverse(cpu, modulus, primitiveRoot); + Span gpu = stackalloc GpuUInt128[4]; + transformed.CopyTo(gpu); + NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < input.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void InverseGpu_staged_matches_cpu_inverse() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Staged; - Span input = stackalloc GpuUInt128[4]; - input[0] = new GpuUInt128(0UL, 5UL); - input[1] = new GpuUInt128(0UL, 7UL); - input[2] = new GpuUInt128(0UL, 9UL); - input[3] = new GpuUInt128(0UL, 11UL); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - Span transformed = stackalloc GpuUInt128[4]; - input.CopyTo(transformed); - NttGpuMath.Forward(transformed, modulus, primitiveRoot); - Span cpu = stackalloc GpuUInt128[4]; - transformed.CopyTo(cpu); - NttGpuMath.Inverse(cpu, modulus, primitiveRoot); - Span gpu = stackalloc GpuUInt128[4]; - transformed.CopyTo(gpu); - NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < input.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void InverseGpu_staged_matches_cpu_inverse() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Staged; + Span input = stackalloc GpuUInt128[4]; + input[0] = new GpuUInt128(0UL, 5UL); + input[1] = new GpuUInt128(0UL, 7UL); + input[2] = new GpuUInt128(0UL, 9UL); + input[3] = new GpuUInt128(0UL, 11UL); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + Span transformed = stackalloc GpuUInt128[4]; + input.CopyTo(transformed); + NttGpuMath.Forward(transformed, modulus, primitiveRoot); + Span cpu = stackalloc GpuUInt128[4]; + transformed.CopyTo(cpu); + NttGpuMath.Inverse(cpu, modulus, primitiveRoot); + Span gpu = stackalloc GpuUInt128[4]; + transformed.CopyTo(gpu); + NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < input.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void Montgomery_twiddle_inverses_multiply_to_r2() - { - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); + [Fact] + [Trait("Category", "Fast")] + public void Montgomery_twiddle_inverses_multiply_to_r2() + { + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); - GpuContextLease gpu = GpuContextPool.Rent(); - try - { - var accelerator = gpu.Accelerator; + GpuContextLease gpu = GpuContextPool.Rent(); + try + { + var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); - var method = typeof(NttGpuMath).GetMethod("GetSquareCache", BindingFlags.NonPublic | BindingFlags.Static); - method.Should().NotBeNull(); - var cache = method!.Invoke(null, new object[] { accelerator, 8, modulus, primitiveRoot })!; - var cacheType = cache.GetType(); + var method = typeof(NttGpuMath).GetMethod("GetSquareCache", BindingFlags.NonPublic | BindingFlags.Static); + method.Should().NotBeNull(); + var cache = method!.Invoke(null, new object[] { accelerator, 8, modulus, primitiveRoot })!; + var cacheType = cache.GetType(); - var twiddlesMont = (MemoryBuffer1D)cacheType.GetProperty("TwiddlesMont")!.GetValue(cache)!; - var twiddlesInvMont = (MemoryBuffer1D)cacheType.GetProperty("TwiddlesInvMont")!.GetValue(cache)!; - var r2 = (ulong)cacheType.GetProperty("MontR2Mod64")!.GetValue(cache)!; - var mod = (ulong)cacheType.GetProperty("ModulusLow")!.GetValue(cache)!; + var twiddlesMont = (MemoryBuffer1D)cacheType.GetProperty("TwiddlesMont")!.GetValue(cache)!; + var twiddlesInvMont = (MemoryBuffer1D)cacheType.GetProperty("TwiddlesInvMont")!.GetValue(cache)!; + var r2 = (ulong)cacheType.GetProperty("MontR2Mod64")!.GetValue(cache)!; + var mod = (ulong)cacheType.GetProperty("ModulusLow")!.GetValue(cache)!; - int count = 7; // length 8 -> 7 twiddles - var forward = new GpuUInt128[count]; - var inverse = new GpuUInt128[count]; - twiddlesMont.CopyToCPU(forward); - twiddlesInvMont.CopyToCPU(inverse); + int count = 7; // length 8 -> 7 twiddles + var forward = new GpuUInt128[count]; + var inverse = new GpuUInt128[count]; + twiddlesMont.CopyToCPU(stream, forward); + twiddlesInvMont.CopyToCPU(stream, inverse); + stream.Synchronize(); + stream.Dispose(); - for (int i = 0; i < count; i++) - { - ulong prod = (ulong)((((BigInteger)forward[i].Low) * inverse[i].Low) % mod); - prod.Should().Be(r2); - } - } - finally - { - gpu.Dispose(); - } - } + for (int i = 0; i < count; i++) + { + ulong prod = (ulong)((((BigInteger)forward[i].Low) * inverse[i].Low) % mod); + prod.Should().Be(r2); + } + } + finally + { + gpu.Dispose(); + } + } - [Fact] - [Trait("Category", "Fast")] - public void ConvolveGpu_reference_matches_cpu_convolve() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Reference; - Span leftCpu = stackalloc GpuUInt128[4]; - Span rightCpu = stackalloc GpuUInt128[4]; - leftCpu[0] = new GpuUInt128(0UL, 1UL); - leftCpu[1] = new GpuUInt128(0UL, 2UL); - leftCpu[2] = new GpuUInt128(0UL, 3UL); - leftCpu[3] = new GpuUInt128(0UL, 4UL); - rightCpu[0] = new GpuUInt128(0UL, 5UL); - rightCpu[1] = new GpuUInt128(0UL, 6UL); - rightCpu[2] = new GpuUInt128(0UL, 7UL); - rightCpu[3] = new GpuUInt128(0UL, 8UL); - Span leftGpu = stackalloc GpuUInt128[4]; - Span rightGpu = stackalloc GpuUInt128[4]; - leftCpu.CopyTo(leftGpu); - rightCpu.CopyTo(rightGpu); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot); - NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot); - for (int i = 0; i < leftCpu.Length; i++) - { - leftGpu[i].Should().Be(leftCpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void ConvolveGpu_reference_matches_cpu_convolve() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Reference; + Span leftCpu = stackalloc GpuUInt128[4]; + Span rightCpu = stackalloc GpuUInt128[4]; + leftCpu[0] = new GpuUInt128(0UL, 1UL); + leftCpu[1] = new GpuUInt128(0UL, 2UL); + leftCpu[2] = new GpuUInt128(0UL, 3UL); + leftCpu[3] = new GpuUInt128(0UL, 4UL); + rightCpu[0] = new GpuUInt128(0UL, 5UL); + rightCpu[1] = new GpuUInt128(0UL, 6UL); + rightCpu[2] = new GpuUInt128(0UL, 7UL); + rightCpu[3] = new GpuUInt128(0UL, 8UL); + Span leftGpu = stackalloc GpuUInt128[4]; + Span rightGpu = stackalloc GpuUInt128[4]; + leftCpu.CopyTo(leftGpu); + rightCpu.CopyTo(rightGpu); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot); + NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot); + for (int i = 0; i < leftCpu.Length; i++) + { + leftGpu[i].Should().Be(leftCpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void ConvolveGpu_staged_matches_cpu_convolve() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Staged; - Span leftCpu = stackalloc GpuUInt128[4]; - Span rightCpu = stackalloc GpuUInt128[4]; - leftCpu[0] = new GpuUInt128(0UL, 1UL); - leftCpu[1] = new GpuUInt128(0UL, 2UL); - leftCpu[2] = new GpuUInt128(0UL, 3UL); - leftCpu[3] = new GpuUInt128(0UL, 4UL); - rightCpu[0] = new GpuUInt128(0UL, 5UL); - rightCpu[1] = new GpuUInt128(0UL, 6UL); - rightCpu[2] = new GpuUInt128(0UL, 7UL); - rightCpu[3] = new GpuUInt128(0UL, 8UL); - Span leftGpu = stackalloc GpuUInt128[4]; - Span rightGpu = stackalloc GpuUInt128[4]; - leftCpu.CopyTo(leftGpu); - rightCpu.CopyTo(rightGpu); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot); - NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot); - for (int i = 0; i < leftCpu.Length; i++) - { - leftGpu[i].Should().Be(leftCpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void ConvolveGpu_staged_matches_cpu_convolve() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Staged; + Span leftCpu = stackalloc GpuUInt128[4]; + Span rightCpu = stackalloc GpuUInt128[4]; + leftCpu[0] = new GpuUInt128(0UL, 1UL); + leftCpu[1] = new GpuUInt128(0UL, 2UL); + leftCpu[2] = new GpuUInt128(0UL, 3UL); + leftCpu[3] = new GpuUInt128(0UL, 4UL); + rightCpu[0] = new GpuUInt128(0UL, 5UL); + rightCpu[1] = new GpuUInt128(0UL, 6UL); + rightCpu[2] = new GpuUInt128(0UL, 7UL); + rightCpu[3] = new GpuUInt128(0UL, 8UL); + Span leftGpu = stackalloc GpuUInt128[4]; + Span rightGpu = stackalloc GpuUInt128[4]; + leftCpu.CopyTo(leftGpu); + rightCpu.CopyTo(rightGpu); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + NttGpuMath.Convolve(leftCpu, rightCpu, modulus, primitiveRoot); + NttGpuMath.ConvolveGpu(leftGpu, rightGpu, modulus, primitiveRoot); + for (int i = 0; i < leftCpu.Length; i++) + { + leftGpu[i].Should().Be(leftCpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void SquareGpu_reference_matches_cpu_square() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Reference; - Span cpu = stackalloc GpuUInt128[4]; - cpu[0] = new GpuUInt128(0UL, 1UL); - cpu[1] = new GpuUInt128(0UL, 2UL); - cpu[2] = new GpuUInt128(0UL, 3UL); - cpu[3] = new GpuUInt128(0UL, 4UL); - Span gpu = stackalloc GpuUInt128[4]; - cpu.CopyTo(gpu); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - NttGpuMath.Square(cpu, modulus, primitiveRoot); - NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < cpu.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void SquareGpu_reference_matches_cpu_square() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Reference; + Span cpu = stackalloc GpuUInt128[4]; + cpu[0] = new GpuUInt128(0UL, 1UL); + cpu[1] = new GpuUInt128(0UL, 2UL); + cpu[2] = new GpuUInt128(0UL, 3UL); + cpu[3] = new GpuUInt128(0UL, 4UL); + Span gpu = stackalloc GpuUInt128[4]; + cpu.CopyTo(gpu); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + NttGpuMath.Square(cpu, modulus, primitiveRoot); + NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < cpu.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void SquareGpu_staged_matches_cpu_square() - { - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Staged; - Span cpu = stackalloc GpuUInt128[4]; - cpu[0] = new GpuUInt128(0UL, 1UL); - cpu[1] = new GpuUInt128(0UL, 2UL); - cpu[2] = new GpuUInt128(0UL, 3UL); - cpu[3] = new GpuUInt128(0UL, 4UL); - Span gpu = stackalloc GpuUInt128[4]; - cpu.CopyTo(gpu); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - NttGpuMath.Square(cpu, modulus, primitiveRoot); - NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < cpu.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + [Fact] + [Trait("Category", "Fast")] + public void SquareGpu_staged_matches_cpu_square() + { + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Staged; + Span cpu = stackalloc GpuUInt128[4]; + cpu[0] = new GpuUInt128(0UL, 1UL); + cpu[1] = new GpuUInt128(0UL, 2UL); + cpu[2] = new GpuUInt128(0UL, 3UL); + cpu[3] = new GpuUInt128(0UL, 4UL); + Span gpu = stackalloc GpuUInt128[4]; + cpu.CopyTo(gpu); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + NttGpuMath.Square(cpu, modulus, primitiveRoot); + NttGpuMath.SquareGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < cpu.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } - [Fact] - [Trait("Category", "Fast")] - public void BitReverseGpu_matches_cpu() - { - GpuUInt128[] cpu = new GpuUInt128[8]; - for (ulong i = 0; i < (ulong)cpu.Length; i++) - { - cpu[i] = new GpuUInt128(0UL, i); - } + [Fact] + [Trait("Category", "Fast")] + public void BitReverseGpu_matches_cpu() + { + GpuUInt128[] cpu = new GpuUInt128[8]; + for (ulong i = 0; i < (ulong)cpu.Length; i++) + { + cpu[i] = new GpuUInt128(0UL, i); + } - GpuUInt128[] gpu = (GpuUInt128[])cpu.Clone(); - NttGpuMath.BitReverse(cpu); - NttGpuMath.BitReverseGpu(gpu); - gpu.Should().Equal(cpu); - } + GpuUInt128[] gpu = (GpuUInt128[])cpu.Clone(); + NttGpuMath.BitReverse(cpu); + NttGpuMath.BitReverseGpu(gpu); + gpu.Should().Equal(cpu); + } - [Fact] - [Trait("Category", "Fast")] - public void DisposeAll_clears_ntt_caches() - { - // Ensure the cache starts empty. - NttGpuMath.ClearCaches(); + [Fact] + [Trait("Category", "Fast")] + public void DisposeAll_clears_ntt_caches() + { + // Ensure the cache starts empty. + NttGpuMath.ClearCaches(); - Span values = stackalloc GpuUInt128[2]; - values[0] = new GpuUInt128(0UL, 1UL); - values[1] = new GpuUInt128(0UL, 2UL); - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); - GpuContextLease gpu = GpuContextPool.Rent(); - try - { - var buffer = gpu.Accelerator.Allocate1D(values.Length); - try - { - buffer.View.CopyFromCPU(ref values[0], values.Length); - NttGpuMath.SquareDevice(gpu.Accelerator, buffer.View, modulus, primitiveRoot); - } - finally - { - buffer.Dispose(); - } - } - finally - { - gpu.Dispose(); - } + Span values = stackalloc GpuUInt128[2]; + values[0] = new GpuUInt128(0UL, 1UL); + values[1] = new GpuUInt128(0UL, 2UL); + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); + GpuContextLease gpu = GpuContextPool.Rent(); + var stream = gpu.Accelerator.CreateStream(); + var buffer = gpu.Accelerator.Allocate1D(values.Length); + buffer.View.CopyFromCPU(stream, ref values[0], values.Length); + NttGpuMath.SquareDevice(gpu.Accelerator, stream, buffer.View, modulus, primitiveRoot); + stream.Synchronize(); + buffer.Dispose(); + stream.Dispose(); + gpu.Dispose(); - // Verify that the square cache now holds entries for the accelerator. - var field = typeof(NttGpuMath).GetField("SquareCache", BindingFlags.NonPublic | BindingFlags.Static)!; - var cacheObj = field.GetValue(null)!; - int count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!; - count.Should().BeGreaterThan(0); + // Verify that the square cache now holds entries for the accelerator. + var field = typeof(NttGpuMath).GetField("SquareCache", BindingFlags.NonPublic | BindingFlags.Static)!; + var cacheObj = field.GetValue(null)!; + int count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!; + count.Should().BeGreaterThan(0); - GpuContextPool.DisposeAll(); + GpuContextPool.DisposeAll(); - count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!; - count.Should().Be(0); - } + count = (int)cacheObj.GetType().GetProperty("Count")!.GetValue(cacheObj)!; + count.Should().Be(0); + } } diff --git a/PerfectNumbers.Core.Tests/Gpu/NttMontgomeryTests.cs b/PerfectNumbers.Core.Tests/Gpu/NttMontgomeryTests.cs index 50440ca..5a1ec02 100644 --- a/PerfectNumbers.Core.Tests/Gpu/NttMontgomeryTests.cs +++ b/PerfectNumbers.Core.Tests/Gpu/NttMontgomeryTests.cs @@ -9,86 +9,84 @@ namespace PerfectNumbers.Core.Tests.Gpu; [Trait("Category", "Fast")] public class NttMontgomeryTests { - [Fact] - [Trait("Category", "Fast")] - public void SquareDevice_montgomery64_matches_cpu_square_on_small_input() - { - // Small 64-bit modulus and tiny input to keep runtime trivial. - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); + [Fact] + [Trait("Category", "Fast")] + public void SquareDevice_montgomery64_matches_cpu_square_on_small_input() + { + // Small 64-bit modulus and tiny input to keep runtime trivial. + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); - Span cpu = stackalloc GpuUInt128[8]; - for (ulong i = 0; i < (ulong)cpu.Length; i++) - { - cpu[(int)i] = new GpuUInt128(0UL, i + 1UL); - } - Span gpu = stackalloc GpuUInt128[16]; - cpu.CopyTo(gpu); - for (int i = cpu.Length; i < gpu.Length; i++) - { - gpu[i] = new GpuUInt128(0UL, 0UL); - } + Span cpu = stackalloc GpuUInt128[8]; + for (ulong i = 0; i < (ulong)cpu.Length; i++) + { + cpu[(int)i] = new GpuUInt128(0UL, i + 1UL); + } + Span gpu = stackalloc GpuUInt128[16]; + cpu.CopyTo(gpu); + for (int i = cpu.Length; i < gpu.Length; i++) + { + gpu[i] = new GpuUInt128(0UL, 0UL); + } - // CPU reference - NttGpuMath.Square(cpu, modulus, primitiveRoot); + // CPU reference + NttGpuMath.Square(cpu, modulus, primitiveRoot); - // Device execution - GpuContextLease context = GpuContextPool.Rent(); - try - { - var accelerator = context.Accelerator; - using var buffer = accelerator.Allocate1D(gpu.Length); - buffer.View.CopyFromCPU(ref gpu[0], gpu.Length); - NttGpuMath.SquareDevice(accelerator, buffer.View, modulus, primitiveRoot); - buffer.View.CopyToCPU(ref gpu[0], gpu.Length); + // Device execution + GpuContextLease context = GpuContextPool.Rent(); + var accelerator = context.Accelerator; + var stream = accelerator.CreateStream(); + using var buffer = accelerator.Allocate1D(gpu.Length); + buffer.View.CopyFromCPU(stream, ref gpu[0], gpu.Length); + NttGpuMath.SquareDevice(accelerator, stream, buffer.View, modulus, primitiveRoot); + buffer.View.CopyToCPU(stream, ref gpu[0], gpu.Length); + stream.Synchronize(); - for (int i = 0; i < cpu.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - context.Dispose(); - } - } + for (int i = 0; i < cpu.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } - [Fact] - [Trait("Category", "Fast")] - public void InverseDevice_montgomery64_matches_cpu_inverse_on_small_input() - { - var modulus = new GpuUInt128(0UL, 17UL); - var primitiveRoot = new GpuUInt128(0UL, 3UL); + stream.Dispose(); + context.Dispose(); + } - Span input = stackalloc GpuUInt128[8]; - for (ulong i = 0; i < (ulong)input.Length; i++) - { - input[(int)i] = new GpuUInt128(0UL, i + 1UL); - } + [Fact] + [Trait("Category", "Fast")] + public void InverseDevice_montgomery64_matches_cpu_inverse_on_small_input() + { + var modulus = new GpuUInt128(0UL, 17UL); + var primitiveRoot = new GpuUInt128(0UL, 3UL); - Span transformed = stackalloc GpuUInt128[input.Length]; - input.CopyTo(transformed); - NttGpuMath.Forward(transformed, modulus, primitiveRoot); + Span input = stackalloc GpuUInt128[8]; + for (ulong i = 0; i < (ulong)input.Length; i++) + { + input[(int)i] = new GpuUInt128(0UL, i + 1UL); + } - Span cpu = stackalloc GpuUInt128[input.Length]; - transformed.CopyTo(cpu); - NttGpuMath.Inverse(cpu, modulus, primitiveRoot); + Span transformed = stackalloc GpuUInt128[input.Length]; + input.CopyTo(transformed); + NttGpuMath.Forward(transformed, modulus, primitiveRoot); - var original = NttGpuMath.GpuTransformBackend; - try - { - NttGpuMath.GpuTransformBackend = NttBackend.Staged; - Span gpu = stackalloc GpuUInt128[input.Length]; - transformed.CopyTo(gpu); - NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot); - for (int i = 0; i < input.Length; i++) - { - gpu[i].Should().Be(cpu[i]); - } - } - finally - { - NttGpuMath.GpuTransformBackend = original; - } - } + Span cpu = stackalloc GpuUInt128[input.Length]; + transformed.CopyTo(cpu); + NttGpuMath.Inverse(cpu, modulus, primitiveRoot); + + var original = NttGpuMath.GpuTransformBackend; + try + { + NttGpuMath.GpuTransformBackend = NttBackend.Staged; + Span gpu = stackalloc GpuUInt128[input.Length]; + transformed.CopyTo(gpu); + NttGpuMath.InverseGpu(gpu, modulus, primitiveRoot); + for (int i = 0; i < input.Length; i++) + { + gpu[i].Should().Be(cpu[i]); + } + } + finally + { + NttGpuMath.GpuTransformBackend = original; + } + } } diff --git a/PerfectNumbers.Core/DivisorCycleCache.cs b/PerfectNumbers.Core/DivisorCycleCache.cs index c6f1309..edb576d 100644 --- a/PerfectNumbers.Core/DivisorCycleCache.cs +++ b/PerfectNumbers.Core/DivisorCycleCache.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using ILGPU; +using ILGPU.Algorithms; using ILGPU.Runtime; using PerfectNumbers.Core.Gpu; @@ -15,7 +16,7 @@ public sealed class DivisorCycleCache private const byte ByteOne = 1; private const int StackBufferThreshold = 128; - private readonly ConcurrentDictionary, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D>> _gpuKernelCache = new(AcceleratorReferenceComparer.Instance); + private readonly ConcurrentDictionary, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D>> _gpuKernelCache = new(AcceleratorReferenceComparer.Instance); private volatile ulong[] _snapshot; private volatile bool _useGpuGeneration = true; private readonly int _divisorCyclesBatchSize; @@ -284,10 +285,11 @@ public sealed class DivisorCycleCache int pending; do { - kernel(length, _divisorCyclesBatchSize, divisorBuffer.View, powBuffer.View, orderBuffer.View, resultBuffer.View, statusBuffer.View); - stream.Synchronize(); - + kernel(stream, length, _divisorCyclesBatchSize, divisorBuffer.View, powBuffer.View, orderBuffer.View, resultBuffer.View, statusBuffer.View); + statusBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(statusSpan), length); + stream.Synchronize(); + pending = 0; for (int i = 0; i < length; i++) { @@ -300,6 +302,7 @@ public sealed class DivisorCycleCache while (pending > 0); resultBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length); + stream.Synchronize(); resultSpan.CopyTo(destination); if (rentedPow is not null) { @@ -312,9 +315,11 @@ public sealed class DivisorCycleCache gpuLease.Dispose(); } - private static Action, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D> LoadKernel(Accelerator accelerator) + private static Action, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D> LoadKernel(Accelerator accelerator) { - return accelerator.LoadAutoGroupedStreamKernel, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel); + var loaded = accelerator.LoadAutoGroupedStreamKernel, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView1D, ArrayView1D, ArrayView1D, ArrayView1D>>(); } private sealed class AcceleratorReferenceComparer : IEqualityComparer diff --git a/PerfectNumbers.Core/Gpu/GpuContextPool.cs b/PerfectNumbers.Core/Gpu/GpuContextPool.cs index 80e2adb..ec73eeb 100644 --- a/PerfectNumbers.Core/Gpu/GpuContextPool.cs +++ b/PerfectNumbers.Core/Gpu/GpuContextPool.cs @@ -102,7 +102,6 @@ public static class GpuContextPool internal static void Return(PooledContext ctx) { - // ctx.Accelerator.Synchronize(); GpuPool.Enqueue(ctx); } } \ No newline at end of file diff --git a/PerfectNumbers.Core/Gpu/GpuKernelLease.cs b/PerfectNumbers.Core/Gpu/GpuKernelLease.cs index fdbc1a5..1a6465c 100644 --- a/PerfectNumbers.Core/Gpu/GpuKernelLease.cs +++ b/PerfectNumbers.Core/Gpu/GpuKernelLease.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using ILGPU; +using ILGPU.Algorithms; using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; @@ -19,7 +20,7 @@ public sealed class GpuKernelLease Kernels = kernels; } - internal static GpuKernelLease Rent(GpuContextLease gpu, KernelContainer kernels) + internal static GpuKernelLease Rent(GpuContextLease gpu, AcceleratorStream stream, KernelContainer kernels) { if (!Pool.TryDequeue(out var lease)) { @@ -28,7 +29,7 @@ public sealed class GpuKernelLease lease._gpu = gpu; lease.Kernels = kernels; - lease.Stream = gpu.Accelerator.CreateStream(); + lease.Stream = stream; return lease; } @@ -66,7 +67,7 @@ public sealed class GpuKernelLease { get { - var accel = Accelerator; + var accel = Accelerator; // avoid capturing 'this' in lambda return GpuKernelPool.InitOnce(ref Kernels!.Incremental, () => { var loaded = accel.LoadAutoGroupedStreamKernel, ArrayView1D>(IncrementalKernels.IncrementalKernelScan); diff --git a/PerfectNumbers.Core/Gpu/GpuKernelPool.cs b/PerfectNumbers.Core/Gpu/GpuKernelPool.cs index 97fa8d8..3622b2b 100644 --- a/PerfectNumbers.Core/Gpu/GpuKernelPool.cs +++ b/PerfectNumbers.Core/Gpu/GpuKernelPool.cs @@ -111,26 +111,26 @@ public class GpuKernelPool return Interlocked.CompareExchange(ref target, newValue, null) ?? newValue; } - internal static KernelContainer GetKernels(Accelerator accelerator) + internal static KernelContainer GetKernels(Accelerator accelerator, AcceleratorStream stream) { - return _kernels.GetOrAdd(accelerator, static (accelerator) => + return _kernels.GetOrAdd(accelerator, (accelerator) => { var kernels = new KernelContainer(); - PreloadStaticTables(kernels, accelerator); + PreloadStaticTables(kernels, accelerator, stream); return kernels; }); } - internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator) + internal static void PreloadStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream) { - EnsureSmallCyclesOnDevice(kernels, accelerator); - EnsureSmallPrimesOnDevice(kernels, accelerator); - EnsureSmallPrimeFactorTables(kernels, accelerator); + EnsureSmallCyclesOnDevice(kernels, accelerator, stream); + EnsureSmallPrimesOnDevice(kernels, accelerator, stream); + EnsureSmallPrimeFactorTables(kernels, accelerator, stream); } // Ensures the small cycles table is uploaded to the device for the given accelerator. // Returns the ArrayView to pass into kernels (when kernels are extended to accept it). - public static ArrayView1D EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator) + public static ArrayView1D EnsureSmallCyclesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream) { if (kernels.SmallCycles is { } buffer) { @@ -139,12 +139,12 @@ public class GpuKernelPool var host = MersenneDivisorCycles.Shared.ExportSmallCyclesSnapshot(); // TODO: Preload this device buffer during startup and keep it immutable so we can delete the lock above in favor of the preloaded snapshot. var device = accelerator.Allocate1D(host.Length); - device.View.CopyFromCPU(host); + device.View.CopyFromCPU(stream, host); kernels.SmallCycles = device; return device.View; } - public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator) + public static ResiduePrimeViews EnsureSmallPrimesOnDevice(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream) { if (kernels.SmallPrimesLastOne is { } lastOne && kernels.SmallPrimesLastSeven is { } lastSeven && @@ -168,13 +168,13 @@ public class GpuKernelPool var hostLastSevenPow2 = PrimesGenerator.SmallPrimesPow2LastSeven; var deviceLastOne = accelerator.Allocate1D(hostLastOne.Length); - deviceLastOne.View.CopyFromCPU(hostLastOne); + deviceLastOne.View.CopyFromCPU(stream, hostLastOne); var deviceLastSeven = accelerator.Allocate1D(hostLastSeven.Length); - deviceLastSeven.View.CopyFromCPU(hostLastSeven); + deviceLastSeven.View.CopyFromCPU(stream, hostLastSeven); var deviceLastOnePow2 = accelerator.Allocate1D(hostLastOnePow2.Length); - deviceLastOnePow2.View.CopyFromCPU(hostLastOnePow2); + deviceLastOnePow2.View.CopyFromCPU(stream, hostLastOnePow2); var deviceLastSevenPow2 = accelerator.Allocate1D(hostLastSevenPow2.Length); - deviceLastSevenPow2.View.CopyFromCPU(hostLastSevenPow2); + deviceLastSevenPow2.View.CopyFromCPU(stream, hostLastSevenPow2); kernels.SmallPrimesLastOne = deviceLastOne; kernels.SmallPrimesLastSeven = deviceLastSeven; @@ -184,7 +184,7 @@ public class GpuKernelPool return new ResiduePrimeViews(deviceLastOne.View, deviceLastSeven.View, deviceLastOnePow2.View, deviceLastSevenPow2.View); } - public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator) + public static SmallPrimeFactorTables EnsureSmallPrimeFactorTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream) { if (kernels.SmallPrimeFactorsPrimes is { } primeBuffer && kernels.SmallPrimeFactorsSquares is { } squareBuffer) { @@ -200,9 +200,9 @@ public class GpuKernelPool var hostSquares = PrimesGenerator.SmallPrimesPow2; var devicePrimes = accelerator.Allocate1D(hostPrimes.Length); - devicePrimes.View.CopyFromCPU(hostPrimes); + devicePrimes.View.CopyFromCPU(stream, hostPrimes); var deviceSquares = accelerator.Allocate1D(hostSquares.Length); - deviceSquares.View.CopyFromCPU(hostSquares); + deviceSquares.View.CopyFromCPU(stream, hostSquares); kernels.SmallPrimeFactorsPrimes = devicePrimes; kernels.SmallPrimeFactorsSquares = deviceSquares; @@ -218,8 +218,9 @@ public class GpuKernelPool { GpuPrimeWorkLimiter.Acquire(); var gpu = Rent(); - var kernels = GetKernels(gpu.Accelerator); - return GpuKernelLease.Rent(gpu, kernels); + var stream = gpu.Accelerator.CreateStream(); + var kernels = GetKernels(gpu.Accelerator, stream); + return GpuKernelLease.Rent(gpu, stream, kernels); } /// diff --git a/PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs b/PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs index 759b2a7..3b97d80 100644 --- a/PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs +++ b/PerfectNumbers.Core/Gpu/GpuScratchBufferPool.cs @@ -44,11 +44,6 @@ public sealed class ScratchBuffer SmallPrimeFactorPrimeSlots = _accelerator.Allocate1D(newSize); SmallPrimeFactorExponentSlots = _accelerator.Allocate1D(newSize); } - // else - // { - // SmallPrimeFactorPrimeSlots.MemSetToZero(); - // SmallPrimeFactorExponentSlots.MemSetToZero(); - // } } public void ResizeSpecialMaxFactors(int newSize) @@ -61,11 +56,6 @@ public sealed class ScratchBuffer SpecialMaxFactors = _accelerator.Allocate1D(newSize); SpecialMaxCandidates = _accelerator.Allocate1D(newSize); } - // else - // { - // SpecialMaxFactors.MemSetToZero(); - // SpecialMaxCandidates.MemSetToZero(); - // } } } diff --git a/PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs b/PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs index d24c9ce..fdc8603 100644 --- a/PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs +++ b/PerfectNumbers.Core/Gpu/GpuStaticTableInitializer.cs @@ -11,13 +11,13 @@ internal static class GpuStaticTableInitializer { private static readonly ConcurrentDictionary Initialized = new(AcceleratorReferenceComparer.Instance); - internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator) + internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream) { Initialized.GetOrAdd(accelerator, acc => { - PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc); - GpuKernelPool.PreloadStaticTables(kernels, acc); - PrimeOrderGpuHeuristics.PreloadStaticTables( acc); + PrimeTester.PrimeTesterGpuContextPool.EnsureStaticTables(kernels, acc, stream); + GpuKernelPool.PreloadStaticTables(kernels, acc, stream); + PrimeOrderGpuHeuristics.PreloadStaticTables( acc, stream); return 0; }); } diff --git a/PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs index 82cbc1c..49eafd0 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/LucasLehmerKernels.cs @@ -305,7 +305,7 @@ internal static class LucasLehmerKernels states[idx] = s; } - public static void Kernel(Index1D index, ulong exponent, GpuUInt128 modulus, ArrayView state) + public static void Kernel(Index1D _, ulong exponent, GpuUInt128 modulus, ArrayView state) { // Lucas–Lehmer iteration in the field GF(2^p-1). // For p < 128, use a Mersenne-specific squaring + reduction to avoid diff --git a/PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs index 1a4f700..5c807bc 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/NttBarrettKernels.cs @@ -1,5 +1,6 @@ using System.Runtime.CompilerServices; using ILGPU; +using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; @@ -32,7 +33,7 @@ internal static class NttBarrettKernels } // Barrett-based squaring: v = v^2 mod n for 128-bit n - public static void SquareBarrett128Kernel(Index1D index, ArrayView data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow) + public static void SquareBarrett128Kernel(Index1D index, AcceleratorStream stream, ArrayView data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow) { var v = data[index]; Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0); diff --git a/PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs index 7e60e7e..c4c4e5c 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/NttButterflyKernels.cs @@ -1,4 +1,5 @@ using ILGPU; +using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; diff --git a/PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs index c58108d..ea914b3 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/NttMontgomeryKernels.cs @@ -1,5 +1,6 @@ using System.Runtime.CompilerServices; using ILGPU; +using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; @@ -29,7 +30,7 @@ internal static class NttMontgomeryKernels data[i2] = new GpuUInt128(0UL, diff); } - public static void ToMont64Kernel(Index1D index, ArrayView data, ulong modulus, ulong nPrime, ulong r2) + public static void ToMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView data, ulong modulus, ulong nPrime, ulong r2) { ulong a = data[index].Low; ulong v = MontMul64(a, r2, modulus, nPrime); @@ -43,7 +44,7 @@ internal static class NttMontgomeryKernels data[index] = new GpuUInt128(0UL, v); } - public static void SquareMont64Kernel(Index1D index, ArrayView data, ulong modulus, ulong nPrime) + public static void SquareMont64Kernel(Index1D index, AcceleratorStream stream, ArrayView data, ulong modulus, ulong nPrime) { ulong aR = data[index].Low; ulong v = MontMul64(aR, aR, modulus, nPrime); diff --git a/PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs index 66a6e28..2bb505c 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/NttPointwiseKernels.cs @@ -1,4 +1,5 @@ using ILGPU; +using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; diff --git a/PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs index 2740246..541d9f1 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/NttTransformKernels.cs @@ -1,4 +1,5 @@ using ILGPU; +using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs index 0648c2c..053feaa 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberDivisorByDivisorGpuTester.cs @@ -24,14 +24,17 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv private ulong _divisorLimit; // private bool _isConfigured; - private readonly ConcurrentDictionary, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>> _kernelCache = new(AcceleratorReferenceComparer.Instance); + private readonly ConcurrentDictionary, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>> _kernelCache = new(AcceleratorReferenceComparer.Instance); private readonly ConcurrentBag _resourcePool = new(); - private readonly ConcurrentBag _acceleratorPool = new(); private readonly ConcurrentBag _sessionPool = new(); - private Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> GetKernel(Accelerator accelerator) => - _kernelCache.GetOrAdd(accelerator, acc => - acc.LoadAutoGroupedStreamKernel, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>(DivisorByDivisorKernels.CheckKernel)); + private Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> GetKernel(Accelerator accelerator) => + _kernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>(DivisorByDivisorKernels.CheckKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>>(); + }); public int GpuBatchSize @@ -92,8 +95,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv ulong processedCount; ulong lastProcessed; - var gpuLease = RentAccelerator(); + var gpuLease = GpuKernelPool.GetKernel(); var accelerator = gpuLease.Accelerator; + var stream = gpuLease.Stream; // Monitor.Enter(gpuLease.ExecutionLock); @@ -104,6 +108,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv prime, allowedMax, kernel, + stream, resources.DivisorDataBuffer, resources.OffsetBuffer, resources.CountBuffer, @@ -122,9 +127,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv out processedCount); ReturnBatchResources(resources); - // Monitor.Exit(gpuLease.ExecutionLock); - ReturnAccelerator(gpuLease); - + // Monitor.Exit(gpuLease.ExecutionLock); + gpuLease.Dispose(); + if (composite) { divisorsExhausted = true; @@ -167,7 +172,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv private static bool CheckDivisors( ulong prime, ulong allowedMax, - Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> kernel, + Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> kernel, + AcceleratorStream stream, MemoryBuffer1D divisorDataBuffer, MemoryBuffer1D offsetBuffer, MemoryBuffer1D countBuffer, @@ -329,19 +335,20 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv continue; } - divisorDataView.CopyFromCPU(ref divisorDataRef, admissibleCount); - offsetView.CopyFromCPU(ref offsetRef, admissibleCount); - countView.CopyFromCPU(ref countRef, admissibleCount); - cycleView.CopyFromCPU(ref cycleRef, admissibleCount); - exponentViewDevice.CopyFromCPU(ref exponentRef, admissibleCount); + divisorDataView.CopyFromCPU(stream, ref divisorDataRef, admissibleCount); + offsetView.CopyFromCPU(stream, ref offsetRef, admissibleCount); + countView.CopyFromCPU(stream, ref countRef, admissibleCount); + cycleView.CopyFromCPU(stream, ref cycleRef, admissibleCount); + exponentViewDevice.CopyFromCPU(stream, ref exponentRef, admissibleCount); int sentinel = int.MaxValue; - hitIndexView.CopyFromCPU(ref sentinel, 1); + hitIndexView.CopyFromCPU(stream, ref sentinel, 1); - kernel(admissibleCount, divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView); + kernel(stream, new Index1D(admissibleCount), divisorDataView, offsetView, countView, exponentViewDevice, cycleView, hitsView, hitIndexView); int firstHit = sentinel; - hitIndexView.CopyToCPU(ref firstHit, 1); + hitIndexView.CopyToCPU(stream, ref firstHit, 1); + stream.Synchronize(); int hitIndex = firstHit >= admissibleCount ? -1 : firstHit; bool hitFound = hitIndex >= 0; composite = hitFound; @@ -477,9 +484,10 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv public sealed class DivisorScanSession : IMersenneNumberDivisorByDivisorTester.IDivisorScanSession { private readonly MersenneNumberDivisorByDivisorGpuTester _owner; - private readonly GpuContextLease _lease; + private readonly GpuKernelLease _lease; private readonly Accelerator _accelerator; - private Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> _kernel = null!; + private readonly AcceleratorStream _stream; + private Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> _kernel = null!; private MemoryBuffer1D _divisorBuffer = null!; private MemoryBuffer1D _offsetBuffer = null!; private MemoryBuffer1D _countBuffer = null!; @@ -493,8 +501,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv internal DivisorScanSession(MersenneNumberDivisorByDivisorGpuTester owner) { _owner = owner; - _lease = GpuContextPool.Rent(); + _lease = GpuKernelPool.GetKernel(); _accelerator = _lease.Accelerator; + _stream = _lease.Stream; _capacity = Math.Max(1, owner._gpuBatchSize); } @@ -593,35 +602,36 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv ArrayView1D exponentView = _exponentsBuffer.View; ArrayView1D hitView = _hitBuffer.View; - GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor); - divisorView.CopyFromCPU(ref partialData, 1); + AcceleratorStream stream = _stream; + GpuDivisorPartialData partialData = new GpuDivisorPartialData(divisor); + divisorView.CopyFromCPU(stream, ref partialData, 1); int offsetValue = 0; - offsetView.CopyFromCPU(ref offsetValue, 1); + offsetView.CopyFromCPU(stream, ref offsetValue, 1); int countValue = length; - countView.CopyFromCPU(ref countValue, 1); + countView.CopyFromCPU(stream, ref countValue, 1); ulong cycleValue = cycle; - cycleView.CopyFromCPU(ref cycleValue, 1); + cycleView.CopyFromCPU(stream, ref cycleValue, 1); Span hostSpan = _hostBuffer.AsSpan(0, length); primes.CopyTo(hostSpan); ref ulong hostRef = ref MemoryMarshal.GetReference(hostSpan); - exponentView.CopyFromCPU(ref hostRef, length); + exponentView.CopyFromCPU(stream, ref hostRef, length); int sentinel = int.MaxValue; - firstHitView.CopyFromCPU(ref sentinel, 1); + firstHitView.CopyFromCPU(stream, ref sentinel, 1); var kernel = _kernel!; - kernel(1, divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView); + kernel(stream, new Index1D(1), divisorView, offsetView, countView, exponentView, cycleView, hitView, firstHitView); Span hitSlice = hits.Slice(0, length); ref byte hitRef = ref MemoryMarshal.GetReference(hitSlice); - hitView.CopyToCPU(ref hitRef, length); - + hitView.CopyToCPU(stream, ref hitRef, length); + stream.Synchronize(); // Monitor.Exit(_lease.ExecutionLock); } @@ -651,17 +661,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv } } - private GpuContextLease RentAccelerator() - { - if (_acceleratorPool.TryTake(out var lease)) - { - return lease; - } - return GpuContextPool.Rent(); - } - - private void ReturnAccelerator(GpuContextLease lease) => _acceleratorPool.Add(lease); private BatchResources RentBatchResources(Accelerator accelerator, int capacity) { diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs index 3114bcb..c5f2079 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberDivisorGpuTester.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using ILGPU; +using ILGPU.Algorithms; using ILGPU.Runtime; namespace PerfectNumbers.Core.Gpu; @@ -9,11 +10,16 @@ namespace PerfectNumbers.Core.Gpu; /// public sealed class MersenneNumberDivisorGpuTester { - private readonly ConcurrentDictionary>> _kernelCache = new(); + private readonly ConcurrentDictionary>> _kernelCache = new(); private readonly ConcurrentDictionary> _resultBuffers = new(); - private Action> GetKernel(Accelerator accelerator) => - _kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel>(DivisorKernels.Kernel)); + private Action> GetKernel(Accelerator accelerator) => + _kernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel>(DivisorKernels.Kernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate>>(); + }); public static void BuildDivisorCandidates() { @@ -43,15 +49,12 @@ public sealed class MersenneNumberDivisorGpuTester var kernel = GetKernel(accelerator); var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D(1)); // There is no point in clearing this buffer. We always override item [0] and never use it beyond item [0] - // resultBuffer.MemSetToZero(); - kernel(1, exponent, divisor, resultBuffer.View); - stream.Synchronize(); + // resultBuffer.MemSetToZero(stream); + kernel(stream, 1, exponent, divisor, resultBuffer.View); Span result = stackalloc byte[1]; resultBuffer.View.CopyToCPU(stream, ref result[0], 1); + stream.Synchronize(); bool divisible = result[0] != 0; - result[0] = 0; - resultBuffer.View.CopyFromCPU(stream, ref result[0], 1); - stream.Synchronize(); stream.Dispose(); gpu.Dispose(); return divisible; diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs index 76c8b72..d336f17 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberIncrementalGpuTester.cs @@ -31,11 +31,11 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u ulong step3 = ((exponent % 3UL) << 1) % 3UL; ulong step5 = ((exponent % 5UL) << 1) % 5UL; GpuUInt128 twoPGpu = (GpuUInt128)twoP; - var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator); + var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream); ResiduePrimeViews primeViews = default; if (_kernelType == GpuKernelType.Pow2Mod) { - primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator); + primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuLease.Kernels, accelerator, stream); } var orderBuffer = accelerator.Allocate1D(batchSize); @@ -92,8 +92,8 @@ public class MersenneNumberIncrementalGpuTester(GpuKernelType kernelType, bool u smallCyclesView); } + orderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(orders), currentSize); stream.Synchronize(); - orderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(orders), currentSize); if (_kernelType == GpuKernelType.Pow2Mod) { for (i = 0; i < currentSize; i++) diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs index f45368f..0638837 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberLucasLehmerGpuTester.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Collections.Concurrent; using ILGPU; +using ILGPU.Algorithms; using ILGPU.Runtime; using UInt128 = System.UInt128; @@ -8,32 +9,60 @@ namespace PerfectNumbers.Core.Gpu; public class MersenneNumberLucasLehmerGpuTester { - private readonly ConcurrentDictionary>> KernelCache = new(); - private readonly ConcurrentDictionary>> AddSmallKernelCache = new(); - private readonly ConcurrentDictionary>> SubSmallKernelCache = new(); - private readonly ConcurrentDictionary>> ReduceKernelCache = new(); - private readonly ConcurrentDictionary, ArrayView>> IsZeroKernelCache = new(); - private readonly ConcurrentDictionary, ArrayView, ArrayView>> BatchKernelCache = new(); + private readonly ConcurrentDictionary>> KernelCache = new(); + private readonly ConcurrentDictionary>> AddSmallKernelCache = new(); + private readonly ConcurrentDictionary>> SubSmallKernelCache = new(); + private readonly ConcurrentDictionary>> ReduceKernelCache = new(); + private readonly ConcurrentDictionary, ArrayView>> IsZeroKernelCache = new(); + private readonly ConcurrentDictionary, ArrayView, ArrayView>> BatchKernelCache = new(); - private Action> GetKernel(Accelerator accelerator) - { - return KernelCache.GetOrAdd(accelerator, accel => accel.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.Kernel)); - } + private Action> GetKernel(Accelerator accelerator) => + KernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.Kernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate>>(); + }); - private Action> GetAddSmallKernel(Accelerator accelerator) => - AddSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.AddSmallKernel)); + private Action> GetAddSmallKernel(Accelerator accelerator) => + AddSmallKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.AddSmallKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate>>(); + }); - private Action> GetSubSmallKernel(Accelerator accelerator) => - SubSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.SubtractSmallKernel)); + private Action> GetSubSmallKernel(Accelerator accelerator) => + SubSmallKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.SubtractSmallKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate>>(); + }); - private Action> GetReduceKernel(Accelerator accelerator) => - ReduceKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.ReduceModMersenneKernel)); + private Action> GetReduceKernel(Accelerator accelerator) => + ReduceKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel>(LucasLehmerKernels.ReduceModMersenneKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate>>(); + }); - private Action, ArrayView> GetIsZeroKernel(Accelerator accelerator) => - IsZeroKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView>(LucasLehmerKernels.IsZeroKernel)); + private Action, ArrayView> GetIsZeroKernel(Accelerator accelerator) => + IsZeroKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ArrayView>(LucasLehmerKernels.IsZeroKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView>>(); + }); - private Action, ArrayView, ArrayView> GetBatchKernel(Accelerator accelerator) => - BatchKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView, ArrayView>(LucasLehmerKernels.KernelBatch)); + private Action, ArrayView, ArrayView> GetBatchKernel(Accelerator accelerator) => + BatchKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ArrayView, ArrayView>(LucasLehmerKernels.KernelBatch); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView, ArrayView>>(); + }); // Configurable LL slice size to keep kernels short. Default 32. public int SliceSize = 32; @@ -70,13 +99,16 @@ public class MersenneNumberLucasLehmerGpuTester { var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var kernel = GetKernel(accelerator); var modulus = new GpuUInt128(((UInt128)1 << (int)exponent) - 1UL); // TODO: Cache these Mersenne moduli per exponent so LL GPU runs skip rebuilding them every launch. var buffer = accelerator.Allocate1D(1); - kernel(1, exponent, modulus, buffer.View); - accelerator.Synchronize(); + kernel(stream, 1, exponent, modulus, buffer.View); + stream.Synchronize(); + // TODO: Replace this with .CopyToCPU with pre-allocated buffer using stream. Synchronize on the stream after copy. result = buffer.GetAsArray1D()[0].IsZero; buffer.Dispose(); + stream.Dispose(); gpu.Dispose(); } @@ -102,12 +134,13 @@ public class MersenneNumberLucasLehmerGpuTester var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var kernel = GetBatchKernel(accelerator); var expBuffer = accelerator.Allocate1D(count); ulong[] expArray = ArrayPool.Shared.Rent(count); exponents.CopyTo(expArray); - expBuffer.View.CopyFromCPU(ref expArray[0], count); + expBuffer.View.CopyFromCPU(stream, ref expArray[0], count); GpuUInt128[] modulusArray = ArrayPool.Shared.Rent(count); for (int i = 0; i < count; i++) @@ -118,18 +151,19 @@ public class MersenneNumberLucasLehmerGpuTester } var modBuffer = accelerator.Allocate1D(count); - modBuffer.View.CopyFromCPU(ref modulusArray[0], count); + modBuffer.View.CopyFromCPU(stream, ref modulusArray[0], count); var stateBuffer = accelerator.Allocate1D(count); - kernel(count, expBuffer.View, modBuffer.View, stateBuffer.View); - accelerator.Synchronize(); - stateBuffer.View.CopyToCPU(ref residues[0], count); + kernel(stream, count, expBuffer.View, modBuffer.View, stateBuffer.View); + stateBuffer.View.CopyToCPU(stream, ref residues[0], count); + stream.Synchronize(); stateBuffer.Dispose(); modBuffer.Dispose(); expBuffer.Dispose(); ArrayPool.Shared.Return(modulusArray); ArrayPool.Shared.Return(expArray); + stream.Dispose(); gpu.Dispose(); } @@ -154,6 +188,7 @@ public class MersenneNumberLucasLehmerGpuTester { var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var addKernel = GetAddSmallKernel(accelerator); var subKernel = GetSubSmallKernel(accelerator); var reduceKernel = GetReduceKernel(accelerator); @@ -167,8 +202,8 @@ public class MersenneNumberLucasLehmerGpuTester transformLength <<= 1; } var stateBuffer = accelerator.Allocate1D(transformLength); - stateBuffer.MemSetToZero(); - addKernel(1, 4UL, stateBuffer.View); + stateBuffer.MemSetToZero(stream); + addKernel(stream, 1, 4UL, stateBuffer.View); // TODO(LL-SLICE): Slice Lucas–Lehmer iterations into short batches to // avoid long-running kernels and TDR. Example: process 8–64 iterations @@ -183,26 +218,25 @@ public class MersenneNumberLucasLehmerGpuTester for (int s = 0; s < iter; s++) { // Square in NTT domain with staged kernels (short runtime per kernel). - NttGpuMath.SquareDevice(accelerator, stateBuffer.View, nttMod, primitiveRoot); - subKernel(1, 2UL, stateBuffer.View); - reduceKernel(1, exponent, stateBuffer.View); + NttGpuMath.SquareDevice(accelerator, stream, stateBuffer.View, nttMod, primitiveRoot); + subKernel(stream, 1, 2UL, stateBuffer.View); + reduceKernel(stream, 1, exponent, stateBuffer.View); } // Synchronize between slices to yield to scheduler and avoid TDR. - accelerator.Synchronize(); i += (ulong)iter; } - reduceKernel(1, exponent, stateBuffer.View); - + reduceKernel(stream, 1, exponent, stateBuffer.View); var resultBuffer = accelerator.Allocate1D(1); - zeroKernel(1, stateBuffer.View, resultBuffer.View); - accelerator.Synchronize(); + zeroKernel(stream, 1, stateBuffer.View, resultBuffer.View); byte[] result = new byte[1]; - resultBuffer.View.CopyToCPU(result); + resultBuffer.View.CopyToCPU(stream, result); + stream.Synchronize(); bool isPrime = result[0] != 0; resultBuffer.Dispose(); stateBuffer.Dispose(); + stream.Dispose(); gpu.Dispose(); return isPrime; } diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs index 89d86f8..74edb11 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberOrderGpuTester.cs @@ -30,13 +30,13 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO int currentSize; int found = 0; - var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator); + var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuLease.Kernels, accelerator, stream); while (kStart <= maxK && Volatile.Read(ref isPrime)) { remaining = maxK - kStart + 1UL; currentSize = remaining > (UInt128)batchSize ? batchSize : (int)remaining; // This clear is required because the result may not always be set. Remove it after the kernels are modified to always set the result. - foundBuffer.MemSetToZero(); + foundBuffer.MemSetToZero(stream); // Precompute residue automaton bases for this batch UInt128 q0 = twoP * kStart + 1UL; // TODO: Replace these modulo recomputations with the divisor-cycle stepping deltas captured in @@ -49,8 +49,8 @@ public class MersenneNumberOrderGpuTester(GpuKernelType kernelType, bool useGpuO kernel(stream, currentSize, exponent, (GpuUInt128)twoP, (GpuUInt128)kStart, last, divMul, kernelArgs, foundBuffer.View, smallCyclesView); + foundBuffer.View.CopyToCPU(stream, ref found, 1); stream.Synchronize(); - foundBuffer.View.CopyToCPU(ref found, 1); if (found != 0) { Volatile.Write(ref isPrime, false); diff --git a/PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs b/PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs index 5c36843..4bfc040 100644 --- a/PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs +++ b/PerfectNumbers.Core/Gpu/MersenneNumberResidueGpuTester.cs @@ -36,15 +36,16 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder) var stream = accelerator.CreateStream(); var kernel = GetPow2ModKernel(accelerator); var resources = RentResources(accelerator, GpuConstants.ScanBatchSize); - var gpuKernels = GpuKernelPool.GetKernels(accelerator); + var gpuKernels = GpuKernelPool.GetKernels(accelerator, stream); var orderBuffer = resources.OrderBuffer; ulong[] orderArray = resources.OrderArray; - int batchSize = resources.Capacity; + int batchSize = resources.Capacity; // Ensure device has small cycles and primes tables for in-kernel lookup - var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator); - ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator); + // TODO: Pass-in the stream down the path + var smallCyclesView = GpuKernelPool.EnsureSmallCyclesOnDevice(gpuKernels, accelerator, stream); + ResiduePrimeViews primeViews = GpuKernelPool.EnsureSmallPrimesOnDevice(gpuKernels, accelerator, stream); UInt128 kStart = 1UL; UInt128 limit = maxK + UInt128.One; @@ -70,9 +71,9 @@ public class MersenneNumberResidueGpuTester(bool useGpuOrder) kernel(stream, currentSize, exponent, twoPGpu, (GpuUInt128)kStart, last, 0UL, kernelArgs, orderBuffer.View, smallCyclesView, primeViews.LastOne, primeViews.LastSeven, primeViews.LastOnePow2, primeViews.LastSevenPow2); - stream.Synchronize(); - orderBuffer.View.CopyToCPU(ref ordersRef, currentSize); + orderBuffer.View.CopyToCPU(stream, ref ordersRef, currentSize); + stream.Synchronize(); if (!Volatile.Read(ref isPrime)) { break; diff --git a/PerfectNumbers.Core/Gpu/NttGpuMath.cs b/PerfectNumbers.Core/Gpu/NttGpuMath.cs index 4dacb85..1fab0f7 100644 --- a/PerfectNumbers.Core/Gpu/NttGpuMath.cs +++ b/PerfectNumbers.Core/Gpu/NttGpuMath.cs @@ -27,25 +27,41 @@ public static class NttGpuMath // Global backend setting for GPU transforms. Used by ForwardGpu/InverseGpu. // TODO(NTT-OPT): Consider passing backend explicitly to avoid global state. public static NttBackend GpuTransformBackend { get; set; } = NttBackend.Reference; - // Controls modular reduction strategy inside staged NTT paths. - public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto; - private static readonly ConcurrentDictionary, ArrayView, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. - private static readonly ConcurrentDictionary, int, int, int, ArrayView, GpuUInt128>> StageKernelCache = new(); // TODO: Same as above – materialize staged kernels during startup so we can drop ConcurrentDictionary usage entirely per the benchmark guidance. - private static readonly ConcurrentDictionary, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Promote to the static kernel table initialized alongside the GpuModularArithmeticBenchmarks fast path. - private static readonly ConcurrentDictionary, int, int, int, ArrayView, ulong, ulong>> StageMontKernelCache = new(); // TODO: Inline the Montgomery kernels into the startup table instead of using a concurrent cache now that we no longer mutate state at runtime. - private static readonly ConcurrentDictionary, int, int, int, ArrayView, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Same plan – reuse the precomputed kernels measured fastest in MontgomeryMultiplyBenchmarks rather than looking them up dynamically. - private static readonly ConcurrentDictionary, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Collapse into the startup kernel array once Barrett128 constants are preloaded. - private static readonly ConcurrentDictionary, ulong, ulong, ulong, ulong>> SquareBarrett128KernelCache = new(); // TODO: Fold these kernels into the same startup table to avoid concurrent access overhead highlighted in GpuModularArithmeticBenchmarks. - private static readonly ConcurrentDictionary, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Prebind and reuse the Montgomery conversions from MontgomeryMultiplyBenchmarks instead of storing them in a concurrent cache. - private static readonly ConcurrentDictionary, ulong, ulong>> FromMont64KernelCache = new(); // TODO: As above – drop ConcurrentDictionary once startup prewarms every kernel. - private static readonly ConcurrentDictionary, ulong, ulong>> SquareMont64KernelCache = new(); // TODO: Inline into the static kernel table established during initialization. - private static readonly ConcurrentDictionary, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Move to the startup kernel table so Montgomery-scaled launches skip concurrent lookups. + // Controls modular reduction strategy inside staged NTT paths. + public static ModReductionMode ReductionMode { get; set; } = ModReductionMode.Auto; + + private static readonly ConcurrentDictionary, ArrayView, GpuUInt128>> MulKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + private static readonly ConcurrentDictionary, int, int, int, ArrayView, GpuUInt128>> StageKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. - private static readonly ConcurrentDictionary, ArrayView, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Warm these forward kernels during initialization and store them in a plain array so the LucasLehmerGpuBenchmarks launch costs disappear. - - private static readonly ConcurrentDictionary, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Same as above – replace with startup-prepared tables per LucasLehmerGpuBenchmarks guidance. - - private static readonly ConcurrentDictionary, int>> BitReverseKernelCache = new(); // TODO: Precompute bit-reversal kernels during boot so we can remove the concurrent lookup overhead flagged in the benchmarks. + private static readonly ConcurrentDictionary, GpuUInt128, GpuUInt128>> ScaleKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, int, int, int, ArrayView, ulong, ulong>> StageMontKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, int, int, int, ArrayView, ulong, ulong, ulong, ulong>> StageBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, ulong, ulong, ulong>> ToMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, ulong, ulong>> FromMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, GpuUInt128, ulong, ulong, ulong, ulong>> ScaleBarrett128KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, int, int, int, ArrayView, GpuUInt128>> SquareMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, ulong, ulong, ulong>> ScaleMont64KernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, ArrayView, int, GpuUInt128, GpuUInt128>> ForwardKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128>> InverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static readonly ConcurrentDictionary, int>> BitReverseKernelCache = new(); // TODO: Replace this concurrent cache with the prewarmed accelerator-indexed tables from GpuModularArithmeticBenchmarks so kernel launches avoid dictionary lookups once the kernels are baked during startup. + + private static Action, int, int, int, ArrayView, GpuUInt128> GetStageKernel(Accelerator accelerator) => + StageKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, int, int, int, ArrayView, GpuUInt128>(NttButterflyKernels.StageKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, int, int, int, ArrayView, GpuUInt128>>(); + }); private sealed class SquareCacheEntry { @@ -72,7 +88,7 @@ public static class NttGpuMath public ulong BarrettMuHigh { get; } public ulong BarrettMuLow { get; } - public SquareCacheEntry(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot) + public SquareCacheEntry(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot) { Buffer = accelerator.Allocate1D(length); Root = new GpuUInt128(primitiveRoot); @@ -136,9 +152,9 @@ public static class NttGpuMath } Twiddles = accelerator.Allocate1D(twiddleCount); - Twiddles.View.CopyFromCPU(ref forward[0], twiddleCount); + Twiddles.View.CopyFromCPU(stream, ref forward[0], twiddleCount); TwiddlesInv = accelerator.Allocate1D(twiddleCount); - TwiddlesInv.View.CopyFromCPU(ref inverse[0], twiddleCount); + TwiddlesInv.View.CopyFromCPU(stream, ref inverse[0], twiddleCount); // Prepare Montgomery constants for 64-bit moduli and twiddles in Montgomery domain. if (modulus.High == 0UL && (ReductionMode == ModReductionMode.Auto || ReductionMode == ModReductionMode.Mont64)) @@ -166,9 +182,9 @@ public static class NttGpuMath } TwiddlesMont = accelerator.Allocate1D(twiddleCount); - TwiddlesMont.View.CopyFromCPU(ref forwardMont[0], twiddleCount); + TwiddlesMont.View.CopyFromCPU(stream, ref forwardMont[0], twiddleCount); TwiddlesInvMont = accelerator.Allocate1D(twiddleCount); - TwiddlesInvMont.View.CopyFromCPU(ref inverseMont[0], twiddleCount); + TwiddlesInvMont.View.CopyFromCPU(stream, ref inverseMont[0], twiddleCount); pool.Return(forwardMont, clearArray: true); pool.Return(inverseMont, clearArray: true); @@ -258,11 +274,11 @@ public static class NttGpuMath // - Store one GPU buffer with all stage twiddles, or per-stage slices. // - Expose accessors to retrieve per-stage views used by butterfly kernels. - private static SquareCacheEntry GetSquareCache(Accelerator accelerator, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot) + private static SquareCacheEntry GetSquareCache(Accelerator accelerator, AcceleratorStream stream, int length, GpuUInt128 modulus, GpuUInt128 primitiveRoot) { var perAccel = SquareCache.GetOrAdd(accelerator, _ => new ConcurrentDictionary()); var key = new SquareCacheKey(length, modulus, primitiveRoot); - return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, length, modulus, primitiveRoot)); + return perAccel.GetOrAdd(key, _ => new SquareCacheEntry(accelerator, stream, length, modulus, primitiveRoot)); } private static readonly ulong[] SmallPrimes = GenerateSmallPrimes(1000); @@ -349,7 +365,8 @@ public static class NttGpuMath SquareMont64KernelCache.Clear(); ScaleMont64KernelCache.Clear(); ForwardKernelCache.Clear(); - InverseKernelCache.Clear(); + InverseKernelCache.Clear(); + BitReverseKernelCache.Clear(); // TODO(NTT-OPT): Clear TwiddleCache once introduced. return; } @@ -373,56 +390,97 @@ public static class NttGpuMath ScaleMont64KernelCache.TryRemove(accelerator, out _); ForwardKernelCache.TryRemove(accelerator, out _); InverseKernelCache.TryRemove(accelerator, out _); + BitReverseKernelCache.TryRemove(accelerator, out _); // TODO(NTT-OPT): Remove per-accelerator twiddle buffers once added. } - private static Action, int> GetBitReverseKernel(Accelerator accelerator) => - BitReverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, int>(NttTransformKernels.BitReverseKernel)); + private static Action, int> GetBitReverseKernel(Accelerator accelerator) => + BitReverseKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, int>(NttTransformKernels.BitReverseKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, int>>(); + }); - private static Action, ArrayView, GpuUInt128> GetMulKernel(Accelerator accelerator) => - MulKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView, GpuUInt128>(NttPointwiseKernels.MulKernel)); + private static Action, ArrayView, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) => + ForwardKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ArrayView, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView, int, GpuUInt128, GpuUInt128>>(); + }); - private static Action, int, int, int, ArrayView, GpuUInt128> GetStageKernel(Accelerator accelerator) => - StageKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, int, int, int, ArrayView, GpuUInt128>(NttButterflyKernels.StageKernel)); + private static Action, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) => + FromMont64KernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ulong, ulong>>(); + }); - private static Action, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) => - ScaleKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel)); + private static Action, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) => + InverseKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128>>(); + }); - private static Action, int, int, int, ArrayView, ulong, ulong> GetStageMontKernel(Accelerator accelerator) => - StageMontKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, int, int, int, ArrayView, ulong, ulong>(NttMontgomeryKernels.StageMontKernel)); + private static Action, ArrayView, GpuUInt128> GetMulKernel(Accelerator accelerator) => + MulKernelCache.GetOrAdd(accelerator, valueFactory: acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ArrayView, GpuUInt128>(NttPointwiseKernels.MulKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ArrayView, GpuUInt128>>(); + }); - private static Action, int, int, int, ArrayView, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) => - StageBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, int, int, int, ArrayView, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel)); + private static Action, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) => + ScaleKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, GpuUInt128, GpuUInt128>>(); + }); - private static Action, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) => - ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel)); + private static Action, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) => + ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, GpuUInt128, ulong, ulong, ulong, ulong>>(); + }); - private static Action, ulong, ulong, ulong, ulong> GetSquareBarrett128Kernel(Accelerator accelerator) => - SquareBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ulong, ulong, ulong, ulong>(NttBarrettKernels.SquareBarrett128Kernel)); + private static Action, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) => + ScaleMont64KernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ulong, ulong, ulong>>(); + }); - private static Action, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) => - ToMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ulong, ulong, ulong>(NttMontgomeryKernels.ToMont64Kernel)); + private static Action, int, int, int, ArrayView, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) => + StageBarrett128KernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, int, int, int, ArrayView, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, int, int, int, ArrayView, ulong, ulong, ulong, ulong>>(); + }); - private static Action, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) => - FromMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel)); + private static Action, int, int, int, ArrayView, ulong, ulong> GetStageMontKernel(Accelerator accelerator) => + StageMontKernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, int, int, int, ArrayView, ulong, ulong>(NttMontgomeryKernels.StageMontKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, int, int, int, ArrayView, ulong, ulong>>(); + }); - private static Action, ulong, ulong> GetSquareMont64Kernel(Accelerator accelerator) => - SquareMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ulong, ulong>(NttMontgomeryKernels.SquareMont64Kernel)); - - private static Action, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) => - ScaleMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel)); - - private static Action, ArrayView, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) => - ForwardKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel)); - - // TODO(NTT-OPT): Introduce stage-wise Cooley–Tukey kernels and a - // per-(accelerator,length,modulus,root) twiddle cache. The plan: - // - Precompute twiddle factors for all stages once and cache on GPU. - // - Replace the current O(n^2) ForwardKernel with O(n log n) butterflies. - // - Provide accessors like GetStageKernel(...) and GetTwiddleBuffer(...). - - private static Action, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) => - InverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel, ArrayView, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel)); + private static Action, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) => + ToMont64KernelCache.GetOrAdd(accelerator, acc => + { + var loaded = acc.LoadAutoGroupedStreamKernel, int>(NttTransformKernels.BitReverseKernel); + var kernel = KernelUtil.GetKernel(loaded); + return kernel.CreateLauncherDelegate, ulong, ulong, ulong>>(); + }); // TODO(NTT-OPT): Mirror the forward stage-wise design for the inverse // transform. Precompute inverse twiddles and a normalization factor @@ -453,13 +511,15 @@ public static class NttGpuMath var array = pool.Rent(n); values.CopyTo(array); var buffer = accelerator.Allocate1D(n); - buffer.View.CopyFromCPU(ref array[0], n); - kernel(n, buffer.View, bits); - accelerator.Synchronize(); - buffer.View.CopyToCPU(ref array[0], n); + var stream = accelerator.CreateStream(); + buffer.View.CopyFromCPU(stream, ref array[0], n); + kernel(stream, n, buffer.View, bits); + buffer.View.CopyToCPU(stream, ref array[0], n); + stream.Synchronize(); array.AsSpan(0, n).CopyTo(values); pool.Return(array, clearArray: true); buffer.Dispose(); + stream.Dispose(); gpu.Dispose(); } @@ -510,6 +570,7 @@ public static class NttGpuMath int n = (int)values.Length; var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var kernel = GetForwardKernel(accelerator); var pool = ArrayPool.Shared; var inputArray = pool.Rent(n); @@ -517,19 +578,20 @@ public static class NttGpuMath values.CopyTo(inputArray); var inputBuffer = accelerator.Allocate1D(n); var outputBuffer = accelerator.Allocate1D(n); - inputBuffer.View.CopyFromCPU(ref inputArray[0], n); + inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n); UInt128 modValue = modulus; UInt128 expBase = (modValue - 1UL) / (ulong)n; var root = new GpuUInt128(primitiveRoot); root.ModPow((ulong)expBase, modulus); - kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, root); - accelerator.Synchronize(); - outputBuffer.View.CopyToCPU(ref outputArray[0], n); + kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, root); + outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n); + stream.Synchronize(); outputArray.AsSpan(0, n).CopyTo(values); pool.Return(inputArray, clearArray: true); pool.Return(outputArray, clearArray: true); outputBuffer.Dispose(); inputBuffer.Dispose(); + stream.Dispose(); gpu.Dispose(); } @@ -607,6 +669,7 @@ public static class NttGpuMath int n = values.Length; var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var kernel = GetInverseKernel(accelerator); var pool = ArrayPool.Shared; var inputArray = pool.Rent(n); @@ -614,7 +677,7 @@ public static class NttGpuMath values.CopyTo(inputArray); var inputBuffer = accelerator.Allocate1D(n); var outputBuffer = accelerator.Allocate1D(n); - inputBuffer.View.CopyFromCPU(ref inputArray[0], n); + inputBuffer.View.CopyFromCPU(stream, ref inputArray[0], n); UInt128 modValue = modulus; UInt128 expBase = (modValue - 1UL) / (ulong)n; var root = new GpuUInt128(primitiveRoot); @@ -638,14 +701,15 @@ public static class NttGpuMath { nInv.ModInv(modulus); } - kernel(n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv); - accelerator.Synchronize(); - outputBuffer.View.CopyToCPU(ref outputArray[0], n); + kernel(stream, n, inputBuffer.View, outputBuffer.View, n, modulus, rootInv, nInv); + outputBuffer.View.CopyToCPU(stream, ref outputArray[0], n); + stream.Synchronize(); outputArray.AsSpan(0, n).CopyTo(values); pool.Return(inputArray, clearArray: true); pool.Return(outputArray, clearArray: true); outputBuffer.Dispose(); inputBuffer.Dispose(); + stream.Dispose(); gpu.Dispose(); } @@ -655,11 +719,14 @@ public static class NttGpuMath int n = values.Length; var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var buffer = accelerator.Allocate1D(n); - buffer.View.CopyFromCPU(ref values[0], n); - var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot); - ForwardDevice(accelerator, buffer.View, n, modulus, cache); - buffer.View.CopyToCPU(ref values[0], n); + buffer.View.CopyFromCPU(stream, ref values[0], n); + var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot); + ForwardDevice(accelerator, stream, buffer.View, n, modulus, cache); + buffer.View.CopyToCPU(stream, ref values[0], n); + stream.Synchronize(); + stream.Dispose(); buffer.Dispose(); gpu.Dispose(); @@ -670,38 +737,38 @@ public static class NttGpuMath int n = values.Length; var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var buffer = accelerator.Allocate1D(n); - buffer.View.CopyFromCPU(ref values[0], n); - var cache = GetSquareCache(accelerator, n, modulus, primitiveRoot); - InverseDevice(accelerator, buffer.View, n, modulus, cache); - buffer.View.CopyToCPU(ref values[0], n); + buffer.View.CopyFromCPU(stream, ref values[0], n); + var cache = GetSquareCache(accelerator, stream, n, modulus, primitiveRoot); + InverseDevice(accelerator, stream, buffer.View, n, modulus, cache); + buffer.View.CopyToCPU(stream, ref values[0], n); + stream.Synchronize(); + stream.Dispose(); buffer.Dispose(); gpu.Dispose(); } - private static void ForwardDevice(Accelerator accelerator, ArrayView data, int length, GpuUInt128 modulus, SquareCacheEntry cache) + private static void ForwardDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView data, int length, GpuUInt128 modulus, SquareCacheEntry cache) { var bitKernel = GetBitReverseKernel(accelerator); - bitKernel(length, data, cache.Bits); - accelerator.Synchronize(); + bitKernel(stream, length, data, cache.Bits); if (cache.UseMontgomery64) { var toMont = GetToMont64Kernel(accelerator); - toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64); - accelerator.Synchronize(); + toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64); var stageKernel = GetStageMontKernel(accelerator); int butterflies = length >> 1; for (int s = 0; s < cache.Bits; s++) { int len = 1 << (s + 1); int half = len >> 1; - stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64); - accelerator.Synchronize(); + stageKernel( stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesMont!.View, cache.ModulusLow, cache.MontNPrime64); } var fromMont = GetFromMont64Kernel(accelerator); - fromMont(length, data, cache.ModulusLow, cache.MontNPrime64); + fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64); } else if (cache.UseBarrett128) { @@ -711,8 +778,7 @@ public static class NttGpuMath { int len = 1 << (s + 1); int half = len >> 1; - stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); - accelerator.Synchronize(); + stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); } } else @@ -723,39 +789,34 @@ public static class NttGpuMath { int len = 1 << (s + 1); int half = len >> 1; - stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus); - accelerator.Synchronize(); + stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.Twiddles.View, modulus); } } - accelerator.Synchronize(); + stream.Synchronize(); } - private static void InverseDevice(Accelerator accelerator, ArrayView data, int length, GpuUInt128 modulus, SquareCacheEntry cache) + private static void InverseDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView data, int length, GpuUInt128 modulus, SquareCacheEntry cache) { var bitKernel = GetBitReverseKernel(accelerator); - bitKernel(length, data, cache.Bits); - accelerator.Synchronize(); + bitKernel(stream, length, data, cache.Bits); if (cache.UseMontgomery64) { var toMont = GetToMont64Kernel(accelerator); - toMont(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64); - accelerator.Synchronize(); + toMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontR2Mod64); var stageKernel = GetStageMontKernel(accelerator); int butterflies = length >> 1; for (int s = 0; s < cache.Bits; s++) { int len = 1 << (s + 1); int half = len >> 1; - stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64); - accelerator.Synchronize(); + stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInvMont!.View, cache.ModulusLow, cache.MontNPrime64); } var scaleKernel = GetScaleMont64Kernel(accelerator); - scaleKernel(length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64); - accelerator.Synchronize(); + scaleKernel(stream, length, data, cache.ModulusLow, cache.MontNPrime64, cache.MontNInvR64); var fromMont = GetFromMont64Kernel(accelerator); - fromMont(length, data, cache.ModulusLow, cache.MontNPrime64); + fromMont(stream, length, data, cache.ModulusLow, cache.MontNPrime64); } else if (cache.UseBarrett128) { @@ -765,12 +826,10 @@ public static class NttGpuMath { int len = 1 << (s + 1); int half = len >> 1; - stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); - accelerator.Synchronize(); + stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); } var scaleKernel = GetScaleBarrett128Kernel(accelerator); - scaleKernel(length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); - accelerator.Synchronize(); + scaleKernel(stream, length, data, cache.NInv, cache.ModulusHigh, cache.ModulusLow, cache.BarrettMuHigh, cache.BarrettMuLow); } else { @@ -780,15 +839,13 @@ public static class NttGpuMath { int len = 1 << (s + 1); int half = len >> 1; - stageKernel(butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus); - accelerator.Synchronize(); + stageKernel(stream, butterflies, data, len, half, cache.StageOffsets[s], cache.TwiddlesInv.View, modulus); } var scaleKernel = GetScaleKernel(accelerator); - scaleKernel(length, data, cache.NInv, modulus); - accelerator.Synchronize(); + scaleKernel(stream, length, data, cache.NInv, modulus); } - accelerator.Synchronize(); + stream.Synchronize(); } public static void Convolve(Span left, Span right, GpuUInt128 modulus, GpuUInt128 primitiveRoot) @@ -871,19 +928,20 @@ public static class NttGpuMath public static void SquareGpu(Span values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) => ConvolveGpu(values, values, modulus, primitiveRoot); - public static void SquareDevice(Accelerator accelerator, ArrayView values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) + public static void SquareDevice(Accelerator accelerator, AcceleratorStream stream, ArrayView values, GpuUInt128 modulus, GpuUInt128 primitiveRoot) { - var cache = GetSquareCache(accelerator, (int)values.Length, modulus, primitiveRoot); - ForwardDevice(accelerator, values, (int)values.Length, modulus, cache); + var cache = GetSquareCache(accelerator, stream, (int)values.Length, modulus, primitiveRoot); + ForwardDevice(accelerator, stream, values, (int)values.Length, modulus, cache); var mul = GetMulKernel(accelerator); - mul((int)values.Length, values, values, modulus); - InverseDevice(accelerator, values, (int)values.Length, modulus, cache); + mul(stream, (int)values.Length, values, values, modulus); + InverseDevice(accelerator, stream, values, (int)values.Length, modulus, cache); } public static void PointwiseMultiply(Span left, Span right, GpuUInt128 modulus) { var gpu = GpuContextPool.Rent(); var accelerator = gpu.Accelerator; + var stream = accelerator.CreateStream(); var kernel = GetMulKernel(accelerator); var pool = ArrayPool.Shared; var leftArray = pool.Rent(left.Length); @@ -892,18 +950,19 @@ public static class NttGpuMath right.CopyTo(rightArray); var leftBuffer = accelerator.Allocate1D(left.Length); var rightBuffer = accelerator.Allocate1D(right.Length); - leftBuffer.View.CopyFromCPU(ref leftArray[0], left.Length); - rightBuffer.View.CopyFromCPU(ref rightArray[0], right.Length); - kernel(left.Length, leftBuffer.View, rightBuffer.View, modulus); - accelerator.Synchronize(); - leftBuffer.View.CopyToCPU(ref leftArray[0], left.Length); - rightBuffer.View.CopyToCPU(ref rightArray[0], right.Length); + leftBuffer.View.CopyFromCPU(stream, ref leftArray[0], left.Length); + rightBuffer.View.CopyFromCPU(stream, ref rightArray[0], right.Length); + kernel(stream, left.Length, leftBuffer.View, rightBuffer.View, modulus); + leftBuffer.View.CopyToCPU(stream, ref leftArray[0], left.Length); + rightBuffer.View.CopyToCPU(stream, ref rightArray[0], right.Length); + stream.Synchronize(); leftArray.AsSpan(0, left.Length).CopyTo(left); rightArray.AsSpan(0, right.Length).CopyTo(right); pool.Return(leftArray, clearArray: true); pool.Return(rightArray, clearArray: true); rightBuffer.Dispose(); leftBuffer.Dispose(); + stream.Dispose(); gpu.Dispose(); } diff --git a/PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs b/PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs index 931bbb9..6233390 100644 --- a/PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs +++ b/PerfectNumbers.Core/Gpu/PrimeOrderGpuHeuristics.cs @@ -18,9 +18,9 @@ internal enum GpuPow2ModStatus internal static partial class PrimeOrderGpuHeuristics { - internal static void PreloadStaticTables(Accelerator accelerator) + internal static void PreloadStaticTables(Accelerator accelerator, AcceleratorStream stream) { - _ = GetSmallPrimeDeviceCache(accelerator); + _ = GetSmallPrimeDeviceCache(accelerator, stream); } private static readonly ConcurrentDictionary OverflowedPrimes = new(); @@ -106,16 +106,16 @@ internal static partial class PrimeOrderGpuHeuristics s_capability = PrimeOrderGpuCapability.Default; } - private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator) + private static SmallPrimeDeviceCache GetSmallPrimeDeviceCache(Accelerator accelerator, AcceleratorStream stream) { - return SmallPrimeDeviceCaches.GetOrAdd(accelerator, static acc => + return SmallPrimeDeviceCaches.GetOrAdd(accelerator, acc => { uint[] primes = PrimesGenerator.SmallPrimes; ulong[] squares = PrimesGenerator.SmallPrimesPow2; var primeBuffer = acc.Allocate1D(primes.Length); - primeBuffer.View.CopyFromCPU(primes); + primeBuffer.View.CopyFromCPU(stream, primes); var squareBuffer = acc.Allocate1D(squares.Length); - squareBuffer.View.CopyFromCPU(squares); + squareBuffer.View.CopyFromCPU(stream, squares); return new SmallPrimeDeviceCache { Primes = primeBuffer, @@ -203,7 +203,7 @@ internal static partial class PrimeOrderGpuHeuristics AcceleratorStream stream = lease.Stream; var kernel = GetPartialFactorKernel(accelerator); - SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator); + SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream); // TODO: We should create / reallocate these buffer only once or if the new required length is bigger than capacity. var factorBuffer = accelerator.Allocate1D(primeTargets.Length); @@ -213,11 +213,11 @@ internal static partial class PrimeOrderGpuHeuristics var fullyFactoredBuffer = accelerator.Allocate1D(1); // TODO: There is no need to clear these buffers because the kernel will always assign values within the required bounds. - // factorBuffer.MemSetToZero(); - // exponentBuffer.MemSetToZero(); - // countBuffer.MemSetToZero(); - // remainingBuffer.MemSetToZero(); - // fullyFactoredBuffer.MemSetToZero(); + // factorBuffer.MemSetToZero(stream); + // exponentBuffer.MemSetToZero(stream); + // countBuffer.MemSetToZero(stream); + // remainingBuffer.MemSetToZero(stream); + // fullyFactoredBuffer.MemSetToZero(stream); kernel( stream, @@ -234,17 +234,18 @@ internal static partial class PrimeOrderGpuHeuristics remainingBuffer.View, fullyFactoredBuffer.View); - stream.Synchronize(); - countBuffer.View.CopyToCPU(ref factorCount, 1); - factorCount = Math.Min(factorCount, primeTargets.Length); - factorBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length); - exponentBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length); - remainingBuffer.View.CopyToCPU(ref remaining, 1); + countBuffer.View.CopyToCPU(stream, ref factorCount, 1); + factorBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(primeTargets), primeTargets.Length); + exponentBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(exponentTargets), exponentTargets.Length); + remainingBuffer.View.CopyToCPU(stream, ref remaining, 1); byte fullyFactoredFlag = 0; - fullyFactoredBuffer.View.CopyToCPU(ref fullyFactoredFlag, 1); - fullyFactored = fullyFactoredFlag != 0; + fullyFactoredBuffer.View.CopyToCPU(stream, ref fullyFactoredFlag, 1); + stream.Synchronize(); + + fullyFactored = fullyFactoredFlag != 0; + factorCount = Math.Min(factorCount, primeTargets.Length); // TODO: These buffers shouldn't be disposed but rather reused accross call with the assigned accelerator. factorBuffer.Dispose(); @@ -369,7 +370,7 @@ internal static partial class PrimeOrderGpuHeuristics AcceleratorStream stream = lease.Stream; var kernel = GetOrderKernel(accelerator); - SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator); + SmallPrimeDeviceCache cache = GetSmallPrimeDeviceCache(accelerator, stream); // TODO: These buffers should be created reallocated once and assigned to an accelerator. Callers should use their own // thread static cache to prevent other threads from using taking accelerators with pre-allocated buffers if they don't @@ -386,16 +387,16 @@ internal static partial class PrimeOrderGpuHeuristics var statusBuffer = accelerator.Allocate1D(1); // TODO: Remove the cleaning after the order kernel is modified to always set the result. - phiFactorBuffer.MemSetToZero(); - phiExponentBuffer.MemSetToZero(); - workFactorBuffer.MemSetToZero(); - workExponentBuffer.MemSetToZero(); - candidateBuffer.MemSetToZero(); - stackIndexBuffer.MemSetToZero(); - stackExponentBuffer.MemSetToZero(); - stackProductBuffer.MemSetToZero(); - resultBuffer.MemSetToZero(); - statusBuffer.MemSetToZero(); + phiFactorBuffer.MemSetToZero(stream); + phiExponentBuffer.MemSetToZero(stream); + workFactorBuffer.MemSetToZero(stream); + workExponentBuffer.MemSetToZero(stream); + candidateBuffer.MemSetToZero(stream); + stackIndexBuffer.MemSetToZero(stream); + stackExponentBuffer.MemSetToZero(stream); + stackProductBuffer.MemSetToZero(stream); + resultBuffer.MemSetToZero(stream); + statusBuffer.MemSetToZero(stream); uint limit = config.SmallFactorLimit == 0 ? uint.MaxValue : config.SmallFactorLimit; byte hasPrevious = previousOrder.HasValue ? (byte)1 : (byte)0; @@ -425,10 +426,11 @@ internal static partial class PrimeOrderGpuHeuristics cache.Count, buffers); - stream.Synchronize(); byte status = 0; - statusBuffer.View.CopyToCPU(ref status, 1); + statusBuffer.View.CopyToCPU(stream, ref status, 1); + resultBuffer.View.CopyToCPU(stream, ref order, 1); + stream.Synchronize(); PrimeOrderKernelStatus kernelStatus = (PrimeOrderKernelStatus)status; if (kernelStatus == PrimeOrderKernelStatus.Fallback) @@ -445,8 +447,6 @@ internal static partial class PrimeOrderGpuHeuristics throw new InvalidOperationException("GPU Pollard Rho stack overflow; increase HeuristicStackCapacity."); } - resultBuffer.View.CopyToCPU(ref order, 1); - if (kernelStatus == PrimeOrderKernelStatus.FactoringFailure) { order = 0UL; @@ -536,15 +536,19 @@ internal static partial class PrimeOrderGpuHeuristics var exponentBuffer = accelerator.Allocate1D(exponents.Length); var remainderBuffer = accelerator.Allocate1D(exponents.Length); - exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponents), exponents.Length); + exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponents), exponents.Length); // Pow2Mod kernel always assigns the required output elements so we don't need to worry about clearing these. - // remainderBuffer.MemSetToZero(); + // remainderBuffer.MemSetToZero(stream); try { kernel(stream, exponents.Length, exponentBuffer.View, divisorData, remainderBuffer.View); + remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(results), exponents.Length); stream.Synchronize(); + exponentBuffer.Dispose(); + remainderBuffer.Dispose(); + lease.Dispose(); } catch (Exception) { @@ -555,10 +559,6 @@ internal static partial class PrimeOrderGpuHeuristics throw; } - remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(results), exponents.Length); - exponentBuffer.Dispose(); - remainderBuffer.Dispose(); - lease.Dispose(); return true; } @@ -603,20 +603,20 @@ internal static partial class PrimeOrderGpuHeuristics exponentSpan[i] = (GpuUInt128)exponents[i]; } - exponentBuffer.View.CopyFromCPU(ref MemoryMarshal.GetReference(exponentSpan), length); + exponentBuffer.View.CopyFromCPU(stream, ref MemoryMarshal.GetReference(exponentSpan), length); // Pow2ModWide kernel always assigns the required output elements so we don't need to worry about clearing these. - // remainderBuffer.MemSetToZero(); + // remainderBuffer.MemSetToZero(stream); GpuUInt128 modulus = (GpuUInt128)prime; kernel(stream, length, exponentBuffer.View, modulus, remainderBuffer.View); - stream.Synchronize(); Span resultSpan = length <= WideStackThreshold ? stackalloc GpuUInt128[length] : new Span(rentedResults = ArrayPool.Shared.Rent(length), 0, length); - remainderBuffer.View.CopyToCPU(ref MemoryMarshal.GetReference(resultSpan), length); + remainderBuffer.View.CopyToCPU(stream, ref MemoryMarshal.GetReference(resultSpan), length); + stream.Synchronize(); for (int i = 0; i < length; i++) { diff --git a/PerfectNumbers.Core/Gpu/SharedGpuContext.cs b/PerfectNumbers.Core/Gpu/SharedGpuContext.cs index b594d66..bb62d32 100644 --- a/PerfectNumbers.Core/Gpu/SharedGpuContext.cs +++ b/PerfectNumbers.Core/Gpu/SharedGpuContext.cs @@ -7,11 +7,12 @@ namespace PerfectNumbers.Core.Gpu; public static class SharedGpuContext { + // TODO: Remove Lazy in favor for better performance with nullable fields. private static readonly Lazy s_context = new(() => ILGPU.Context.CreateDefault(), LazyThreadSafetyMode.ExecutionAndPublication); private static readonly Lazy s_device = new(() => s_context.Value.GetPreferredDevice(false), LazyThreadSafetyMode.ExecutionAndPublication); private static readonly Lazy s_accelerator = new(() => s_device.Value.CreateAccelerator(s_context.Value), LazyThreadSafetyMode.ExecutionAndPublication); - internal static Context Context => s_context.Value; + internal static Context Context => s_context.Value; internal static Device Device => s_device.Value; diff --git a/PerfectNumbers.Core/HeuristicCombinedPrimeTester.cs b/PerfectNumbers.Core/HeuristicCombinedPrimeTester.cs index 7194f02..4ea3259 100644 --- a/PerfectNumbers.Core/HeuristicCombinedPrimeTester.cs +++ b/PerfectNumbers.Core/HeuristicCombinedPrimeTester.cs @@ -237,17 +237,18 @@ public sealed class HeuristicCombinedPrimeTester int divisorLength = GetCombinedDivisors(nMod10).Length; int compositeFlag = 0; - flagView1D.CopyFromCPU(ref compositeFlag, 1); + flagView1D.CopyFromCPU(stream, ref compositeFlag, 1); kernel( + stream, divisorLength, flagView, n, maxDivisorSquare, HeuristicGpuDivisorTableKind.Combined, gpu.HeuristicGpuTables); + flagView1D.CopyToCPU(stream, ref compositeFlag, 1); stream.Synchronize(); - flagView1D.CopyToCPU(stream, ref compositeFlag, 1); compositeDetected = compositeFlag != 0; gpu.Dispose(); diff --git a/PerfectNumbers.Core/HeuristicPrimeTester.cs b/PerfectNumbers.Core/HeuristicPrimeTester.cs index 18ca587..7b30cc1 100644 --- a/PerfectNumbers.Core/HeuristicPrimeTester.cs +++ b/PerfectNumbers.Core/HeuristicPrimeTester.cs @@ -253,17 +253,19 @@ public sealed class HeuristicPrimeTester int groupALength = HeuristicPrimeSieves.GroupADivisors.Length; - flagView1D.CopyFromCPU(ref compositeFlag, 1); + flagView1D.CopyFromCPU(stream, ref compositeFlag, 1); kernel( + stream, groupALength, flagView, n, maxDivisorSquare, HeuristicGpuDivisorTableKind.GroupA, gpu.HeuristicGpuTables); - stream.Synchronize(); flagView1D.CopyToCPU(stream, ref compositeFlag, 1); + stream.Synchronize(); + compositeDetected = compositeFlag != 0; if (!compositeDetected) @@ -288,16 +290,17 @@ public sealed class HeuristicPrimeTester int divisorLength = GetGroupBDivisors(ending).Length; compositeFlag = 0; - flagView1D.CopyFromCPU(ref compositeFlag, 1); + flagView1D.CopyFromCPU(stream, ref compositeFlag, 1); kernel( + stream, divisorLength, flagView, n, maxDivisorSquare, tableKind, gpu.HeuristicGpuTables); - accelerator.Synchronize(); - flagView1D.CopyToCPU(ref compositeFlag, 1); + flagView1D.CopyToCPU(stream, ref compositeFlag, 1); + stream.Synchronize(); compositeDetected = compositeFlag != 0; } } diff --git a/PerfectNumbers.Core/MersenneDivisorCycles.cs b/PerfectNumbers.Core/MersenneDivisorCycles.cs index 0662b5e..0e00a38 100644 --- a/PerfectNumbers.Core/MersenneDivisorCycles.cs +++ b/PerfectNumbers.Core/MersenneDivisorCycles.cs @@ -7,6 +7,7 @@ using System.Text; using System.Runtime.InteropServices; using PerfectNumbers.Core.Gpu; using ILGPU; +using ILGPU.Algorithms; using ILGPU.Runtime; namespace PerfectNumbers.Core; @@ -806,10 +807,12 @@ public class MersenneDivisorCycles var lease = GpuKernelPool.GetKernel(); var accelerator = lease.Accelerator; var stream = lease.Stream; - var kernel = accelerator.LoadAutoGroupedStreamKernel< + var loaded = accelerator.LoadAutoGroupedStreamKernel< Index1D, ArrayView1D, ArrayView1D>(DivisorCycleKernels.GpuDivisorCycleKernel); + var kernel = KernelUtil.GetKernel(loaded); + var launcher = kernel.CreateLauncherDelegate, ArrayView1D>>(); using Stream outputStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, BufferSize10M, useAsync: true); @@ -860,14 +863,15 @@ public class MersenneDivisorCycles var bufferDiv = accelerator.Allocate1D(validDivisors); var bufferCycle = accelerator.Allocate1D(idx); - kernel( + launcher( + stream, idx, bufferDiv.View, bufferCycle.View); + bufferCycle.View.CopyToCPU(stream, ref outCycles[0], idx); stream.Synchronize(); - bufferCycle.View.CopyToCPU(stream, ref outCycles[0], idx); bufferCycle.Dispose(); bufferDiv.Dispose(); @@ -893,7 +897,6 @@ public class MersenneDivisorCycles pool.Return(divisors, clearArray: false); pool.Return(outCycles, clearArray: false); - stream.Dispose(); lease.Dispose(); } diff --git a/PerfectNumbers.Core/MersenneNumberTester.cs b/PerfectNumbers.Core/MersenneNumberTester.cs index a8620a3..51a7aa2 100644 --- a/PerfectNumbers.Core/MersenneNumberTester.cs +++ b/PerfectNumbers.Core/MersenneNumberTester.cs @@ -16,27 +16,27 @@ public enum GpuKernelType } public sealed class MersenneNumberTester( - bool useIncremental = true, - bool useOrderCache = false, - GpuKernelType kernelType = GpuKernelType.Incremental, - bool useOrder = false, - bool useGpuLucas = true, - bool useGpuScan = true, - bool useGpuOrder = true, - bool useResidue = true, - ulong maxK = 5_000_000UL) + bool useIncremental = true, + bool useOrderCache = false, + GpuKernelType kernelType = GpuKernelType.Incremental, + bool useOrder = false, + bool useGpuLucas = true, + bool useGpuScan = true, + bool useGpuOrder = true, + bool useResidue = true, + ulong maxK = 5_000_000UL) { - private readonly bool _useResidue = useResidue; - private readonly bool _useIncremental = useIncremental && !useResidue; - private readonly ulong _maxK = maxK; + private readonly bool _useResidue = useResidue; + private readonly bool _useIncremental = useIncremental && !useResidue; + private readonly ulong _maxK = maxK; private readonly GpuKernelType _kernelType = kernelType; - private readonly bool _useGpuLucas = useGpuLucas; - private readonly bool _useGpuScan = useGpuScan; // device for pow2mod/incremental scanning - private readonly bool _useGpuOrder = useGpuOrder; // device for order computations - private static readonly ConcurrentDictionary OrderCache = new(); - // TODO: Swap this ConcurrentDictionary for the pooled dictionary variant highlighted in - // Pow2MontgomeryModBenchmarks once order warmups reuse deterministic divisor-cycle snapshots; the pooled approach removed - // the locking overhead when scanning p >= 138M in those benchmarks. + private readonly bool _useGpuLucas = useGpuLucas; + private readonly bool _useGpuScan = useGpuScan; // device for pow2mod/incremental scanning + private readonly bool _useGpuOrder = useGpuOrder; // device for order computations + private static readonly ConcurrentDictionary OrderCache = new(); + // TODO: Swap this ConcurrentDictionary for the pooled dictionary variant highlighted in + // Pow2MontgomeryModBenchmarks once order warmups reuse deterministic divisor-cycle snapshots; the pooled approach removed + // the locking overhead when scanning p >= 138M in those benchmarks. private readonly MersenneNumberIncrementalGpuTester _incrementalGpuTester = new(kernelType, useGpuOrder); private readonly MersenneNumberIncrementalCpuTester _incrementalCpuTester = new(kernelType); private readonly MersenneNumberOrderGpuTester _orderGpuTester = new(kernelType, useGpuOrder); @@ -49,7 +49,7 @@ public sealed class MersenneNumberTester( // TODO: Ensure Program passes useResidue correctly and that residue-vs-incremental-vs-LL selection respects CLI flags. // Also consider injecting a shared cycles cache (MersenneDivisorCycles.Shared) to both CPU and GPU testers. - public void WarmUpOrders(ulong exponent, ulong limit = 5_000_000UL) + public void WarmUpOrders(ulong exponent, ulong limit = 5_000_000UL) { bool cacheEnabled = useOrderCache; if (!cacheEnabled) @@ -57,34 +57,34 @@ public sealed class MersenneNumberTester( return; } - UInt128 twoP = (UInt128)exponent << 1; // 2 * p - LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One; - // CPU path: compute orders directly and cache - if (!_useGpuOrder) - { - UInt128 k = UInt128.One; + UInt128 twoP = (UInt128)exponent << 1; // 2 * p + LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One; + // CPU path: compute orders directly and cache + if (!_useGpuOrder) + { + UInt128 k = UInt128.One; UInt128 q; ulong remainder; for (; k <= limit; k++) - { - q = twoP * k + UInt128.One; - remainder = q.Mod10(); - // TODO: Replace these repeated Mod10/Mod8/Mod3/Mod5 calls with a residue automaton walk so - // order warmups reuse the optimized cycle stepping validated in the residue benchmarks. - bool shouldCheck = lastDigit == LastDigit.Seven - ? remainder == 7UL || remainder == 9UL || remainder == 3UL - : remainder == 1UL || remainder == 3UL || remainder == 9UL; + { + q = twoP * k + UInt128.One; + remainder = q.Mod10(); + // TODO: Replace these repeated Mod10/Mod8/Mod3/Mod5 calls with a residue automaton walk so + // order warmups reuse the optimized cycle stepping validated in the residue benchmarks. + bool shouldCheck = lastDigit == LastDigit.Seven + ? remainder == 7UL || remainder == 9UL || remainder == 3UL + : remainder == 1UL || remainder == 3UL || remainder == 9UL; - // Early modular filters for odd order and small-prime rejections + // Early modular filters for odd order and small-prime rejections if (shouldCheck) { - ulong mod8 = q.Mod8(); - if (mod8 != 1UL && mod8 != 7UL) - { - continue; - } - else if (q.Mod3() == 0UL || q.Mod5() == 0UL) - { + ulong mod8 = q.Mod8(); + if (mod8 != 1UL && mod8 != 7UL) + { + continue; + } + else if (q.Mod3() == 0UL || q.Mod5() == 0UL) + { continue; } } @@ -94,9 +94,9 @@ public sealed class MersenneNumberTester( continue; } - ulong ord = q.CalculateOrder(); - OrderCache[q] = ord; - } + ulong ord = q.CalculateOrder(); + OrderCache[q] = ord; + } return; } @@ -108,23 +108,23 @@ public sealed class MersenneNumberTester( ulong remainder2; for (; k2 <= limit; k2++) { - q2 = twoP * k2 + 1UL; - remainder2 = q2.Mod10(); - // TODO: Reuse the same residue-automaton fast path here so the GPU warmup staging avoids `%` and - // branches the benchmarks showed slower than the tracked stepping helper. - bool shouldCheck2 = lastDigit == LastDigit.Seven - ? remainder2 == 7UL || remainder2 == 9UL || remainder2 == 3UL - : remainder2 == 1UL || remainder2 == 3UL || remainder2 == 9UL; + q2 = twoP * k2 + 1UL; + remainder2 = q2.Mod10(); + // TODO: Reuse the same residue-automaton fast path here so the GPU warmup staging avoids `%` and + // branches the benchmarks showed slower than the tracked stepping helper. + bool shouldCheck2 = lastDigit == LastDigit.Seven + ? remainder2 == 7UL || remainder2 == 9UL || remainder2 == 3UL + : remainder2 == 1UL || remainder2 == 3UL || remainder2 == 9UL; - // Early modular filters for odd order and small-prime rejections + // Early modular filters for odd order and small-prime rejections if (shouldCheck2) { - ulong mod8 = q2.Mod8(); + ulong mod8 = q2.Mod8(); if (mod8 != 1UL && mod8 != 7UL) { shouldCheck2 = false; } - else if (q2.Mod3() == 0UL || q2.Mod5() == 0UL) + else if (q2.Mod3() == 0UL || q2.Mod5() == 0UL) { shouldCheck2 = false; } @@ -146,104 +146,105 @@ public sealed class MersenneNumberTester( UInt128[] qs = qsBuffer; - var gpuLease = GpuKernelPool.GetKernel(); - var accelerator = gpuLease.Accelerator; - var stream = gpuLease.Stream; - var orderKernel = gpuLease.OrderKernel!; + var gpuLease = GpuKernelPool.GetKernel(); + var accelerator = gpuLease.Accelerator; + var stream = gpuLease.Stream; + var orderKernel = gpuLease.OrderKernel!; - // Guard long-running kernels by chunking the warm-up across batches. - // Reuse the same ScanBatchSize knob used for scanning. - int batchSize = GpuConstants.ScanBatchSize; - ulong divMul = (ulong)((((UInt128)1 << 64) - 1UL) / exponent) + 1UL; - int offset = 0; - while (offset < idx) - { - int count = Math.Min(batchSize, idx - offset); - var qBuffer = accelerator.Allocate1D(count); - var orderBuffer = accelerator.Allocate1D(count); - // Convert to device-friendly GpuUInt128 on the fly - var tmp = System.Buffers.ArrayPool.Shared.Rent(count); - for (int i = 0; i < count; i++) - { - tmp[i] = (PerfectNumbers.Core.Gpu.GpuUInt128)qs[offset + i]; - } - // Copy only the populated elements to the device buffer to avoid overrun on pooled arrays. - qBuffer.View.CopyFromCPU(ref tmp[0], count); - System.Buffers.ArrayPool.Shared.Return(tmp); - orderKernel(stream, count, exponent, divMul, qBuffer.View, orderBuffer.View); - stream.Synchronize(); + // Guard long-running kernels by chunking the warm-up across batches. + // Reuse the same ScanBatchSize knob used for scanning. + int batchSize = GpuConstants.ScanBatchSize; + ulong divMul = (ulong)((((UInt128)1 << 64) - 1UL) / exponent) + 1UL; + int offset = 0; + while (offset < idx) + { + int count = Math.Min(batchSize, idx - offset); + var qBuffer = accelerator.Allocate1D(count); + var orderBuffer = accelerator.Allocate1D(count); + // Convert to device-friendly GpuUInt128 on the fly + var tmp = System.Buffers.ArrayPool.Shared.Rent(count); + for (int i = 0; i < count; i++) + { + tmp[i] = (PerfectNumbers.Core.Gpu.GpuUInt128)qs[offset + i]; + } + // Copy only the populated elements to the device buffer to avoid overrun on pooled arrays. + qBuffer.View.CopyFromCPU(stream, ref tmp[0], count); + System.Buffers.ArrayPool.Shared.Return(tmp); + orderKernel(stream, count, exponent, divMul, qBuffer.View, orderBuffer.View); + stream.Synchronize(); - ulong[] orders = orderBuffer.GetAsArray1D(); - for (int i = 0; i < count; i++) - { - ulong order = orders[i]; - if (order == 0UL) - { - order = qs[offset + i].CalculateOrder(); - } + // TODO: Change this to .CopyToCPU and use pre-allocated buffer with fixed, max capacity, using stream. Synchronize on the stream after copy. + ulong[] orders = orderBuffer.GetAsArray1D(); + for (int i = 0; i < count; i++) + { + ulong order = orders[i]; + if (order == 0UL) + { + order = qs[offset + i].CalculateOrder(); + } - if (useOrderCache) - { - OrderCache[qs[offset + i]] = order; - } - } + if (useOrderCache) + { + OrderCache[qs[offset + i]] = order; + } + } - offset += count; - orderBuffer.Dispose(); - qBuffer.Dispose(); - } + offset += count; + orderBuffer.Dispose(); + qBuffer.Dispose(); + } - gpuLease.Dispose(); - ArrayPool.Shared.Return(qs); - } + gpuLease.Dispose(); + ArrayPool.Shared.Return(qs); + } - public bool IsMersennePrime(ulong exponent) - { - // Safe early rejections using known orders for tiny primes: - // 7 | M_p iff 3 | p. Avoid rejecting p == 3 where M_3 == 7 is prime. - if ((exponent % 3UL) == 0UL && exponent != 3UL) - { - // TODO: Replace this `% 3` check with ULongExtensions.Mod3 to align the early rejection with - // the benchmarked bitmask helper instead of generic modulo for CPU workloads. - return false; - } + public bool IsMersennePrime(ulong exponent) + { + // Safe early rejections using known orders for tiny primes: + // 7 | M_p iff 3 | p. Avoid rejecting p == 3 where M_3 == 7 is prime. + if ((exponent % 3UL) == 0UL && exponent != 3UL) + { + // TODO: Replace this `% 3` check with ULongExtensions.Mod3 to align the early rejection with + // the benchmarked bitmask helper instead of generic modulo for CPU workloads. + return false; + } - if ((exponent & 3UL) == 1UL && exponent.SharesFactorWithExponentMinusOne()) - { - return false; - } + if ((exponent & 3UL) == 1UL && exponent.SharesFactorWithExponentMinusOne()) + { + return false; + } - // Skip even exponents: for even p > 2, M_p is composite; here p is large. - if ((exponent & 1UL) == 0UL) - { - return false; - } + // Skip even exponents: for even p > 2, M_p is composite; here p is large. + if ((exponent & 1UL) == 0UL) + { + return false; + } bool prePrime = true; - UInt128 twoP = (UInt128)exponent << 1; // 2 * p - // UInt128 maxDivisor = new(ulong.MaxValue, ulong.MaxValue); - UInt128 maxK = UInt128Numbers.Two.Pow(exponent) / 2 + 1; - LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One; + UInt128 twoP = (UInt128)exponent << 1; // 2 * p + // UInt128 maxDivisor = new(ulong.MaxValue, ulong.MaxValue); + UInt128 maxK = UInt128Numbers.Two.Pow(exponent) / 2 + 1; + LastDigit lastDigit = (exponent & 3UL) == 3UL ? LastDigit.Seven : LastDigit.One; - // In residue mode the residue scan is the final primality check. - if (_useResidue) - { - if (_maxK < maxK) - { - maxK = _maxK; - } + // In residue mode the residue scan is the final primality check. + if (_useResidue) + { + if (_maxK < maxK) + { + maxK = _maxK; + } - if (_useGpuScan) - { - _residueGpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime); - } - else - { - _residueCpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime); - } + if (_useGpuScan) + { + _residueGpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime); + } + else + { + _residueCpuTester!.Scan(exponent, twoP, lastDigit, maxK, ref prePrime); + } - return prePrime; - } + return prePrime; + } if (!_useIncremental) { @@ -296,8 +297,8 @@ public sealed class MersenneNumberTester( } } - return isPrime; - } + return isPrime; + } // Limit of k to scan in the pre-Lucas prescan public const int PreLucasPreScanK = 5_000_000; @@ -309,12 +310,12 @@ public sealed class MersenneNumberTester( } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void LegacyOrderKernel(Index1D index, ulong exponent, ArrayView qs, ArrayView orders) - { - UInt128 q = qs[index]; - UInt128 pow = exponent.PowMod(q); - orders[index] = pow == 1UL ? exponent : 0UL; - } + private static void LegacyOrderKernel(Index1D index, ulong exponent, ArrayView qs, ArrayView orders) + { + UInt128 q = qs[index]; + UInt128 pow = exponent.PowMod(q); + orders[index] = pow == 1UL ? exponent : 0UL; + } } diff --git a/PerfectNumbers.Core/PrimeOrderCalculator.Gpu.cs b/PerfectNumbers.Core/PrimeOrderCalculator.Gpu.cs index fa3120d..d47a303 100644 --- a/PerfectNumbers.Core/PrimeOrderCalculator.Gpu.cs +++ b/PerfectNumbers.Core/PrimeOrderCalculator.Gpu.cs @@ -26,7 +26,7 @@ internal static partial class PrimeOrderCalculator ScratchBuffer scratch = GpuScratchBufferPool.Rent(accelerator, GpuSmallPrimeFactorSlots, 0); - SmallPrimeFactorTables tables = GpuKernelPool.EnsureSmallPrimeFactorTables(kernels, accelerator); + SmallPrimeFactorTables tables = GpuKernelPool.EnsureSmallPrimeFactorTables(kernels, accelerator, stream); if (kernels.SmallPrimeFactorsPrimes is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsPrimes)} is null"); if (kernels.SmallPrimeFactorsSquares is null) throw new NullReferenceException($"{nameof(kernels.SmallPrimeFactorsSquares)} is null"); @@ -52,10 +52,11 @@ internal static partial class PrimeOrderCalculator smallPrimeFactorCountSlotView, smallPrimeFactorRemainingSlotView); - stream.Synchronize(); factorCount = 0; smallPrimeFactorCountSlotView.CopyToCPU(stream, ref factorCount, 1); + stream.Synchronize(); + factorCount = Math.Min(factorCount, GpuSmallPrimeFactorSlots); if (factorCount > 0) @@ -66,6 +67,7 @@ internal static partial class PrimeOrderCalculator } smallPrimeFactorRemainingSlotView.CopyToCPU(stream, ref remaining, 1); + stream.Synchronize(); for (int i = 0; i < factorCount; i++) { @@ -121,11 +123,11 @@ internal static partial class PrimeOrderCalculator divisorData, scratch.SpecialMaxCandidates.View, specialMaxResultView); - stream.Synchronize(); Span result = stackalloc ulong[1]; // specialMaxResultView.CopyToCPU(stream, ref result, 1); specialMaxResultView.CopyToCPU(stream, in result); + stream.Synchronize(); // stream.Synchronize(); // return result != 0; diff --git a/PerfectNumbers.Core/PrimeTester.cs b/PerfectNumbers.Core/PrimeTester.cs index 96e02f3..895f9f2 100644 --- a/PerfectNumbers.Core/PrimeTester.cs +++ b/PerfectNumbers.Core/PrimeTester.cs @@ -111,6 +111,7 @@ public sealed class PrimeTester inputView.CopyFromCPU(stream, ref value, 1); kernel( + stream, 1, inputView, gpu.DevicePrimesDefault.View, @@ -124,8 +125,8 @@ public sealed class PrimeTester gpu.DevicePrimesPow2LastThree.View, gpu.DevicePrimesPow2LastNine.View, outputView); - stream.Synchronize(); outputView.CopyToCPU(stream, ref flag, 1); + stream.Synchronize(); gpu.Dispose(); GpuPrimeWorkLimiter.Release(); @@ -182,9 +183,10 @@ public sealed class PrimeTester var valueSlice = values.Slice(pos, count); ref ulong valueRef = ref MemoryMarshal.GetReference(valueSlice); - inputView.CopyFromCPU(ref valueRef, count); + inputView.CopyFromCPU(stream, ref valueRef, count); kernel( + stream, count, inputView, gpu.DevicePrimesDefault.View, @@ -198,8 +200,7 @@ public sealed class PrimeTester gpu.DevicePrimesPow2LastThree.View, gpu.DevicePrimesPow2LastNine.View, outputView); - stream.Synchronize(); - + var resultSlice = results.Slice(pos, count); ref byte resultRef = ref MemoryMarshal.GetReference(resultSlice); outputView.CopyToCPU(stream, ref resultRef, count); @@ -207,6 +208,7 @@ public sealed class PrimeTester pos += count; } + stream.Synchronize(); gpu.Dispose(); GpuPrimeWorkLimiter.Release(); } @@ -219,13 +221,13 @@ public sealed class PrimeTester private static readonly object SharedTablesLock = new(); private static readonly Dictionary SharedTables = new(AcceleratorReferenceComparer.Instance); - private static SharedHeuristicGpuTables RentSharedTables(Accelerator accelerator) + private static SharedHeuristicGpuTables RentSharedTables(Accelerator accelerator, AcceleratorStream stream) { lock (SharedTablesLock) { if (!SharedTables.TryGetValue(accelerator, out var tables)) { - tables = new SharedHeuristicGpuTables(accelerator); + tables = new SharedHeuristicGpuTables(accelerator, stream); SharedTables.Add(accelerator, tables); } @@ -247,9 +249,9 @@ public sealed class PrimeTester } } - internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator) + internal static void EnsureStaticTables(KernelContainer kernels, Accelerator accelerator, AcceleratorStream stream) { - var tables = RentSharedTables(accelerator); + var tables = RentSharedTables(accelerator, stream); var heuristicTables = tables.CreateHeuristicDivisorTables(); HeuristicGpuDivisorTables.InitializeShared(in heuristicTables); ReleaseSharedTables(tables); @@ -270,38 +272,40 @@ public sealed class PrimeTester } Accelerator accelerator = SharedGpuContext.Accelerator; - KernelContainer kernels = GpuKernelPool.GetKernels(accelerator); - GpuStaticTableInitializer.EnsureStaticTables(kernels, accelerator); + AcceleratorStream stream = accelerator.CreateStream(); + KernelContainer kernels = GpuKernelPool.GetKernels(accelerator, stream); + + GpuStaticTableInitializer.EnsureStaticTables(kernels, accelerator, stream); int toCreate = target - WarmedLeaseCount; for (int i = 0; i < toCreate; i++) { - Pool.Enqueue(new PrimeTesterGpuContextLease(minBufferCapacity)); + Pool.Enqueue(new PrimeTesterGpuContextLease(minBufferCapacity, stream)); } + stream.Synchronize(); + stream.Dispose(); WarmedLeaseCount = target; } } - internal static PrimeTesterGpuContextLease Rent(int minBufferCapacity = 1) - { - if (!Pool.TryDequeue(out var lease)) - { - lease = new PrimeTesterGpuContextLease(minBufferCapacity); - } - else - { - lease.EnsureCapacity(minBufferCapacity); - } + internal static PrimeTesterGpuContextLease Rent(int minBufferCapacity = 1) + { + if (!Pool.TryDequeue(out var lease)) + { + lease = new PrimeTesterGpuContextLease(minBufferCapacity); + } + else + { + lease.EnsureCapacity(minBufferCapacity); + } - lease.ResetReturnFlag(); - return lease; - } + return lease; + } internal static void Return(PrimeTesterGpuContextLease lease) { - lease.Accelerator.Synchronize(); Pool.Enqueue(lease); } @@ -373,35 +377,23 @@ public sealed class PrimeTester } } - internal sealed class PrimeTesterGpuContextLease : IDisposable + internal sealed class PrimeTesterGpuContextLease { - - private bool _returned; private readonly SharedHeuristicGpuTables _sharedTables; private readonly HeuristicGpuDivisorTables _heuristicDivisorTables; - internal readonly Context AcceleratorContext; public readonly Accelerator Accelerator; - private AcceleratorStream? _stream; + private AcceleratorStream? _stream; public AcceleratorStream Stream { - get - { - if (_stream is { } stream) - { - return stream; - } - - stream = Accelerator.CreateStream(); - _stream = stream; - return stream; - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => _stream ?? Accelerator.CreateStream(); } - - public static readonly Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> Kernel = SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>(PrimeTesterKernels.SmallPrimeSieveKernel); - public static readonly Action, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables> HeuristicTrialDivisionKernel = SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel); - + public static readonly Action, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView> Kernel = KernelUtil.GetKernel(SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>(PrimeTesterKernels.SmallPrimeSieveKernel)).CreateLauncherDelegate, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>>(); + + public static readonly Action, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables> HeuristicTrialDivisionKernel = KernelUtil.GetKernel(SharedGpuContext.Accelerator.LoadAutoGroupedStreamKernel, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel)).CreateLauncherDelegate, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>>(); + public MemoryBuffer1D Input = null!; public MemoryBuffer1D Output = null!; public MemoryBuffer1D HeuristicFlag = null!; @@ -440,17 +432,34 @@ public sealed class PrimeTester { AcceleratorContext = SharedGpuContext.Context; Accelerator = SharedGpuContext.Accelerator; - // GpuStaticTableInitializer.EnsureStaticTables(Accelerator); - // Kernel = Accelerator.LoadAutoGroupedStreamKernel, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>(PrimeTesterKernels.SmallPrimeSieveKernel); - // HeuristicTrialDivisionKernel = Accelerator.LoadAutoGroupedStreamKernel, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel); + AcceleratorStream stream = SharedGpuContext.Accelerator.CreateStream(); + var kernels = GpuKernelPool.GetKernels(Accelerator, stream); + GpuStaticTableInitializer.EnsureStaticTables(kernels, Accelerator, stream); - _sharedTables = RentSharedTables(Accelerator); + _sharedTables = RentSharedTables(Accelerator, stream); + + _heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables(); + HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables); + + EnsureCapacity(minBufferCapacity); + stream.Synchronize(); + stream.Dispose(); + } + + internal PrimeTesterGpuContextLease(int minBufferCapacity, AcceleratorStream stream) + { + AcceleratorContext = SharedGpuContext.Context; + Accelerator = SharedGpuContext.Accelerator; + // GpuStaticTableInitializer.EnsureStaticTables(Accelerator); + // Kernel = Accelerator.LoadAutoGroupedStreamKernel, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView, ArrayView>(PrimeTesterKernels.SmallPrimeSieveKernel); + // HeuristicTrialDivisionKernel = Accelerator.LoadAutoGroupedStreamKernel, ulong, ulong, HeuristicGpuDivisorTableKind, HeuristicGpuDivisorTables>(PrimeTesterKernels.HeuristicTrialDivisionKernel); + + _sharedTables = RentSharedTables(Accelerator, stream); _heuristicDivisorTables = _sharedTables.CreateHeuristicDivisorTables(); HeuristicGpuDivisorTables.InitializeShared(in _heuristicDivisorTables); EnsureCapacity(minBufferCapacity); - _returned = false; } public void EnsureCapacity(int minCapacity) @@ -470,25 +479,8 @@ public sealed class PrimeTester BufferCapacity = minCapacity; } - internal void ResetReturnFlag() - { - _returned = false; - } - public void Dispose() { - if (_returned) - { - return; - } - - _returned = true; - if (_stream is { } stream) - { - stream.Dispose(); - _stream = null; - } - Return(this); } @@ -497,6 +489,8 @@ public sealed class PrimeTester Input?.Dispose(); Output?.Dispose(); HeuristicFlag?.Dispose(); + _stream?.Dispose(); + _stream = null; ReleaseSharedTables(_sharedTables); // These resources are shared between GPU leases // Accelerator.Dispose(); @@ -523,86 +517,86 @@ public sealed class PrimeTester { private int _referenceCount; - internal SharedHeuristicGpuTables(Accelerator accelerator) + internal SharedHeuristicGpuTables(Accelerator accelerator, AcceleratorStream stream) { var primesDefault = DivisorGenerator.SmallPrimes; DevicePrimesDefault = accelerator.Allocate1D(primesDefault.Length); - DevicePrimesDefault.View.CopyFromCPU(primesDefault); + DevicePrimesDefault.View.CopyFromCPU(stream, primesDefault); var primesLastOne = DivisorGenerator.SmallPrimesLastOne; DevicePrimesLastOne = accelerator.Allocate1D(primesLastOne.Length); - DevicePrimesLastOne.View.CopyFromCPU(primesLastOne); + DevicePrimesLastOne.View.CopyFromCPU(stream, primesLastOne); var primesLastSeven = DivisorGenerator.SmallPrimesLastSeven; DevicePrimesLastSeven = accelerator.Allocate1D(primesLastSeven.Length); - DevicePrimesLastSeven.View.CopyFromCPU(primesLastSeven); + DevicePrimesLastSeven.View.CopyFromCPU(stream, primesLastSeven); var primesLastThree = DivisorGenerator.SmallPrimesLastThree; DevicePrimesLastThree = accelerator.Allocate1D(primesLastThree.Length); - DevicePrimesLastThree.View.CopyFromCPU(primesLastThree); + DevicePrimesLastThree.View.CopyFromCPU(stream, primesLastThree); var primesLastNine = DivisorGenerator.SmallPrimesLastNine; DevicePrimesLastNine = accelerator.Allocate1D(primesLastNine.Length); - DevicePrimesLastNine.View.CopyFromCPU(primesLastNine); + DevicePrimesLastNine.View.CopyFromCPU(stream, primesLastNine); var primesPow2Default = DivisorGenerator.SmallPrimesPow2; DevicePrimesPow2Default = accelerator.Allocate1D(primesPow2Default.Length); - DevicePrimesPow2Default.View.CopyFromCPU(primesPow2Default); + DevicePrimesPow2Default.View.CopyFromCPU(stream, primesPow2Default); var primesPow2LastOne = DivisorGenerator.SmallPrimesPow2LastOne; DevicePrimesPow2LastOne = accelerator.Allocate1D(primesPow2LastOne.Length); - DevicePrimesPow2LastOne.View.CopyFromCPU(primesPow2LastOne); + DevicePrimesPow2LastOne.View.CopyFromCPU(stream, primesPow2LastOne); var primesPow2LastSeven = DivisorGenerator.SmallPrimesPow2LastSeven; DevicePrimesPow2LastSeven = accelerator.Allocate1D(primesPow2LastSeven.Length); - DevicePrimesPow2LastSeven.View.CopyFromCPU(primesPow2LastSeven); + DevicePrimesPow2LastSeven.View.CopyFromCPU(stream, primesPow2LastSeven); var primesPow2LastThree = DivisorGenerator.SmallPrimesPow2LastThree; DevicePrimesPow2LastThree = accelerator.Allocate1D(primesPow2LastThree.Length); - DevicePrimesPow2LastThree.View.CopyFromCPU(primesPow2LastThree); + DevicePrimesPow2LastThree.View.CopyFromCPU(stream, primesPow2LastThree); var primesPow2LastNine = DivisorGenerator.SmallPrimesPow2LastNine; DevicePrimesPow2LastNine = accelerator.Allocate1D(primesPow2LastNine.Length); - DevicePrimesPow2LastNine.View.CopyFromCPU(primesPow2LastNine); + DevicePrimesPow2LastNine.View.CopyFromCPU(stream, primesPow2LastNine); HeuristicPrimeSieves.EnsureInitialized(); HeuristicCombinedPrimeTester.EnsureInitialized(); var heuristicGroupA = HeuristicPrimeSieves.GroupADivisors; - HeuristicGroupADivisors = CopySpanToDevice(accelerator, heuristicGroupA); + HeuristicGroupADivisors = CopySpanToDevice(accelerator, stream, heuristicGroupA); var heuristicGroupASquares = HeuristicPrimeSieves.GroupADivisorSquares; - HeuristicGroupADivisorSquares = CopySpanToDevice(accelerator, heuristicGroupASquares); + HeuristicGroupADivisorSquares = CopySpanToDevice(accelerator, stream, heuristicGroupASquares); - HeuristicGroupBDivisorsEnding1 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastOneWithoutLastThree); - HeuristicGroupBDivisorSquaresEnding1 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastOneWithoutLastThree); + HeuristicGroupBDivisorsEnding1 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastOneWithoutLastThree); + HeuristicGroupBDivisorSquaresEnding1 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastOneWithoutLastThree); - HeuristicGroupBDivisorsEnding7 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastSevenWithoutLastThree); - HeuristicGroupBDivisorSquaresEnding7 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastSevenWithoutLastThree); + HeuristicGroupBDivisorsEnding7 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastSevenWithoutLastThree); + HeuristicGroupBDivisorSquaresEnding7 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastSevenWithoutLastThree); - HeuristicGroupBDivisorsEnding9 = CopyUintSpanToDevice(accelerator, DivisorGenerator.SmallPrimesLastNineWithoutLastThree); - HeuristicGroupBDivisorSquaresEnding9 = CopySpanToDevice(accelerator, DivisorGenerator.SmallPrimesPow2LastNineWithoutLastThree); + HeuristicGroupBDivisorsEnding9 = CopyUintSpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesLastNineWithoutLastThree); + HeuristicGroupBDivisorSquaresEnding9 = CopySpanToDevice(accelerator, stream, DivisorGenerator.SmallPrimesPow2LastNineWithoutLastThree); var combinedEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1Span; - HeuristicCombinedDivisorsEnding1 = CopySpanToDevice(accelerator, combinedEnding1); + HeuristicCombinedDivisorsEnding1 = CopySpanToDevice(accelerator, stream, combinedEnding1); var combinedSquaresEnding1 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding1SquaresSpan; - HeuristicCombinedDivisorSquaresEnding1 = CopySpanToDevice(accelerator, combinedSquaresEnding1); + HeuristicCombinedDivisorSquaresEnding1 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding1); var combinedEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3Span; - HeuristicCombinedDivisorsEnding3 = CopySpanToDevice(accelerator, combinedEnding3); + HeuristicCombinedDivisorsEnding3 = CopySpanToDevice(accelerator, stream, combinedEnding3); var combinedSquaresEnding3 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding3SquaresSpan; - HeuristicCombinedDivisorSquaresEnding3 = CopySpanToDevice(accelerator, combinedSquaresEnding3); + HeuristicCombinedDivisorSquaresEnding3 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding3); var combinedEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7Span; - HeuristicCombinedDivisorsEnding7 = CopySpanToDevice(accelerator, combinedEnding7); + HeuristicCombinedDivisorsEnding7 = CopySpanToDevice(accelerator, stream, combinedEnding7); var combinedSquaresEnding7 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding7SquaresSpan; - HeuristicCombinedDivisorSquaresEnding7 = CopySpanToDevice(accelerator, combinedSquaresEnding7); + HeuristicCombinedDivisorSquaresEnding7 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding7); var combinedEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9Span; - HeuristicCombinedDivisorsEnding9 = CopySpanToDevice(accelerator, combinedEnding9); + HeuristicCombinedDivisorsEnding9 = CopySpanToDevice(accelerator, stream, combinedEnding9); var combinedSquaresEnding9 = HeuristicCombinedPrimeTester.CombinedDivisorsEnding9SquaresSpan; - HeuristicCombinedDivisorSquaresEnding9 = CopySpanToDevice(accelerator, combinedSquaresEnding9); + HeuristicCombinedDivisorSquaresEnding9 = CopySpanToDevice(accelerator, stream, combinedSquaresEnding9); } internal MemoryBuffer1D DevicePrimesDefault { get; } @@ -666,19 +660,19 @@ public sealed class PrimeTester } } - private static MemoryBuffer1D CopySpanToDevice(Accelerator accelerator, ReadOnlySpan span) + private static MemoryBuffer1D CopySpanToDevice(Accelerator accelerator, AcceleratorStream stream, ReadOnlySpan span) { var buffer = accelerator.Allocate1D(span.Length); if (!span.IsEmpty) { ref ulong sourceRef = ref MemoryMarshal.GetReference(span); - buffer.View.CopyFromCPU(ref sourceRef, span.Length); + buffer.View.CopyFromCPU(stream, ref sourceRef, span.Length); } return buffer; } - private static MemoryBuffer1D CopyUintSpanToDevice(Accelerator accelerator, ReadOnlySpan span) + private static MemoryBuffer1D CopyUintSpanToDevice(Accelerator accelerator, AcceleratorStream stream, ReadOnlySpan span) { var buffer = accelerator.Allocate1D(span.Length); if (!span.IsEmpty) @@ -689,7 +683,7 @@ public sealed class PrimeTester converted[i] = span[i]; } - buffer.View.CopyFromCPU(converted); + buffer.View.CopyFromCPU(stream, converted); } return buffer; @@ -760,7 +754,8 @@ public sealed class PrimeTester var gpu = PrimeTesterGpuContextPool.Rent(); var accelerator = gpu.Accelerator; var stream = gpu.Stream; - var kernel = accelerator.LoadAutoGroupedStreamKernel, ArrayView>(PrimeTesterKernels.SharesFactorKernel); + + var kernel = KernelUtil.GetKernel(accelerator.LoadAutoGroupedStreamKernel, ArrayView>(PrimeTesterKernels.SharesFactorKernel)).CreateLauncherDelegate, ArrayView>>(); int length = values.Length; var inputBuffer = accelerator.Allocate1D(length); @@ -770,9 +765,9 @@ public sealed class PrimeTester ulong[] temp = pool.Rent(length); values.CopyTo(temp); inputBuffer.View.CopyFromCPU(stream, ref temp[0], length); - kernel(length, inputBuffer.View, resultBuffer.View); - stream.Synchronize(); + kernel(stream, length, inputBuffer.View, resultBuffer.View); resultBuffer.View.CopyToCPU(stream, ref results[0], length); + stream.Synchronize(); pool.Return(temp, clearArray: false); resultBuffer.Dispose(); inputBuffer.Dispose(); diff --git a/PerfectNumbers.Core/ULongExtensions.Gpu.cs b/PerfectNumbers.Core/ULongExtensions.Gpu.cs index 85993ae..1569d2f 100644 --- a/PerfectNumbers.Core/ULongExtensions.Gpu.cs +++ b/PerfectNumbers.Core/ULongExtensions.Gpu.cs @@ -459,6 +459,7 @@ public static partial class ULongExtensions GpuKernelLease lease = GpuKernelPool.GetKernel(); Accelerator accelerator = lease.Accelerator; + AcceleratorStream stream = lease.Stream; // Keep this commented out. It should never happen in production code. // if (accelerator.AcceleratorType == AcceleratorType.CPU) // { @@ -470,16 +471,14 @@ public static partial class ULongExtensions var exponentBuffer = accelerator.Allocate1D(1); var resultBuffer = accelerator.Allocate1D(1); - exponentBuffer.View.CopyFromCPU(ref exponent, 1); + exponentBuffer.View.CopyFromCPU(stream, ref exponent, 1); // We don't need to worry about any left-overs here. - // resultBuffer.MemSetToZero(); + // resultBuffer.MemSetToZero(stream); var kernel = useKeepKernel ? kernelGroup.KeepMontgomery : kernelGroup.ConvertToStandard; - AcceleratorStream stream = lease.Stream; kernel(stream, 1, exponentBuffer.View, divisor, resultBuffer.View); - stream.Synchronize(); - resultBuffer.View.CopyToCPU(stream, ref result, 1); + stream.Synchronize(); resultBuffer.Dispose(); exponentBuffer.Dispose();