Document GPU kernel audit progress
This commit is contained in:
Marek Lemańczyk committed
1 parent
1e25ea0ccf
commit
6217ea7888
31 files changed
+3496
-3246
No files matched your search
@@ -131,22 +131,13 @@ public class GpuKernelCompilationTests
|
||||
?? throw new InvalidOperationException("PrimeOrderGpuHeuristics.GetPow2ModWideKernel not found.");
|
||||
|
||||
private static readonly Action<Index1D, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>> MersenneDivisorCyclesKernel =
|
||||
(Action<Index1D, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>)Delegate.CreateDelegate(
|
||||
typeof(Action<Index1D, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>),
|
||||
typeof(MersenneDivisorCycles).GetMethod("GpuDivisorCycleKernel", BindingFlags.NonPublic | BindingFlags.Static)
|
||||
?? throw new InvalidOperationException("MersenneDivisorCycles.GpuDivisorCycleKernel not found."));
|
||||
DivisorCycleKernels.GpuDivisorCycleKernel;
|
||||
|
||||
private static readonly Action<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<byte>> PrimeTesterSmallPrimeKernel =
|
||||
(Action<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<byte>>)Delegate.CreateDelegate(
|
||||
typeof(Action<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<byte>>),
|
||||
typeof(PrimeTester).GetMethod("SmallPrimeSieveKernel", BindingFlags.NonPublic | BindingFlags.Static)
|
||||
?? throw new InvalidOperationException("PrimeTester.SmallPrimeSieveKernel not found."));
|
||||
PrimeTesterKernels.SmallPrimeSieveKernel;
|
||||
|
||||
private static readonly Action<Index1D, ArrayView<ulong>, ArrayView<byte>> PrimeTesterSharesFactorKernel =
|
||||
(Action<Index1D, ArrayView<ulong>, ArrayView<byte>>)Delegate.CreateDelegate(
|
||||
typeof(Action<Index1D, ArrayView<ulong>, ArrayView<byte>>),
|
||||
typeof(PrimeTester).GetMethod("SharesFactorKernel", BindingFlags.NonPublic | BindingFlags.Static)
|
||||
?? throw new InvalidOperationException("PrimeTester.SharesFactorKernel not found."));
|
||||
PrimeTesterKernels.SharesFactorKernel;
|
||||
|
||||
public static IEnumerable<object[]> KernelLoaders()
|
||||
{
|
||||
@@ -180,9 +171,9 @@ public class GpuKernelCompilationTests
|
||||
yield return Loader("PrimeOrderGpuHeuristics.CalculateOrderKernel", CompilePrimeOrderKernel);
|
||||
yield return Loader("PrimeOrderGpuHeuristics.Pow2ModKernel", CompilePrimeOrderPow2ModKernel);
|
||||
yield return Loader("PrimeOrderGpuHeuristics.Pow2ModKernelWide", CompilePrimeOrderPow2ModWideKernel);
|
||||
yield return Loader("MersenneDivisorCycles.GpuDivisorCycleKernel", CompileMersenneDivisorCyclesKernel);
|
||||
yield return Loader("PrimeTester.SmallPrimeSieveKernel", CompilePrimeTesterSmallPrimeKernel);
|
||||
yield return Loader("PrimeTester.SharesFactorKernel", CompilePrimeTesterSharesFactorKernel);
|
||||
yield return Loader("DivisorCycleKernels.GpuDivisorCycleKernel", CompileMersenneDivisorCyclesKernel);
|
||||
yield return Loader("PrimeTesterKernels.SmallPrimeSieveKernel", CompilePrimeTesterSmallPrimeKernel);
|
||||
yield return Loader("PrimeTesterKernels.SharesFactorKernel", CompilePrimeTesterSharesFactorKernel);
|
||||
yield return Loader("GpuKernelPool.KernelLeaseKernels", CompileGpuKernelPoolKernels);
|
||||
}
|
||||
|
||||
|
||||
@@ -269,80 +269,7 @@ public sealed class DivisorCycleCache
|
||||
|
||||
private static Action<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>> LoadKernel(Accelerator accelerator)
|
||||
{
|
||||
return accelerator.LoadAutoGroupedStreamKernel<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>(GpuAdvanceDivisorCyclesKernel);
|
||||
}
|
||||
|
||||
private static void GpuAdvanceDivisorCyclesKernel(
|
||||
Index1D index,
|
||||
int steps,
|
||||
ArrayView1D<ulong, Stride1D.Dense> divisors,
|
||||
ArrayView1D<ulong, Stride1D.Dense> pow,
|
||||
ArrayView1D<ulong, Stride1D.Dense> order,
|
||||
ArrayView1D<ulong, Stride1D.Dense> cycles,
|
||||
ArrayView1D<byte, Stride1D.Dense> status)
|
||||
{
|
||||
if (status[index] != ByteZero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong divisor = divisors[index];
|
||||
|
||||
if ((divisor & (divisor - 1UL)) == 0UL)
|
||||
{
|
||||
cycles[index] = 1UL;
|
||||
status[index] = ByteOne;
|
||||
pow[index] = 1UL;
|
||||
order[index] = 1UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong currentPow = pow[index];
|
||||
ulong currentOrder = order[index];
|
||||
|
||||
if (divisor <= 3UL)
|
||||
{
|
||||
while (currentPow != 1UL)
|
||||
{
|
||||
currentPow += currentPow;
|
||||
if (currentPow >= divisor)
|
||||
{
|
||||
currentPow -= divisor;
|
||||
}
|
||||
|
||||
currentOrder++;
|
||||
}
|
||||
|
||||
cycles[index] = currentOrder;
|
||||
status[index] = ByteOne;
|
||||
pow[index] = currentPow;
|
||||
order[index] = currentOrder;
|
||||
return;
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
currentPow += currentPow;
|
||||
if (currentPow >= divisor)
|
||||
{
|
||||
currentPow -= divisor;
|
||||
}
|
||||
|
||||
currentOrder++;
|
||||
|
||||
if (currentPow == 1UL)
|
||||
{
|
||||
cycles[index] = currentOrder;
|
||||
status[index] = ByteOne;
|
||||
pow[index] = currentPow;
|
||||
order[index] = currentOrder;
|
||||
return;
|
||||
}
|
||||
}
|
||||
while (--steps != 0);
|
||||
|
||||
pow[index] = currentPow;
|
||||
order[index] = currentOrder;
|
||||
return accelerator.LoadAutoGroupedStreamKernel<Index1D, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<byte, Stride1D.Dense>>(DivisorCycleKernels.GpuAdvanceDivisorCyclesKernel);
|
||||
}
|
||||
|
||||
private sealed class AcceleratorReferenceComparer : IEqualityComparer<Accelerator>
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
using static PerfectNumbers.Core.Gpu.GpuContextPool;
|
||||
@@ -66,7 +65,7 @@ public sealed class GpuKernelLease
|
||||
{
|
||||
return KernelContainer.InitOnce(ref _kernels!.Order, () =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, ulong, ArrayView<GpuUInt128>, ArrayView<ulong>>(OrderKernelScan);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, ulong, ArrayView<GpuUInt128>, ArrayView<ulong>>(OrderKernels.OrderKernelScan);
|
||||
var kernel = KernelUtil.GetKernel(loaded);
|
||||
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ulong, ArrayView<GpuUInt128>, ArrayView<ulong>>>();
|
||||
});
|
||||
@@ -83,7 +82,7 @@ public sealed class GpuKernelLease
|
||||
{
|
||||
return KernelContainer.InitOnce(ref _kernels!.Pow2Mod, () =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>(Pow2ModKernelScan);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>(Pow2ModKernels.Pow2ModKernelScan);
|
||||
var kernel = KernelUtil.GetKernel(loaded);
|
||||
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||
});
|
||||
@@ -100,7 +99,7 @@ public sealed class GpuKernelLease
|
||||
{
|
||||
return KernelContainer.InitOnce(ref _kernels!.Incremental, () =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernelScan);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernels.IncrementalKernelScan);
|
||||
var kernel = KernelUtil.GetKernel(loaded);
|
||||
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ulong, ulong, ulong, ulong, ArrayView<ulong>, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||
});
|
||||
@@ -117,7 +116,7 @@ public sealed class GpuKernelLease
|
||||
{
|
||||
return KernelContainer.InitOnce(ref _kernels!.IncrementalOrder, () =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalOrderKernelScan);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>(IncrementalKernels.IncrementalOrderKernelScan);
|
||||
var kernel = KernelUtil.GetKernel(loaded);
|
||||
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||
});
|
||||
@@ -134,7 +133,7 @@ public sealed class GpuKernelLease
|
||||
{
|
||||
return KernelContainer.InitOnce(ref _kernels!.Pow2ModOrder, () =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>(Pow2ModOrderKernelScan);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>(Pow2ModKernels.Pow2ModOrderKernelScan);
|
||||
var kernel = KernelUtil.GetKernel(loaded);
|
||||
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, GpuUInt128, GpuUInt128, byte, ulong, ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||
});
|
||||
@@ -151,7 +150,7 @@ public sealed class GpuKernelLease
|
||||
{
|
||||
return KernelContainer.InitOnce(ref _kernels!.SmallPrimeFactor, () =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, uint, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>(SmallPrimeFactorKernelScan);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ulong, uint, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>(SmallPrimeFactorKernels.SmallPrimeFactorKernelScan);
|
||||
var kernel = KernelUtil.GetKernel(loaded);
|
||||
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, uint, ArrayView1D<uint, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||
});
|
||||
@@ -159,211 +158,6 @@ public sealed class GpuKernelLease
|
||||
}
|
||||
}
|
||||
|
||||
private static void SmallPrimeFactorKernelScan(
|
||||
Index1D index,
|
||||
ulong value,
|
||||
uint limit,
|
||||
ArrayView1D<uint, Stride1D.Dense> primes,
|
||||
ArrayView1D<ulong, Stride1D.Dense> squares,
|
||||
int primeCount,
|
||||
ArrayView1D<ulong, Stride1D.Dense> primeSlots,
|
||||
ArrayView1D<int, Stride1D.Dense> exponentSlots,
|
||||
ArrayView1D<int, Stride1D.Dense> countSlot,
|
||||
ArrayView1D<ulong, Stride1D.Dense> remainingSlot)
|
||||
{
|
||||
if (index != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
uint effectiveLimit = limit == 0 ? uint.MaxValue : limit;
|
||||
ulong remaining = value;
|
||||
int count = 0;
|
||||
int capacity = (int)primeSlots.Length;
|
||||
|
||||
for (int i = 0; i < primeCount && remaining > 1UL && count < capacity; i++)
|
||||
{
|
||||
uint primeCandidate = primes[i];
|
||||
if (primeCandidate == 0U || primeCandidate > effectiveLimit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
ulong primeSquare = squares[i];
|
||||
if (primeSquare != 0UL && primeSquare > remaining)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
ulong primeValue = primeCandidate;
|
||||
if ((remaining % primeValue) != 0UL)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
do
|
||||
{
|
||||
remaining /= primeValue;
|
||||
exponent++;
|
||||
}
|
||||
while ((remaining % primeValue) == 0UL);
|
||||
|
||||
primeSlots[count] = primeValue;
|
||||
exponentSlots[count] = exponent;
|
||||
count++;
|
||||
}
|
||||
|
||||
countSlot[0] = count;
|
||||
remainingSlot[0] = remaining;
|
||||
}
|
||||
|
||||
// TODO: Plumb the small-cycles device buffer into all kernels that can benefit
|
||||
// (some already accept it). Consider a compact type (byte/ushort) for memory footprint.
|
||||
|
||||
private static void IncrementalKernelScan(Index1D index, ulong exponent, GpuUInt128 twoP, GpuUInt128 kStart, byte lastIsSeven, ulong divMul,
|
||||
ulong q0m10, ulong q0m8, ulong q0m3, ulong q0m5, ArrayView<ulong> orders,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallCycles)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
// TODO: Replace these `%` computations with the precomputed Mod3/Mod5 tables so GPU kernels reuse cached residues
|
||||
// instead of performing modulo operations that the benchmarks showed slower on wide sweeps.
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
// residue automaton
|
||||
ulong step10 = (exponent.Mod10() << 1).Mod10();
|
||||
ulong r10 = q0m10 + (step10 * idx).Mod10();
|
||||
r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL; // skip q ≡ 5 (mod 10); other residues handled by r8/r3/r5
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong step8 = ((exponent & 7UL) << 1) & 7UL;
|
||||
ulong r8 = (q0m8 + ((step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// TODO: Swap these `%` filters to the shared Mod helpers so the residue automaton matches the benchmarked
|
||||
// bitmask-based implementation for GPU workloads.
|
||||
ulong step3 = ((exponent % 3UL) << 1) % 3UL;
|
||||
ulong r3 = q0m3 + (step3 * idxMod3); r3 -= (r3 >= 3UL) ? 3UL : 0UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong step5Loc = ((exponent % 5UL) << 1) % 5UL;
|
||||
ulong r5 = q0m5 + (step5Loc * idxMod5); if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldCheck)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: Is this expected that the value of kStart is modified?
|
||||
GpuUInt128 k = kStart + (GpuUInt128)idx;
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: q will be lowered by GpuUInt128.One here. Is this expected? Maybe that's why we have issues with the incremental kernel? Review other places where we use GpuUInt128 operators.
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
// TODO: Do we have the check here because we don't support high values or this is expected result?
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Replace these Pow2Mod calls with the ProcessEightBitWindows helper when the shared windowed
|
||||
// scalar implementation lands; benchmarks showed the windowed kernel trimming per-divisor runtime by ~2.4×.
|
||||
if (GpuUInt128.Pow2Mod(phi64, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
orders[index] = 1UL;
|
||||
}
|
||||
|
||||
private static void OrderKernelScan(Index1D index, ulong exponent, ulong divMul, ArrayView<GpuUInt128> qs, ArrayView<ulong> orders)
|
||||
{
|
||||
GpuUInt128 q = qs[index];
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Once the ProcessEightBitWindows helper is available, switch this order kernel to that faster
|
||||
// Pow2Mod variant so cycle checks inherit the same gains observed in GpuPow2ModBenchmarks.
|
||||
GpuUInt128 pow = GpuUInt128.Pow2Mod(phi64, in readOnlyQ);
|
||||
if (pow != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
orders[index] = exponent;
|
||||
}
|
||||
|
||||
private void Dispose(bool disposing)
|
||||
{
|
||||
if (_disposed)
|
||||
@@ -390,316 +184,6 @@ public sealed class GpuKernelLease
|
||||
Pool.Enqueue(this);
|
||||
}
|
||||
|
||||
private static void Pow2ModKernelScan(Index1D index, ulong exponent, GpuUInt128 twoP, GpuUInt128 kStart, byte lastIsSeven, ulong _,
|
||||
ResidueAutomatonArgs ra, ArrayView<ulong> orders,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallCycles,
|
||||
ArrayView1D<uint, Stride1D.Dense> smallPrimesLastOne,
|
||||
ArrayView1D<uint, Stride1D.Dense> smallPrimesLastSeven,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallPrimesPow2LastOne,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallPrimesPow2LastSeven)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
ulong r10 = ra.Q0M10 + (ra.Step10 * idx).Mod10(); r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL;
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong r8 = (ra.Q0M8 + ((ra.Step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r3 = ra.Q0M3 + (ra.Step3 * idxMod3);
|
||||
if (r3 >= 6UL) r3 -= 6UL;
|
||||
if (r3 >= 3UL) r3 -= 3UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r5 = ra.Q0M5 + (ra.Step5 * idxMod5);
|
||||
if (r5 >= 15UL) r5 -= 15UL;
|
||||
if (r5 >= 10UL) r5 -= 10UL;
|
||||
if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!shouldCheck)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: kStart is modified after this. Is this expected?
|
||||
kStart.Add(idx);
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(kStart);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
// cycle should always be initialized if we're within array limit in production code
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
// TODO: Swap Pow2Minus1Mod for the eight-bit window helper once the scalar version switches;
|
||||
// benchmarks show the windowed variant cuts large-divisor scans from ~51 µs to ~21 µs.
|
||||
if (GpuUInt128.Pow2Minus1Mod(exponent, in readOnlyQ) != GpuUInt128.Zero)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ArrayView1D<uint, Stride1D.Dense> primes = lastIsSeven != 0 ? smallPrimesLastSeven : smallPrimesLastOne;
|
||||
ArrayView1D<ulong, Stride1D.Dense> primesPow2 = lastIsSeven != 0 ? smallPrimesPow2LastSeven : smallPrimesPow2LastOne;
|
||||
int primesLen = (int)primes.Length;
|
||||
for (int i = 0; i < primesLen; i++)
|
||||
{
|
||||
ulong square = primesPow2[i];
|
||||
if (new GpuUInt128(0UL, square) > q)
|
||||
{
|
||||
break;
|
||||
}
|
||||
ulong prime = primes[i];
|
||||
if (Mod128By64(q, prime) == 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
orders[index] = exponent;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong FastDiv64Gpu(ulong value, ulong divisor, ulong mul)
|
||||
{
|
||||
ulong quotient = GpuUInt128.MulHigh(value, mul);
|
||||
GpuUInt128 remainder = new(0UL, value);
|
||||
GpuUInt128 product = new(GpuUInt128.MulHigh(quotient, divisor), quotient * divisor);
|
||||
remainder.Sub(product);
|
||||
|
||||
if (remainder.High != 0UL || remainder.Low >= divisor)
|
||||
{
|
||||
quotient++;
|
||||
}
|
||||
|
||||
return quotient;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong Mod128By64(GpuUInt128 value, ulong modulus)
|
||||
{
|
||||
ulong rem = 0UL;
|
||||
ulong part = value.High;
|
||||
for (int i = 0; i < 64; i++)
|
||||
{
|
||||
rem = (rem << 1) | (part >> 63);
|
||||
if (rem >= modulus)
|
||||
{
|
||||
rem -= modulus;
|
||||
}
|
||||
part <<= 1;
|
||||
}
|
||||
|
||||
part = value.Low;
|
||||
for (int i = 0; i < 64; i++)
|
||||
{
|
||||
rem = (rem << 1) | (part >> 63);
|
||||
if (rem >= modulus)
|
||||
{
|
||||
rem -= modulus;
|
||||
}
|
||||
part <<= 1;
|
||||
}
|
||||
|
||||
return rem;
|
||||
}
|
||||
|
||||
// Device-friendly small cycle length for 2 mod divisor
|
||||
private static ulong CalculateCycleLengthSmall(ulong divisor)
|
||||
{
|
||||
if ((divisor & (divisor - 1UL)) == 0UL)
|
||||
return 1UL;
|
||||
|
||||
ulong order = 1UL, pow = 2UL;
|
||||
while (pow != 1UL)
|
||||
{
|
||||
pow <<= 1;
|
||||
if (pow > divisor)
|
||||
pow -= divisor;
|
||||
|
||||
order++;
|
||||
}
|
||||
|
||||
return order;
|
||||
}
|
||||
|
||||
private static void IncrementalOrderKernelScan(Index1D index, ulong exponent, GpuUInt128 twoP, GpuUInt128 kStart, byte lastIsSeven, ulong divMul,
|
||||
ResidueAutomatonArgs ra, ArrayView<int> found, ArrayView1D<ulong, Stride1D.Dense> smallCycles)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
ulong r10 = ra.Q0M10 + (ra.Step10 * idx).Mod10(); r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL;
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong r8 = (ra.Q0M8 + ((ra.Step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r3 = ra.Q0M3 + (ra.Step3 * idxMod3);
|
||||
if (r3 >= 6UL) r3 -= 6UL;
|
||||
if (r3 >= 3UL) r3 -= 3UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r5 = ra.Q0M5 + (ra.Step5 * idxMod5);
|
||||
if (r5 >= 15UL) r5 -= 15UL;
|
||||
if (r5 >= 10UL) r5 -= 10UL;
|
||||
if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!shouldCheck)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 k = kStart + (GpuUInt128)idx;
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Upgrade this pow2mod order kernel to the ProcessEightBitWindows helper once available so GPU residue
|
||||
// scans avoid the single-bit ladder that benchmarks found to be 2.3× slower on large exponents.
|
||||
if (GpuUInt128.Pow2Mod(phi64, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Atomic.Or(ref found[0], 1);
|
||||
}
|
||||
|
||||
private static void Pow2ModOrderKernelScan(Index1D index, ulong exponent, GpuUInt128 twoP, GpuUInt128 kStart, byte lastIsSeven, ulong _,
|
||||
ResidueAutomatonArgs ra, ArrayView<int> found, ArrayView1D<ulong, Stride1D.Dense> smallCycles)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
ulong r10 = ra.Q0M10 + (ra.Step10 * idx).Mod10(); r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL;
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong r8 = (ra.Q0M8 + ((ra.Step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r3 = ra.Q0M3 + (ra.Step3 * idxMod3);
|
||||
if (r3 >= 6UL) r3 -= 6UL;
|
||||
if (r3 >= 3UL) r3 -= 3UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r5 = ra.Q0M5 + (ra.Step5 * idxMod5);
|
||||
if (r5 >= 15UL) r5 -= 15UL;
|
||||
if (r5 >= 10UL) r5 -= 10UL;
|
||||
if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!shouldCheck)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 k = kStart + (GpuUInt128)idx;
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
// TODO: Replace this Pow2Mod check with the ProcessEightBitWindows helper once Pow2Minus1Mod adopts it;
|
||||
// residue order scans will then benefit from the same 2× speedup the benchmarked windowed kernel delivered.
|
||||
if (GpuUInt128.Pow2Mod(exponent, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Atomic.Or(ref found[0], 1);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing: false);
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# GPU Kernel Refactor Audit Progress
|
||||
|
||||
## Review Summary
|
||||
- [x] MersenneNumberDivisorByDivisorGpuTester: Confirmed kernel bindings, resource pooling, and filtering comments (including Montgomery data handling notes) remain intact with no functional changes.
|
||||
- [x] MersenneNumberDivisorGpuTester: Verified the divisibility helper, candidate bootstrap, and divisor-cycle integration preserve their original comments and TODO guidance verbatim.
|
||||
- [x] MersenneNumberIncrementalGpuTester: Checked that the incremental scan workflow, residue preparation comments, and TODO placeholders stayed untouched.
|
||||
- [x] MersenneNumberLucasLehmerGpuTester: Audited the Lucas–Lehmer kernels, TODO annotations, and GPU limiter usage to ensure they were moved verbatim.
|
||||
- [x] MersenneNumberOrderGpuTester: Confirmed residue automaton notes, TODO entries, and kernel selections remain unchanged.
|
||||
- [x] MersenneNumberResidueGpuTester: Ensured the residue scan notes about cycle integration and buffer zeroing are still present without logic edits.
|
||||
- [x] NttGpuMath: Reviewed cached kernel launchers and TODO lists to verify that only structural moves occurred.
|
||||
- [x] Pow2OddPowerTable (PrimeOrderGpuHeuristics partial): Checked the odd-power ladder implementation and comments for exact retention.
|
||||
- [x] PrimeOrderGpuHeuristics: Validated that partial factor kernels, capability overrides, and extensive TODO documentation remain verbatim.
|
||||
- [x] ReadOnlyGpuUInt128: Confirmed helper constructors and implicit conversions were untouched by the refactor.
|
||||
|
||||
## Next Step
|
||||
Audit complete for the GPU namespace after the kernel extraction. No further classes remain pending in this pass.
|
||||
@@ -0,0 +1,153 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class DivisorByDivisorKernels
|
||||
{
|
||||
public static void CheckKernel(Index1D index, ArrayView<MontgomeryDivisorData> divisors, ArrayView<ulong> exponents, ArrayView<byte> hits)
|
||||
{
|
||||
MontgomeryDivisorData divisor = divisors[index];
|
||||
ulong modulus = divisor.Modulus;
|
||||
if (modulus <= 1UL || (modulus & 1UL) == 0UL)
|
||||
{
|
||||
hits[index] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong exponent = exponents[index];
|
||||
hits[index] = ULongExtensions.Pow2MontgomeryModWindowedKernel(exponent, divisor, keepMontgomery: false) == 1UL ? (byte)1 : (byte)0;
|
||||
}
|
||||
|
||||
public static void ComputeRemainderDeltasKernel(Index1D index, ArrayView<ulong> gaps, ArrayView<byte> deltas, byte modulus)
|
||||
{
|
||||
deltas[index] = (byte)(gaps[index] % modulus);
|
||||
}
|
||||
|
||||
public static void AccumulateRemaindersKernel(
|
||||
Index1D index,
|
||||
ArrayView<byte> deltas,
|
||||
ArrayView<byte> remainders,
|
||||
byte baseRemainder,
|
||||
byte modulus)
|
||||
{
|
||||
int globalIndex = index;
|
||||
int length = (int)remainders.Length;
|
||||
if (globalIndex >= length)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
int localIndex = Group.IdxX;
|
||||
int groupSize = Group.Dimension.X;
|
||||
var shared = SharedMemory.GetDynamic<int>();
|
||||
|
||||
shared[localIndex] = deltas[globalIndex];
|
||||
Group.Barrier();
|
||||
|
||||
int offset = 1;
|
||||
int mod = modulus;
|
||||
while (offset < groupSize)
|
||||
{
|
||||
int addend = 0;
|
||||
if (localIndex >= offset)
|
||||
{
|
||||
addend = shared[localIndex - offset];
|
||||
}
|
||||
|
||||
Group.Barrier();
|
||||
|
||||
if (localIndex >= offset)
|
||||
{
|
||||
int sum = shared[localIndex] + addend;
|
||||
if (sum >= mod)
|
||||
{
|
||||
sum %= mod;
|
||||
}
|
||||
|
||||
shared[localIndex] = sum;
|
||||
}
|
||||
|
||||
Group.Barrier();
|
||||
|
||||
offset <<= 1;
|
||||
}
|
||||
|
||||
if (globalIndex == 0)
|
||||
{
|
||||
remainders[0] = baseRemainder;
|
||||
return;
|
||||
}
|
||||
|
||||
int remainder = baseRemainder + shared[localIndex];
|
||||
if (remainder >= mod)
|
||||
{
|
||||
remainder %= mod;
|
||||
}
|
||||
|
||||
remainders[globalIndex] = (byte)remainder;
|
||||
}
|
||||
|
||||
public static void EvaluateCandidateMaskKernel(
|
||||
Index1D index,
|
||||
ArrayView<byte> remainder10,
|
||||
ArrayView<byte> remainder8,
|
||||
ArrayView<byte> remainder5,
|
||||
ArrayView<byte> remainder3,
|
||||
byte lastIsSevenFlag,
|
||||
ArrayView<byte> mask)
|
||||
{
|
||||
int globalIndex = index;
|
||||
int length = (int)mask.Length;
|
||||
if (globalIndex >= length)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
byte value10 = remainder10[globalIndex];
|
||||
bool accept10;
|
||||
if (lastIsSevenFlag != 0)
|
||||
{
|
||||
accept10 = value10 == 3 || value10 == 7 || value10 == 9;
|
||||
}
|
||||
else
|
||||
{
|
||||
accept10 = value10 == 1 || value10 == 3 || value10 == 7 || value10 == 9;
|
||||
}
|
||||
|
||||
if (!accept10)
|
||||
{
|
||||
mask[globalIndex] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
byte value8 = remainder8[globalIndex];
|
||||
if (value8 != 1 && value8 != 7)
|
||||
{
|
||||
mask[globalIndex] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (remainder3[globalIndex] == 0 || remainder5[globalIndex] == 0)
|
||||
{
|
||||
mask[globalIndex] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
mask[globalIndex] = 1;
|
||||
}
|
||||
|
||||
public static void ComputeMontgomeryExponentKernel(Index1D index, MontgomeryDivisorData divisor, ArrayView<ulong> exponents, ArrayView<ulong> results)
|
||||
{
|
||||
ulong modulus = divisor.Modulus;
|
||||
if (modulus <= 1UL || (modulus & 1UL) == 0UL)
|
||||
{
|
||||
results[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong exponent = exponents[index];
|
||||
|
||||
results[index] = ULongExtensions.Pow2MontgomeryModWindowedKernel(exponent, divisor, keepMontgomery: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class DivisorCycleKernels
|
||||
{
|
||||
private const byte ByteZero = 0;
|
||||
private const byte ByteOne = 1;
|
||||
|
||||
// GPU kernel for divisor cycle calculation
|
||||
public static void GpuDivisorCycleKernel(
|
||||
Index1D index,
|
||||
ArrayView1D<ulong, Stride1D.Dense> divisors,
|
||||
ArrayView1D<ulong, Stride1D.Dense> outCycles)
|
||||
{
|
||||
int i = index.X;
|
||||
outCycles[i] = CalculateCycleLengthGpu(divisors[i]);
|
||||
}
|
||||
|
||||
// GPU-friendly version of cycle length calculation
|
||||
/// <summary>
|
||||
/// GPU-friendly cycle calculator that unrolls sixteen doubling steps; it wins the 8,388,607 benchmark and stays within ~2%
|
||||
/// of the octo loop at divisor 131,071.
|
||||
/// </summary>
|
||||
public static ulong CalculateCycleLengthGpu(ulong divisor)
|
||||
{
|
||||
if ((divisor & (divisor - 1UL)) == 0UL)
|
||||
return 1UL;
|
||||
|
||||
ulong order = 1UL;
|
||||
ulong pow = 2UL;
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool GpuStep(ref ulong pow, ulong divisor, ref ulong order)
|
||||
{
|
||||
pow += pow;
|
||||
if (pow >= divisor)
|
||||
pow -= divisor;
|
||||
|
||||
order++;
|
||||
return pow == 1UL;
|
||||
}
|
||||
|
||||
public static void GpuAdvanceDivisorCyclesKernel(
|
||||
Index1D index,
|
||||
int steps,
|
||||
ArrayView1D<ulong, Stride1D.Dense> divisors,
|
||||
ArrayView1D<ulong, Stride1D.Dense> pow,
|
||||
ArrayView1D<ulong, Stride1D.Dense> order,
|
||||
ArrayView1D<ulong, Stride1D.Dense> cycles,
|
||||
ArrayView1D<byte, Stride1D.Dense> status)
|
||||
{
|
||||
if (status[index] != ByteZero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong divisor = divisors[index];
|
||||
|
||||
if ((divisor & (divisor - 1UL)) == 0UL)
|
||||
{
|
||||
cycles[index] = 1UL;
|
||||
status[index] = ByteOne;
|
||||
pow[index] = 1UL;
|
||||
order[index] = 1UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong currentPow = pow[index];
|
||||
ulong currentOrder = order[index];
|
||||
|
||||
if (divisor <= 3UL)
|
||||
{
|
||||
while (currentPow != 1UL)
|
||||
{
|
||||
currentPow += currentPow;
|
||||
if (currentPow >= divisor)
|
||||
{
|
||||
currentPow -= divisor;
|
||||
}
|
||||
|
||||
currentOrder++;
|
||||
}
|
||||
|
||||
cycles[index] = currentOrder;
|
||||
status[index] = ByteOne;
|
||||
pow[index] = currentPow;
|
||||
order[index] = currentOrder;
|
||||
return;
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
currentPow += currentPow;
|
||||
if (currentPow >= divisor)
|
||||
{
|
||||
currentPow -= divisor;
|
||||
}
|
||||
|
||||
currentOrder++;
|
||||
|
||||
if (currentPow == 1UL)
|
||||
{
|
||||
cycles[index] = currentOrder;
|
||||
status[index] = ByteOne;
|
||||
pow[index] = currentPow;
|
||||
order[index] = currentOrder;
|
||||
return;
|
||||
}
|
||||
}
|
||||
while (--steps != 0);
|
||||
|
||||
pow[index] = currentPow;
|
||||
order[index] = currentOrder;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Algorithms;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class DivisorKernels
|
||||
{
|
||||
public static void Kernel(Index1D _, ulong exponent, ReadOnlyGpuUInt128 divisor, ArrayView<byte> result)
|
||||
{
|
||||
if (divisor.High == 0UL && divisor.Low <= 1UL)
|
||||
{
|
||||
result[0] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 baseVal;
|
||||
int x = divisor.High == 0UL
|
||||
? 63 - XMath.LeadingZeroCount(divisor.Low)
|
||||
: 127 - XMath.LeadingZeroCount(divisor.High);
|
||||
ReadOnlyGpuUInt128 readOnlyMod = divisor;
|
||||
if (x == 0)
|
||||
{
|
||||
// TODO: Swap Pow2Mod for the ProcessEightBitWindows helper once Pow2Minus1Mod adopts it;
|
||||
// GpuPow2ModBenchmarks showed the windowed kernel cutting large divisors from ~51 µs to ~21 µs.
|
||||
baseVal = GpuUInt128.Pow2Mod(exponent, in readOnlyMod);
|
||||
result[0] = baseVal.High == 0UL && baseVal.Low == 1UL ? (byte)1 : (byte)0;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong ux = (ulong)x;
|
||||
if (x >= 64)
|
||||
{
|
||||
baseVal = new GpuUInt128(1UL << (x - 64), 0UL);
|
||||
}
|
||||
else
|
||||
{
|
||||
baseVal = new GpuUInt128(0UL, 1UL << x);
|
||||
}
|
||||
|
||||
GpuUInt128 pow = baseVal;
|
||||
pow.ModPow(exponent / ux, in readOnlyMod);
|
||||
// TODO: Replace this trailing Pow2Mod call with the ProcessEightBitWindows variant once the shared
|
||||
// windowed helper lands so mixed-radix decompositions benefit from the same GPU speedup.
|
||||
var part2 = GpuUInt128.Pow2Mod(exponent % ux, in readOnlyMod);
|
||||
GpuUInt128 product = pow;
|
||||
ReadOnlyGpuUInt128 part2ReadOnly = part2.AsReadOnly();
|
||||
product.MulMod(in part2ReadOnly, in readOnlyMod);
|
||||
result[0] = product.High == 0UL && product.Low == 1UL ? (byte)1 : (byte)0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Algorithms;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class IncrementalKernels
|
||||
{
|
||||
// TODO: Plumb the small-cycles device buffer into all kernels that can benefit
|
||||
// (some already accept it). Consider a compact type (byte/ushort) for memory footprint.
|
||||
public static void IncrementalKernelScan(
|
||||
Index1D index,
|
||||
ulong exponent,
|
||||
GpuUInt128 twoP,
|
||||
GpuUInt128 kStart,
|
||||
byte lastIsSeven,
|
||||
ulong divMul,
|
||||
ulong q0m10,
|
||||
ulong q0m8,
|
||||
ulong q0m3,
|
||||
ulong q0m5,
|
||||
ArrayView<ulong> orders,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallCycles)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
// TODO: Replace these `%` computations with the precomputed Mod3/Mod5 tables so GPU kernels reuse cached residues
|
||||
// instead of performing modulo operations that the benchmarks showed slower on wide sweeps.
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
// residue automaton
|
||||
ulong step10 = (exponent.Mod10() << 1).Mod10();
|
||||
ulong r10 = q0m10 + (step10 * idx).Mod10();
|
||||
r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL; // skip q ≡ 5 (mod 10); other residues handled by r8/r3/r5
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong step8 = ((exponent & 7UL) << 1) & 7UL;
|
||||
ulong r8 = (q0m8 + ((step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// TODO: Swap these `%` filters to the shared Mod helpers so the residue automaton matches the benchmarked
|
||||
// bitmask-based implementation for GPU workloads.
|
||||
ulong step3 = ((exponent % 3UL) << 1) % 3UL;
|
||||
ulong r3 = q0m3 + (step3 * idxMod3); r3 -= (r3 >= 3UL) ? 3UL : 0UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong step5Loc = ((exponent % 5UL) << 1) % 5UL;
|
||||
ulong r5 = q0m5 + (step5Loc * idxMod5); if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldCheck)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: Is this expected that the value of kStart is modified?
|
||||
GpuUInt128 k = kStart + (GpuUInt128)idx;
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: q will be lowered by GpuUInt128.One here. Is this expected? Maybe that's why we have issues with the incremental kernel? Review other places where we use GpuUInt128 operators.
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
// TODO: Do we have the check here because we don't support high values or this is expected result?
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Replace these Pow2Mod calls with the ProcessEightBitWindows helper when the shared windowed
|
||||
// scalar implementation lands; benchmarks showed the windowed kernel trimming per-divisor runtime by ~2.4×.
|
||||
if (GpuUInt128.Pow2Mod(phi64, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = KernelMathHelpers.FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
orders[index] = 1UL;
|
||||
}
|
||||
|
||||
public static void IncrementalOrderKernelScan(
|
||||
Index1D index,
|
||||
ulong exponent,
|
||||
GpuUInt128 twoP,
|
||||
GpuUInt128 kStart,
|
||||
byte lastIsSeven,
|
||||
ulong divMul,
|
||||
ResidueAutomatonArgs ra,
|
||||
ArrayView<int> found,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallCycles)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
ulong r10 = ra.Q0M10 + (ra.Step10 * idx).Mod10(); r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL;
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong r8 = (ra.Q0M8 + ((ra.Step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r3 = ra.Q0M3 + (ra.Step3 * idxMod3);
|
||||
if (r3 >= 6UL) r3 -= 6UL;
|
||||
if (r3 >= 3UL) r3 -= 3UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r5 = ra.Q0M5 + (ra.Step5 * idxMod5);
|
||||
if (r5 >= 15UL) r5 -= 15UL;
|
||||
if (r5 >= 10UL) r5 -= 10UL;
|
||||
if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!shouldCheck)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 k = kStart + (GpuUInt128)idx;
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Upgrade this pow2mod order kernel to the ProcessEightBitWindows helper once available so GPU residue
|
||||
// scans avoid the single-bit ladder that benchmarks found to be 2.3× slower on large exponents.
|
||||
if (GpuUInt128.Pow2Mod(phi64, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = KernelMathHelpers.FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Atomic.Or(ref found[0], 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class KernelMathHelpers
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static ulong FastDiv64Gpu(ulong value, ulong divisor, ulong mul)
|
||||
{
|
||||
ulong quotient = GpuUInt128.MulHigh(value, mul);
|
||||
GpuUInt128 remainder = new(0UL, value);
|
||||
GpuUInt128 product = new(GpuUInt128.MulHigh(quotient, divisor), quotient * divisor);
|
||||
remainder.Sub(product);
|
||||
|
||||
if (remainder.High != 0UL || remainder.Low >= divisor)
|
||||
{
|
||||
quotient++;
|
||||
}
|
||||
|
||||
return quotient;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static ulong Mod128By64(GpuUInt128 value, ulong modulus)
|
||||
{
|
||||
ulong rem = 0UL;
|
||||
ulong part = value.High;
|
||||
for (int i = 0; i < 64; i++)
|
||||
{
|
||||
rem = (rem << 1) | (part >> 63);
|
||||
if (rem >= modulus)
|
||||
{
|
||||
rem -= modulus;
|
||||
}
|
||||
part <<= 1;
|
||||
}
|
||||
|
||||
part = value.Low;
|
||||
for (int i = 0; i < 64; i++)
|
||||
{
|
||||
rem = (rem << 1) | (part >> 63);
|
||||
if (rem >= modulus)
|
||||
{
|
||||
rem -= modulus;
|
||||
}
|
||||
part <<= 1;
|
||||
}
|
||||
|
||||
return rem;
|
||||
}
|
||||
|
||||
public static ulong CalculateCycleLengthSmall(ulong divisor)
|
||||
{
|
||||
if ((divisor & (divisor - 1UL)) == 0UL)
|
||||
return 1UL;
|
||||
|
||||
ulong order = 1UL, pow = 2UL;
|
||||
while (pow != 1UL)
|
||||
{
|
||||
pow <<= 1;
|
||||
if (pow > divisor)
|
||||
pow -= divisor;
|
||||
|
||||
order++;
|
||||
}
|
||||
|
||||
return order;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,452 @@
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
using UInt128 = System.UInt128;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class LucasLehmerKernels
|
||||
{
|
||||
public static void AddSmallKernel(Index1D index, ulong add, ArrayView<GpuUInt128> value)
|
||||
{
|
||||
ulong carry = add;
|
||||
for (int i = 0; i < value.Length && carry != 0UL; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong low = limb.Low + carry;
|
||||
ulong carryOut = low < carry ? 1UL : 0UL;
|
||||
ulong high = limb.High + carryOut;
|
||||
carry = high < limb.High ? 1UL : 0UL;
|
||||
value[i] = new GpuUInt128(high, low);
|
||||
}
|
||||
}
|
||||
|
||||
public static void SubtractSmallKernel(Index1D index, ulong subtract, ArrayView<GpuUInt128> value)
|
||||
{
|
||||
ulong borrow = subtract;
|
||||
for (int i = 0; i < value.Length && borrow != 0UL; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong low = limb.Low;
|
||||
ulong high = limb.High;
|
||||
if (low >= borrow)
|
||||
{
|
||||
low -= borrow;
|
||||
borrow = 0UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
low = unchecked(low - borrow);
|
||||
borrow = 1UL;
|
||||
if (high != 0UL)
|
||||
{
|
||||
high--;
|
||||
borrow = 0UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
high = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
value[i] = new GpuUInt128(high, low);
|
||||
}
|
||||
}
|
||||
|
||||
public static void ReduceModMersenneKernel(Index1D index, ulong exponent, ArrayView<GpuUInt128> value)
|
||||
{
|
||||
int limbCount = (int)((exponent + 127UL) / 128UL);
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
if (limb.IsZero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int target = i - limbCount;
|
||||
var sum = value[target];
|
||||
var original = sum;
|
||||
sum.Add(limb);
|
||||
value[target] = sum;
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
if (sum.CompareTo(original) < 0)
|
||||
{
|
||||
int j = target + 1;
|
||||
while (j < limbCount)
|
||||
{
|
||||
var next = value[j];
|
||||
var nextOriginal = next;
|
||||
next.Add(1UL);
|
||||
value[j] = next;
|
||||
if (next.CompareTo(nextOriginal) >= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int topBits = (int)exponent.Mod128();
|
||||
if (topBits == 0)
|
||||
{
|
||||
topBits = 128;
|
||||
}
|
||||
|
||||
int topIndex = limbCount - 1;
|
||||
ulong topLowMask = topBits >= 64 ? ulong.MaxValue : (1UL << topBits) - 1UL;
|
||||
ulong topHighMask = topBits > 64 ? (1UL << (topBits - 64)) - 1UL : 0UL;
|
||||
var top = value[topIndex];
|
||||
var carryBits = top >> topBits;
|
||||
top = new(top.High & topHighMask, top.Low & topLowMask);
|
||||
value[topIndex] = top;
|
||||
|
||||
if (!carryBits.IsZero)
|
||||
{
|
||||
int j = 0;
|
||||
while (j < limbCount && !carryBits.IsZero)
|
||||
{
|
||||
var cur = value[j];
|
||||
var curOriginal = cur;
|
||||
cur.Add(carryBits);
|
||||
value[j] = cur;
|
||||
carryBits = cur.CompareTo(curOriginal) < 0 ? new GpuUInt128(0UL, 1UL) : new GpuUInt128(0UL, 0UL);
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
bool geq = true;
|
||||
for (int i = topIndex; i >= 0; i--)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong modHigh = i == topIndex ? topHighMask : ulong.MaxValue;
|
||||
ulong modLow = i == topIndex ? topLowMask : ulong.MaxValue;
|
||||
if (limb.High > modHigh || (limb.High == modHigh && limb.Low > modLow))
|
||||
{
|
||||
geq = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (limb.High < modHigh || (limb.High == modHigh && limb.Low < modLow))
|
||||
{
|
||||
geq = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (geq)
|
||||
{
|
||||
ulong borrow = 0UL;
|
||||
for (int i = 0; i < limbCount - 1; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong newLow = limb.Low - ulong.MaxValue - borrow;
|
||||
borrow = limb.Low < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
ulong newHigh = limb.High - ulong.MaxValue - borrow;
|
||||
borrow = limb.High < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
value[i] = new GpuUInt128(newHigh, newLow);
|
||||
}
|
||||
|
||||
var topLimb = value[topIndex];
|
||||
ulong newTopLow = topLimb.Low - topLowMask - borrow;
|
||||
borrow = topLimb.Low < topLowMask + borrow ? 1UL : 0UL;
|
||||
ulong newTopHigh = topLimb.High - topHighMask - borrow;
|
||||
value[topIndex] = new GpuUInt128(newTopHigh, newTopLow);
|
||||
}
|
||||
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
}
|
||||
}
|
||||
|
||||
public static void IsZeroKernel(Index1D index, ArrayView<GpuUInt128> value, ArrayView<byte> result)
|
||||
{
|
||||
byte isZero = 1;
|
||||
for (int i = 0; i < value.Length; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
if (limb.High != 0UL || limb.Low != 0UL)
|
||||
{
|
||||
isZero = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result[0] = isZero;
|
||||
}
|
||||
|
||||
private static void ReduceModMersenne(Span<GpuUInt128> value, ulong exponent)
|
||||
{
|
||||
int limbCount = (int)((exponent + 127UL) / 128UL);
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
if (limb.IsZero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int target = i - limbCount;
|
||||
UInt128 original = value[target];
|
||||
UInt128 sum = original + (UInt128)limb;
|
||||
bool carry = sum < original;
|
||||
value[target] = new GpuUInt128(sum);
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
if (carry)
|
||||
{
|
||||
int j = target + 1;
|
||||
while (j < limbCount)
|
||||
{
|
||||
UInt128 next = value[j];
|
||||
UInt128 nextSum = next + 1UL;
|
||||
value[j] = new GpuUInt128(nextSum);
|
||||
if (nextSum != 0UL)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int topBits = (int)exponent.Mod128();
|
||||
if (topBits == 0)
|
||||
{
|
||||
topBits = 128;
|
||||
}
|
||||
|
||||
int topIndex = limbCount - 1;
|
||||
ulong topLowMask = topBits >= 64 ? ulong.MaxValue : (1UL << topBits) - 1UL;
|
||||
ulong topHighMask = topBits > 64 ? (1UL << (topBits - 64)) - 1UL : 0UL;
|
||||
UInt128 mask = ((UInt128)topHighMask << 64) | topLowMask;
|
||||
UInt128 top = value[topIndex];
|
||||
UInt128 carryBits = top >> topBits;
|
||||
top &= mask;
|
||||
value[topIndex] = new GpuUInt128(top);
|
||||
if (carryBits != 0UL)
|
||||
{
|
||||
int j = 0;
|
||||
UInt128 carry = carryBits;
|
||||
while (j < limbCount && carry != 0UL)
|
||||
{
|
||||
UInt128 cur = value[j];
|
||||
UInt128 sum = cur + carry;
|
||||
bool overflow = sum < cur;
|
||||
value[j] = new GpuUInt128(sum);
|
||||
carry = overflow ? 1UL : 0UL;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
bool geq = true;
|
||||
for (int i = topIndex; i >= 0; i--)
|
||||
{
|
||||
UInt128 limb = value[i];
|
||||
UInt128 modLimb = i == topIndex ? mask : UInt128.MaxValue;
|
||||
if (limb > modLimb)
|
||||
{
|
||||
geq = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (limb < modLimb)
|
||||
{
|
||||
geq = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (geq)
|
||||
{
|
||||
ulong borrow = 0UL;
|
||||
for (int i = 0; i < limbCount - 1; i++)
|
||||
{
|
||||
ulong low = value[i].Low;
|
||||
ulong high = value[i].High;
|
||||
ulong newLow = low - ulong.MaxValue - borrow;
|
||||
borrow = low < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
ulong newHigh = high - ulong.MaxValue - borrow;
|
||||
borrow = high < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
value[i] = new GpuUInt128(newHigh, newLow);
|
||||
}
|
||||
|
||||
ulong topLow = value[topIndex].Low;
|
||||
ulong topHigh = value[topIndex].High;
|
||||
ulong newTopLow = topLow - topLowMask - borrow;
|
||||
borrow = topLow < topLowMask + borrow ? 1UL : 0UL;
|
||||
ulong newTopHigh = topHigh - topHighMask - borrow;
|
||||
value[topIndex] = new GpuUInt128(newTopHigh, newTopLow);
|
||||
}
|
||||
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
}
|
||||
}
|
||||
|
||||
public static void KernelBatch(Index1D index, ArrayView<ulong> exponents, ArrayView<GpuUInt128> moduli, ArrayView<GpuUInt128> states)
|
||||
{
|
||||
int idx = index.X;
|
||||
ulong exponent = exponents[idx];
|
||||
GpuUInt128 modulus = moduli[idx];
|
||||
var s = new GpuUInt128(0UL, 4UL);
|
||||
ulong limit = exponent - 2UL;
|
||||
for (ulong i = 0UL; i < limit; i++)
|
||||
{
|
||||
s = SquareModMersenne128(s, exponent);
|
||||
s.SubMod(2UL, modulus);
|
||||
}
|
||||
|
||||
states[idx] = s;
|
||||
}
|
||||
|
||||
public static void Kernel(Index1D index, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
|
||||
{
|
||||
// Lucas–Lehmer iteration in the field GF(2^p-1).
|
||||
// For p < 128, use a Mersenne-specific squaring + reduction to avoid
|
||||
// the generic 128-bit long-division-like reduction.
|
||||
var s = new GpuUInt128(0UL, 4UL);
|
||||
ulong i = 0UL, limit = exponent - 2UL;
|
||||
if (exponent < 128UL)
|
||||
{
|
||||
for (; i < limit; i++)
|
||||
{
|
||||
s = SquareModMersenne128(s, exponent);
|
||||
s.SubMod(2UL, modulus);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (; i < limit; i++)
|
||||
{
|
||||
s.SquareMod(modulus);
|
||||
s.SubMod(2UL, modulus);
|
||||
}
|
||||
}
|
||||
|
||||
state[0] = s;
|
||||
}
|
||||
|
||||
// Squares a 128-bit value and reduces modulo M_p = 2^p - 1 using
|
||||
// Mersenne folding. Valid for 0 < p < 128 and input s in [0, M_p].
|
||||
private static GpuUInt128 SquareModMersenne128(GpuUInt128 s, ulong p)
|
||||
{
|
||||
// Full square: (H*2^64 + L)^2 = H^2*2^128 + 2HL*2^64 + L^2
|
||||
SquareFull128(s, out var q3, out var q2, out var q1, out var q0);
|
||||
|
||||
// v = low128 + high128 (fold by 128 bits once)
|
||||
var vHigh = new GpuUInt128(q3, q2);
|
||||
var vLow = new GpuUInt128(q1, q0);
|
||||
vLow.Add(vHigh);
|
||||
|
||||
// Mask to keep only p low bits and fold the carry bits (v >> p)
|
||||
int topBits = (int)p.Mod128();
|
||||
if (topBits == 0)
|
||||
{
|
||||
topBits = 128;
|
||||
}
|
||||
|
||||
ulong maskLow = topBits >= 64 ? ulong.MaxValue : (1UL << topBits) - 1UL;
|
||||
ulong maskHigh = topBits > 64 ? (1UL << (topBits - 64)) - 1UL : 0UL;
|
||||
|
||||
var carryBits = vLow >> topBits;
|
||||
vLow = new(vLow.High & maskHigh, vLow.Low & maskLow);
|
||||
|
||||
// Propagate carryBits back into the masked value (single-limb fold)
|
||||
GpuUInt128 before;
|
||||
while (!carryBits.IsZero)
|
||||
{
|
||||
before = vLow;
|
||||
vLow.Add(carryBits);
|
||||
carryBits = vLow.CompareTo(before) < 0 ? GpuUInt128.One : GpuUInt128.Zero;
|
||||
}
|
||||
|
||||
// Final correction if v >= modulus (mask)
|
||||
if (vLow.High > maskHigh || (vLow.High == maskHigh && vLow.Low >= maskLow))
|
||||
{
|
||||
ulong borrow = 0UL;
|
||||
ulong newLow = vLow.Low - maskLow;
|
||||
borrow = vLow.Low < maskLow ? 1UL : 0UL;
|
||||
ulong newHigh = vLow.High - maskHigh - borrow;
|
||||
vLow = new GpuUInt128(newHigh, newLow);
|
||||
}
|
||||
|
||||
return vLow;
|
||||
}
|
||||
|
||||
// Helper: 128-bit square -> 256-bit product as four 64-bit limbs (q3..q0)
|
||||
private static void SquareFull128(GpuUInt128 value, out ulong q3, out ulong q2, out ulong q1, out ulong q0)
|
||||
{
|
||||
ulong L = value.Low;
|
||||
ulong H = value.High;
|
||||
Mul64Parts(L, L, out var hLL, out var lLL); // L^2
|
||||
Mul64Parts(H, H, out var hHH, out var lHH); // H^2
|
||||
Mul64Parts(L, H, out var hLH, out var lLH); // L*H
|
||||
|
||||
// double LH
|
||||
ulong dLH_low = lLH << 1;
|
||||
ulong carry = (lLH >> 63) & 1UL;
|
||||
ulong dLH_high = (hLH << 1) | carry;
|
||||
|
||||
q0 = lLL;
|
||||
ulong sum1 = hLL + dLH_low;
|
||||
ulong c1 = sum1 < hLL ? 1UL : 0UL;
|
||||
q1 = sum1;
|
||||
|
||||
ulong sum2 = lHH + dLH_high;
|
||||
ulong c2 = sum2 < lHH ? 1UL : 0UL;
|
||||
sum2 += c1;
|
||||
if (sum2 < c1)
|
||||
{
|
||||
c2++;
|
||||
}
|
||||
q2 = sum2;
|
||||
q3 = hHH + c2;
|
||||
}
|
||||
|
||||
// 64x64 -> 128 multiply (high,low)
|
||||
[MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul64Parts(ulong a, ulong b, out ulong high, out ulong low)
|
||||
{
|
||||
ulong a0 = (uint)a;
|
||||
ulong a1 = a >> 32;
|
||||
ulong b0 = (uint)b;
|
||||
ulong b1 = b >> 32;
|
||||
|
||||
ulong lo = a0 * b0;
|
||||
ulong mid1 = a1 * b0;
|
||||
ulong mid2 = a0 * b1;
|
||||
ulong hi = a1 * b1;
|
||||
|
||||
ulong carry = (lo >> 32) + (uint)mid1 + (uint)mid2;
|
||||
low = (lo & 0xFFFFFFFFUL) | (carry << 32);
|
||||
hi += (mid1 >> 32) + (mid2 >> 32) + (carry >> 32);
|
||||
high = hi;
|
||||
}
|
||||
|
||||
[MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong ModPow64(ulong value, ulong exponent, ulong modulus)
|
||||
{
|
||||
ulong result = 1UL;
|
||||
ulong baseValue = value % modulus;
|
||||
ulong exp = exponent;
|
||||
|
||||
while (exp != 0UL)
|
||||
{
|
||||
if ((exp & 1UL) != 0UL)
|
||||
{
|
||||
result = (ulong)(((UInt128)result * baseValue) % modulus);
|
||||
}
|
||||
|
||||
baseValue = (ulong)(((UInt128)baseValue * baseValue) % modulus);
|
||||
exp >>= 1;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class NttBarrettKernels
|
||||
{
|
||||
// Barrett-based scaling: v = v * scale mod n for 128-bit n
|
||||
public static void ScaleBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, GpuUInt128 scale, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
|
||||
{
|
||||
var v = data[index];
|
||||
// 128x128 -> 256
|
||||
Mul128Full(v.High, v.Low, scale.High, scale.Low, out var z3, out var z2, out var z1, out var z0);
|
||||
// q approx
|
||||
Mul3x2Top(z3, z2, z1, muHigh, muLow, out var qHigh, out var qLow);
|
||||
// q*n
|
||||
Mul128Full(qHigh, qLow, modHigh, modLow, out var qq3, out var qq2, out var qq1, out var qq0);
|
||||
// r = z - q*n
|
||||
Sub256(z3, z2, z1, z0, qq3, qq2, qq1, qq0, out var r3, out var r2, out var r1, out var r0);
|
||||
ulong resHigh = r1;
|
||||
ulong resLow = r0;
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
}
|
||||
}
|
||||
|
||||
data[index] = new GpuUInt128(resHigh, resLow);
|
||||
}
|
||||
|
||||
// Barrett-based squaring: v = v^2 mod n for 128-bit n
|
||||
public static void SquareBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
|
||||
{
|
||||
var v = data[index];
|
||||
Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0);
|
||||
Mul3x2Top(z3, z2, z1, muHigh, muLow, out var qHigh, out var qLow);
|
||||
Mul128Full(qHigh, qLow, modHigh, modLow, out var qq3, out var qq2, out var qq1, out var qq0);
|
||||
Sub256(z3, z2, z1, z0, qq3, qq2, qq1, qq0, out var r3, out var r2, out var r1, out var r0);
|
||||
ulong resHigh = r1;
|
||||
ulong resLow = r0;
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
}
|
||||
}
|
||||
data[index] = new GpuUInt128(resHigh, resLow);
|
||||
}
|
||||
|
||||
// Barrett reduction based stage for 128-bit moduli (no '%').
|
||||
public static void StageBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, int len, int half, int stageOffset, ArrayView<GpuUInt128> twiddles, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
|
||||
{
|
||||
int t = index.X;
|
||||
// TODO: Use the bitmask-based remainder helper here as well to remove `%` from the butterfly stage and align with the
|
||||
// optimized kernels highlighted in the GPU pow2mod benchmarks.
|
||||
int j = t % half;
|
||||
int block = t / half;
|
||||
int k = block * len;
|
||||
int i1 = k + j;
|
||||
int i2 = i1 + half;
|
||||
var u = data[i1];
|
||||
var v = data[i2];
|
||||
var w = twiddles[stageOffset + j];
|
||||
|
||||
// Compute v * w (128x128) -> 256-bit z3..z0
|
||||
Mul128Full(v.High, v.Low, w.High, w.Low, out var z3, out var z2, out var z1, out var z0);
|
||||
|
||||
// t = floor(z / b) where b=2^64 => take (z3,z2,z1)
|
||||
// Compute q = floor((t * mu) / b^3). Only need top two limbs of 5-limb product.
|
||||
Mul3x2Top(z3, z2, z1, muHigh, muLow, out var qHigh, out var qLow);
|
||||
|
||||
// q * n (128x128) -> 256-bit qq3..qq0
|
||||
Mul128Full(qHigh, qLow, modHigh, modLow, out var qq3, out var qq2, out var qq1, out var qq0);
|
||||
|
||||
// r = z - q*n (256-bit)
|
||||
Sub256(z3, z2, z1, z0, qq3, qq2, qq1, qq0, out var r3, out var r2, out var r1, out var r0);
|
||||
|
||||
// Reduce r to [0, n) by at most two subtractions
|
||||
// Discard higher limbs (should be zero if r < 2n), keep low 128-bit
|
||||
ulong resHigh = r1; // r1 is limb1 after subtraction (since lower two limbs are r1 (high) and r0 (low))
|
||||
ulong resLow = r0;
|
||||
|
||||
// while (res >= n) res -= n; (at most twice)
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
}
|
||||
}
|
||||
|
||||
var sum = new GpuUInt128(u);
|
||||
var mulred = new GpuUInt128(resHigh, resLow);
|
||||
sum.AddMod(mulred, new GpuUInt128(modHigh, modLow));
|
||||
u.SubMod(mulred, new GpuUInt128(modHigh, modLow));
|
||||
data[i1] = sum;
|
||||
data[i2] = u;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul128Full(ulong aHigh, ulong aLow, ulong bHigh, ulong bLow, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
{
|
||||
// Full 128x128 -> 256 multiply using 64-bit limbs
|
||||
Mul64Parts(aLow, bLow, out var h0, out var l0);
|
||||
Mul64Parts(aLow, bHigh, out var h1, out var l1);
|
||||
Mul64Parts(aHigh, bLow, out var h2, out var l2);
|
||||
Mul64Parts(aHigh, bHigh, out var h3, out var l3);
|
||||
|
||||
p0 = l0;
|
||||
ulong carry = 0UL;
|
||||
ulong sum = h0;
|
||||
sum += l1; if (sum < l1) carry++;
|
||||
sum += l2; if (sum < l2) carry++;
|
||||
p1 = sum;
|
||||
|
||||
sum = h1;
|
||||
sum += h2; ulong carry2 = sum < h2 ? 1UL : 0UL;
|
||||
sum += l3; if (sum < l3) carry2++;
|
||||
sum += carry; if (sum < carry) carry2++;
|
||||
p2 = sum;
|
||||
p3 = h3 + carry2;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul3x2Top(ulong t2, ulong t1, ulong t0, ulong muHigh, ulong muLow, out ulong top1, out ulong top0)
|
||||
{
|
||||
// Compute top two limbs of ( (t2,t1,t0) * (muHigh,muLow) ) after shifting right by 192 bits.
|
||||
// We build full 5 limbs and return p4 (top1) and p3 (top0).
|
||||
ulong p0 = 0, p1 = 0, p2 = 0, p3 = 0, p4 = 0;
|
||||
|
||||
void Add128To(ref ulong lo, ref ulong hi, ulong addLo, ulong addHi)
|
||||
{
|
||||
ulong s = lo + addLo;
|
||||
ulong c = s < lo ? 1UL : 0UL;
|
||||
lo = s;
|
||||
ulong s2 = hi + addHi + c;
|
||||
hi = s2;
|
||||
}
|
||||
|
||||
// t0 * muLow -> contributes to p0,p1
|
||||
Mul64Parts(t0, muLow, out var h, out var l);
|
||||
Add128To(ref p0, ref p1, l, h);
|
||||
|
||||
// t0 * muHigh -> to p1,p2
|
||||
Mul64Parts(t0, muHigh, out h, out l);
|
||||
Add128To(ref p1, ref p2, l, h);
|
||||
|
||||
// t1 * muLow -> to p1,p2
|
||||
Mul64Parts(t1, muLow, out h, out l);
|
||||
Add128To(ref p1, ref p2, l, h);
|
||||
|
||||
// t1 * muHigh -> to p2,p3
|
||||
Mul64Parts(t1, muHigh, out h, out l);
|
||||
Add128To(ref p2, ref p3, l, h);
|
||||
|
||||
// t2 * muLow -> to p2,p3
|
||||
Mul64Parts(t2, muLow, out h, out l);
|
||||
Add128To(ref p2, ref p3, l, h);
|
||||
|
||||
// t2 * muHigh -> to p3,p4
|
||||
Mul64Parts(t2, muHigh, out h, out l);
|
||||
Add128To(ref p3, ref p4, l, h);
|
||||
|
||||
top1 = p4; top0 = p3;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Sub256(ulong a3, ulong a2, ulong a1, ulong a0, ulong b3, ulong b2, ulong b1, ulong b0, out ulong r3, out ulong r2, out ulong r1, out ulong r0)
|
||||
{
|
||||
ulong borrow = 0;
|
||||
r0 = SubWithBorrow(a0, b0, ref borrow);
|
||||
r1 = SubWithBorrow(a1, b1, ref borrow);
|
||||
r2 = SubWithBorrow(a2, b2, ref borrow);
|
||||
r3 = SubWithBorrow(a3, b3, ref borrow);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong SubWithBorrow(ulong a, ulong b, ref ulong borrow)
|
||||
{
|
||||
ulong res = a - b - borrow;
|
||||
borrow = ((a ^ b) & (a ^ res) & 0x8000_0000_0000_0000UL) != 0 ? 1UL : (a < b + borrow ? 1UL : 0UL);
|
||||
return res;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool Geq128(ulong aHigh, ulong aLow, ulong bHigh, ulong bLow)
|
||||
{
|
||||
return aHigh > bHigh || (aHigh == bHigh && aLow >= bLow);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Sub128(ref ulong aHigh, ref ulong aLow, ulong bHigh, ulong bLow)
|
||||
{
|
||||
ulong borrow = 0;
|
||||
ulong low = aLow - bLow;
|
||||
borrow = aLow < bLow ? 1UL : 0UL;
|
||||
ulong high = aHigh - bHigh - borrow;
|
||||
aHigh = high; aLow = low;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul64Parts(ulong a, ulong b, out ulong high, out ulong low)
|
||||
{
|
||||
// 64x64 -> 128 using 32-bit limbs
|
||||
ulong a0 = (uint)a;
|
||||
ulong a1 = a >> 32;
|
||||
ulong b0 = (uint)b;
|
||||
ulong b1 = b >> 32;
|
||||
|
||||
ulong lo = a0 * b0;
|
||||
ulong mid1 = a1 * b0;
|
||||
ulong mid2 = a0 * b1;
|
||||
ulong hi = a1 * b1;
|
||||
|
||||
ulong carry = (lo >> 32) + (uint)mid1 + (uint)mid2;
|
||||
low = (lo & 0xFFFFFFFFUL) | (carry << 32);
|
||||
hi += (mid1 >> 32) + (mid2 >> 32) + (carry >> 32);
|
||||
high = hi;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using ILGPU;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class NttButterflyKernels
|
||||
{
|
||||
public static void StageKernel(Index1D index, ArrayView<GpuUInt128> data, int len, int half, int stageOffset, ArrayView<GpuUInt128> twiddles, GpuUInt128 modulus)
|
||||
{
|
||||
// TODO(NTT-OPT): Consider shared-memory tiling to improve locality:
|
||||
// load a block of size `len` into shared memory, do butterflies, write back.
|
||||
// Requires explicit grouped kernels and chosen group size.
|
||||
int t = index.X;
|
||||
// TODO: Replace this `%` with a bitmask when `half` is a power of two so the stage index math matches the faster
|
||||
// residue strategy from the GPU residue benchmarks.
|
||||
int j = t % half;
|
||||
int block = t / half;
|
||||
int k = block * len;
|
||||
int i1 = k + j;
|
||||
int i2 = i1 + half;
|
||||
var u = data[i1];
|
||||
var v = data[i2];
|
||||
var w = twiddles[stageOffset + j];
|
||||
v.MulMod(w, modulus);
|
||||
var sum = new GpuUInt128(u);
|
||||
sum.AddMod(v, modulus);
|
||||
u.SubMod(v, modulus);
|
||||
data[i1] = sum;
|
||||
data[i2] = u;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class NttMontgomeryKernels
|
||||
{
|
||||
// TODO(MOD-OPT): Stage kernel using 64-bit Montgomery multiplication. Requires data and twiddles in Montgomery domain.
|
||||
public static void StageMontKernel(Index1D index, ArrayView<GpuUInt128> data, int len, int half, int stageOffset, ArrayView<GpuUInt128> twiddlesMont, ulong modulus, ulong nPrime)
|
||||
{
|
||||
int t = index.X;
|
||||
// TODO: Apply the bitmask remainder helper to this stage too so every butterfly path drops the slower `%` operation.
|
||||
int j = t % half;
|
||||
int block = t / half;
|
||||
int k = block * len;
|
||||
int i1 = k + j;
|
||||
int i2 = i1 + half;
|
||||
ulong u = data[i1].Low;
|
||||
ulong v = data[i2].Low;
|
||||
ulong wR = twiddlesMont[stageOffset + j].Low;
|
||||
v = MontMul64(v, wR, modulus, nPrime);
|
||||
ulong sum = u + v;
|
||||
if (sum >= modulus)
|
||||
{
|
||||
sum -= modulus;
|
||||
}
|
||||
ulong diff = u >= v ? u - v : unchecked(u + modulus - v);
|
||||
data[i1] = new GpuUInt128(0UL, sum);
|
||||
data[i2] = new GpuUInt128(0UL, diff);
|
||||
}
|
||||
|
||||
public static void ToMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
|
||||
{
|
||||
ulong a = data[index].Low;
|
||||
ulong v = MontMul64(a, r2, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
public static void FromMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
|
||||
{
|
||||
ulong aR = data[index].Low;
|
||||
ulong v = MontMul64(aR, 1UL, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
public static void SquareMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
|
||||
{
|
||||
ulong aR = data[index].Low;
|
||||
ulong v = MontMul64(aR, aR, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
public static void ScaleMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong scaleMont)
|
||||
{
|
||||
ulong aR = data[index].Low;
|
||||
ulong v = MontMul64(aR, scaleMont, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
// Computes Montgomery multiplication for 64-bit operands in Montgomery domain.
|
||||
// Returns a*b*R^{-1} mod modulus, where R=2^64 and nPrime = -modulus^{-1} mod 2^64.
|
||||
private static ulong MontMul64(ulong aR, ulong bR, ulong modulus, ulong nPrime)
|
||||
{
|
||||
// t = aR * bR (128-bit)
|
||||
ulong tLow, tHigh;
|
||||
Mul64Parts(aR, bR, out tHigh, out tLow);
|
||||
// m = (tLow * nPrime) mod 2^64 -> just low 64 bits of the product
|
||||
ulong mLow, mHigh;
|
||||
Mul64Parts(tLow, nPrime, out mHigh, out mLow);
|
||||
// u = (t + m*modulus) >> 64
|
||||
ulong mmLow, mmHigh;
|
||||
Mul64Parts(mLow, modulus, out mmHigh, out mmLow);
|
||||
// add tLow + mmLow, keep carry
|
||||
ulong carry = 0UL;
|
||||
ulong low = tLow + mmLow;
|
||||
if (low < tLow)
|
||||
{
|
||||
carry = 1UL;
|
||||
}
|
||||
// high = tHigh + mmHigh + carry
|
||||
ulong high = tHigh + mmHigh + carry;
|
||||
ulong u = high; // this is (t + m*n) >> 64
|
||||
if (u >= modulus)
|
||||
{
|
||||
u -= modulus;
|
||||
}
|
||||
return u;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul64Parts(ulong a, ulong b, out ulong high, out ulong low)
|
||||
{
|
||||
// 64x64 -> 128 using 32-bit limbs
|
||||
ulong a0 = (uint)a;
|
||||
ulong a1 = a >> 32;
|
||||
ulong b0 = (uint)b;
|
||||
ulong b1 = b >> 32;
|
||||
|
||||
ulong lo = a0 * b0;
|
||||
ulong mid1 = a1 * b0;
|
||||
ulong mid2 = a0 * b1;
|
||||
ulong hi = a1 * b1;
|
||||
|
||||
ulong carry = (lo >> 32) + (uint)mid1 + (uint)mid2;
|
||||
low = (lo & 0xFFFFFFFFUL) | (carry << 32);
|
||||
hi += (mid1 >> 32) + (mid2 >> 32) + (carry >> 32);
|
||||
high = hi;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using ILGPU;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class NttPointwiseKernels
|
||||
{
|
||||
public static void MulKernel(Index1D index, ArrayView<GpuUInt128> a, ArrayView<GpuUInt128> b, GpuUInt128 modulus)
|
||||
{
|
||||
var val = a[index];
|
||||
val.MulMod(b[index], modulus);
|
||||
a[index] = val;
|
||||
}
|
||||
|
||||
public static void ScaleKernel(Index1D index, ArrayView<GpuUInt128> data, GpuUInt128 scale, GpuUInt128 modulus)
|
||||
{
|
||||
var v = data[index];
|
||||
v.MulMod(scale, modulus);
|
||||
data[index] = v;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using ILGPU;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class NttTransformKernels
|
||||
{
|
||||
public static void ForwardKernel(Index1D index, ArrayView<GpuUInt128> input, ArrayView<GpuUInt128> output, int length, GpuUInt128 modulus, GpuUInt128 root)
|
||||
{
|
||||
// TODO(NTT-OPT): Replace this reference O(n^2) kernel with a stage-wise
|
||||
// Cooley–Tukey butterfly kernel (O(n log n)). This version uses a per-
|
||||
// element ModPow in the hot loop which is far too slow and causes long
|
||||
// single-kernel runtimes. After twiddle precomputation, each butterfly
|
||||
// should perform: (u, v) -> (u+v*w, u-v*w) with a single MulMod.
|
||||
var sum = new GpuUInt128(0UL, 0UL);
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
var term = input[i];
|
||||
var power = new GpuUInt128(root);
|
||||
power.ModPow((ulong)index.X * (ulong)i, modulus);
|
||||
term.MulMod(power, modulus);
|
||||
sum.AddMod(term, modulus);
|
||||
}
|
||||
|
||||
output[index] = sum;
|
||||
}
|
||||
|
||||
public static void InverseKernel(Index1D index, ArrayView<GpuUInt128> input, ArrayView<GpuUInt128> output, int length, GpuUInt128 modulus, GpuUInt128 rootInv, GpuUInt128 nInv)
|
||||
{
|
||||
// TODO(NTT-OPT): Replace this O(n^2) inverse with stage-wise butterflies
|
||||
// using precomputed inverse twiddles. Apply final scaling by nInv after
|
||||
// all stages, preferably in a small dedicated kernel.
|
||||
var sum = new GpuUInt128(0UL, 0UL);
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
var term = input[i];
|
||||
var power = new GpuUInt128(rootInv);
|
||||
power.ModPow((ulong)index.X * (ulong)i, modulus);
|
||||
term.MulMod(power, modulus);
|
||||
sum.AddMod(term, modulus);
|
||||
}
|
||||
|
||||
sum.MulMod(nInv, modulus);
|
||||
output[index] = sum;
|
||||
}
|
||||
|
||||
public static void BitReverseKernel(Index1D index, ArrayView<GpuUInt128> data, int bits)
|
||||
{
|
||||
int i = index.X;
|
||||
int j = ReverseBits(i, bits);
|
||||
if (i < j)
|
||||
{
|
||||
var tmp = data[i];
|
||||
data[i] = data[j];
|
||||
data[j] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
internal static int ReverseBits(int value, int bits)
|
||||
{
|
||||
int result = 0;
|
||||
for (int i = 0; i < bits; i++)
|
||||
{
|
||||
result = (result << 1) | (value & 1);
|
||||
value >>= 1;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using System.Numerics;
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
using PeterO.Numbers;
|
||||
using Open.Numeric.Primes;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class NumericProbeKernels
|
||||
{
|
||||
public static void OpenNumericPrimesIsPrimeKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
ulong value = (ulong)(index + 2);
|
||||
bool isPrime = Prime.Numbers.IsPrime(value);
|
||||
data[index] = isPrime ? 1 : 0;
|
||||
}
|
||||
|
||||
public static void BigIntegerKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
BigInteger value = new BigInteger(data[index]);
|
||||
value += BigInteger.One;
|
||||
value -= BigInteger.One;
|
||||
value *= new BigInteger(2);
|
||||
value /= new BigInteger(3);
|
||||
value %= new BigInteger(5);
|
||||
int comparison = value.CompareTo(BigInteger.Zero);
|
||||
data[index] = comparison;
|
||||
}
|
||||
|
||||
public static void EIntegerKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
EInteger value = EInteger.FromInt32(data[index]);
|
||||
value = value + EInteger.One;
|
||||
value = value - EInteger.One;
|
||||
value = value * EInteger.FromInt32(2);
|
||||
value = value.Divide(EInteger.FromInt32(3));
|
||||
value = value.Remainder(EInteger.FromInt32(5));
|
||||
int comparison = value.CompareTo(EInteger.Zero);
|
||||
data[index] = comparison;
|
||||
}
|
||||
|
||||
public static void ERationalKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
ERational value = ERational.FromInt32(data[index]);
|
||||
ERational adjusted = (value + ERational.FromInt32(1)) - ERational.FromInt32(1);
|
||||
ERational product = adjusted * ERational.FromInt32(2);
|
||||
ERational quotient = product / ERational.FromInt32(3);
|
||||
int comparison = quotient.CompareTo(ERational.FromInt32(0));
|
||||
data[index] = comparison;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class OrderKernels
|
||||
{
|
||||
public static void OrderKernelScan(
|
||||
Index1D index,
|
||||
ulong exponent,
|
||||
ulong divMul,
|
||||
ArrayView<GpuUInt128> qs,
|
||||
ArrayView<ulong> orders)
|
||||
{
|
||||
GpuUInt128 q = qs[index];
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Once the ProcessEightBitWindows helper is available, switch this order kernel to that faster
|
||||
// Pow2Mod variant so cycle checks inherit the same gains observed in GpuPow2ModBenchmarks.
|
||||
GpuUInt128 pow = GpuUInt128.Pow2Mod(phi64, in readOnlyQ);
|
||||
if (pow != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = KernelMathHelpers.FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
orders[index] = exponent;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Algorithms;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class Pow2ModKernels
|
||||
{
|
||||
public static void Pow2ModKernelScan(
|
||||
Index1D index,
|
||||
ulong exponent,
|
||||
GpuUInt128 twoP,
|
||||
GpuUInt128 kStart,
|
||||
byte lastIsSeven,
|
||||
ulong _,
|
||||
ResidueAutomatonArgs ra,
|
||||
ArrayView<ulong> orders,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallCycles,
|
||||
ArrayView1D<uint, Stride1D.Dense> smallPrimesLastOne,
|
||||
ArrayView1D<uint, Stride1D.Dense> smallPrimesLastSeven,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallPrimesPow2LastOne,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallPrimesPow2LastSeven)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
ulong r10 = ra.Q0M10 + (ra.Step10 * idx).Mod10(); r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL;
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong r8 = (ra.Q0M8 + ((ra.Step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r3 = ra.Q0M3 + (ra.Step3 * idxMod3);
|
||||
if (r3 >= 6UL) r3 -= 6UL;
|
||||
if (r3 >= 3UL) r3 -= 3UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r5 = ra.Q0M5 + (ra.Step5 * idxMod5);
|
||||
if (r5 >= 15UL) r5 -= 15UL;
|
||||
if (r5 >= 10UL) r5 -= 10UL;
|
||||
if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!shouldCheck)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: kStart is modified after this. Is this expected?
|
||||
kStart.Add(idx);
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(kStart);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
// cycle should always be initialized if we're within array limit in production code
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
// TODO: Swap Pow2Minus1Mod for the eight-bit window helper once the scalar version switches;
|
||||
// benchmarks show the windowed variant cuts large-divisor scans from ~51 µs to ~21 µs.
|
||||
if (GpuUInt128.Pow2Minus1Mod(exponent, in readOnlyQ) != GpuUInt128.Zero)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ArrayView1D<uint, Stride1D.Dense> primes = lastIsSeven != 0 ? smallPrimesLastSeven : smallPrimesLastOne;
|
||||
ArrayView1D<ulong, Stride1D.Dense> primesPow2 = lastIsSeven != 0 ? smallPrimesPow2LastSeven : smallPrimesPow2LastOne;
|
||||
int primesLen = (int)primes.Length;
|
||||
for (int i = 0; i < primesLen; i++)
|
||||
{
|
||||
ulong square = primesPow2[i];
|
||||
if (new GpuUInt128(0UL, square) > q)
|
||||
{
|
||||
break;
|
||||
}
|
||||
ulong prime = primes[i];
|
||||
if (KernelMathHelpers.Mod128By64(q, prime) == 0UL)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
orders[index] = exponent;
|
||||
}
|
||||
|
||||
public static void Pow2ModOrderKernelScan(
|
||||
Index1D index,
|
||||
ulong exponent,
|
||||
GpuUInt128 twoP,
|
||||
GpuUInt128 kStart,
|
||||
byte lastIsSeven,
|
||||
ulong _,
|
||||
ResidueAutomatonArgs ra,
|
||||
ArrayView<int> found,
|
||||
ArrayView1D<ulong, Stride1D.Dense> smallCycles)
|
||||
{
|
||||
ulong idx = (ulong)index.X;
|
||||
ulong idxMod3 = idx % 3UL;
|
||||
ulong idxMod5 = idx % 5UL;
|
||||
ulong r10 = ra.Q0M10 + (ra.Step10 * idx).Mod10(); r10 -= (r10 >= 10UL) ? 10UL : 0UL;
|
||||
bool shouldCheck = r10 != 5UL;
|
||||
if (shouldCheck)
|
||||
{
|
||||
ulong r8 = (ra.Q0M8 + ((ra.Step8 * idx) & 7UL)) & 7UL;
|
||||
if (r8 != 1UL && r8 != 7UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r3 = ra.Q0M3 + (ra.Step3 * idxMod3);
|
||||
if (r3 >= 6UL) r3 -= 6UL;
|
||||
if (r3 >= 3UL) r3 -= 3UL;
|
||||
if (r3 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong r5 = ra.Q0M5 + (ra.Step5 * idxMod5);
|
||||
if (r5 >= 15UL) r5 -= 15UL;
|
||||
if (r5 >= 10UL) r5 -= 10UL;
|
||||
if (r5 >= 5UL) r5 -= 5UL;
|
||||
if (r5 == 0UL)
|
||||
{
|
||||
shouldCheck = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!shouldCheck)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 k = kStart + (GpuUInt128)idx;
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
if (cycle != 0UL && cycle <= exponent && (exponent % cycle) != 0UL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
// TODO: Replace this Pow2Mod check with the ProcessEightBitWindows helper once Pow2Minus1Mod adopts it;
|
||||
// residue order scans will then benefit from the same 2× speedup the benchmarked windowed kernel delivered.
|
||||
if (GpuUInt128.Pow2Mod(exponent, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Atomic.Or(ref found[0], 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class Pow2MontgomeryKernels
|
||||
{
|
||||
public static void Pow2MontgomeryKernel(
|
||||
Index1D index,
|
||||
ArrayView1D<ulong, Stride1D.Dense> exponents,
|
||||
MontgomeryDivisorData divisor,
|
||||
byte keepMontgomery,
|
||||
ArrayView1D<ulong, Stride1D.Dense> results)
|
||||
{
|
||||
results[index] = ULongExtensions.Pow2MontgomeryModWindowedKernel(exponents[index], divisor, keepMontgomery != 0);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,99 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
using ILGPU.Algorithms;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class PrimeTesterKernels
|
||||
{
|
||||
// GPU kernel: small-prime sieve only. Returns 1 if passes sieve (probable prime), 0 otherwise.
|
||||
public static void SmallPrimeSieveKernel(Index1D index, ArrayView<ulong> numbers, ArrayView<uint> smallPrimes, ArrayView<byte> results)
|
||||
{
|
||||
ulong n = numbers[index];
|
||||
if (n <= 3UL)
|
||||
{
|
||||
results[index] = (byte)(n >= 2UL ? 1 : 0);
|
||||
return;
|
||||
}
|
||||
|
||||
if ((n & 1UL) == 0UL || (n > 5UL && (n % 5UL) == 0UL))
|
||||
{
|
||||
// TODO: Replace the `% 5` branch with the GPU Mod5 helper once the sieve kernel can
|
||||
// reuse the benchmarked bitmask reduction instead of modulo on every candidate.
|
||||
results[index] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (n.Mod10() == 1UL)
|
||||
{
|
||||
// Early reject special GCD heuristic with floor(log2 n)
|
||||
ulong m = 63UL - (ulong)XMath.LeadingZeroCount(n);
|
||||
if (BinaryGcdGpu(n, m) != 1UL)
|
||||
{
|
||||
results[index] = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
long len = smallPrimes.Length;
|
||||
for (int i = 0; i < len; i++)
|
||||
{
|
||||
ulong p = smallPrimes[i];
|
||||
if (p * p > n)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if ((n % p) == 0UL)
|
||||
{
|
||||
// TODO: Route this modulo through the shared divisor-cycle cache once exposed to GPU kernels
|
||||
// so batched sieves avoid per-prime `%` operations that profiling showed expensive.
|
||||
results[index] = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
results[index] = 1;
|
||||
}
|
||||
|
||||
public static void SharesFactorKernel(Index1D index, ArrayView<ulong> numbers, ArrayView<byte> results)
|
||||
{
|
||||
ulong n = numbers[index];
|
||||
ulong m = 63UL - (ulong)XMath.LeadingZeroCount(n);
|
||||
ulong gcd = BinaryGcdGpu(n, m);
|
||||
results[index] = gcd == 1UL ? (byte)0 : (byte)1;
|
||||
}
|
||||
|
||||
private static ulong BinaryGcdGpu(ulong u, ulong v)
|
||||
{
|
||||
// TODO: Replace this inline GPU binary GCD with the kernel extracted from
|
||||
// GpuUInt128BinaryGcdBenchmarks via GpuKernelPool so device callers reuse the
|
||||
// fully unrolled ladder instead of this branchy fallback.
|
||||
if (u == 0UL)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
|
||||
if (v == 0UL)
|
||||
{
|
||||
return u;
|
||||
}
|
||||
|
||||
int shift = XMath.TrailingZeroCount(u | v);
|
||||
u >>= XMath.TrailingZeroCount(u);
|
||||
|
||||
do
|
||||
{
|
||||
v >>= XMath.TrailingZeroCount(v);
|
||||
if (u > v)
|
||||
{
|
||||
(u, v) = (v, u);
|
||||
}
|
||||
|
||||
v -= u;
|
||||
}
|
||||
while (v != 0UL);
|
||||
|
||||
return u << shift;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
internal static class SmallPrimeFactorKernels
|
||||
{
|
||||
public static void SmallPrimeFactorKernelScan(
|
||||
Index1D index,
|
||||
ulong value,
|
||||
uint limit,
|
||||
ArrayView1D<uint, Stride1D.Dense> primes,
|
||||
ArrayView1D<ulong, Stride1D.Dense> squares,
|
||||
int primeCount,
|
||||
ArrayView1D<ulong, Stride1D.Dense> primeSlots,
|
||||
ArrayView1D<int, Stride1D.Dense> exponentSlots,
|
||||
ArrayView1D<int, Stride1D.Dense> countSlot,
|
||||
ArrayView1D<ulong, Stride1D.Dense> remainingSlot)
|
||||
{
|
||||
if (index != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
uint effectiveLimit = limit == 0 ? uint.MaxValue : limit;
|
||||
ulong remaining = value;
|
||||
int count = 0;
|
||||
int capacity = (int)primeSlots.Length;
|
||||
|
||||
for (int i = 0; i < primeCount && remaining > 1UL && count < capacity; i++)
|
||||
{
|
||||
uint primeCandidate = primes[i];
|
||||
if (primeCandidate == 0U || primeCandidate > effectiveLimit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
ulong primeSquare = squares[i];
|
||||
if (primeSquare != 0UL && primeSquare > remaining)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
ulong primeValue = primeCandidate;
|
||||
if ((remaining % primeValue) != 0UL)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int exponent = 0;
|
||||
do
|
||||
{
|
||||
remaining /= primeValue;
|
||||
exponent++;
|
||||
}
|
||||
while ((remaining % primeValue) == 0UL);
|
||||
|
||||
primeSlots[count] = primeValue;
|
||||
exponentSlots[count] = exponent;
|
||||
count++;
|
||||
}
|
||||
|
||||
countSlot[0] = count;
|
||||
remainingSlot[0] = remaining;
|
||||
}
|
||||
}
|
||||
@@ -28,23 +28,23 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
private readonly ConcurrentBag<DivisorScanSession> _sessionPool = new();
|
||||
|
||||
private Action<Index1D, ArrayView<MontgomeryDivisorData>, ArrayView<ulong>, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
|
||||
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<MontgomeryDivisorData>, ArrayView<ulong>, ArrayView<byte>>(CheckKernel));
|
||||
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<MontgomeryDivisorData>, ArrayView<ulong>, ArrayView<byte>>(DivisorByDivisorKernels.CheckKernel));
|
||||
|
||||
private Action<Index1D, MontgomeryDivisorData, ArrayView<ulong>, ArrayView<ulong>> GetKernelByPrimeExponent(Accelerator accelerator) =>
|
||||
_kernelExponentCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, MontgomeryDivisorData, ArrayView<ulong>, ArrayView<ulong>>(ComputeMontgomeryExponentKernel));
|
||||
_kernelExponentCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, MontgomeryDivisorData, ArrayView<ulong>, ArrayView<ulong>>(DivisorByDivisorKernels.ComputeMontgomeryExponentKernel));
|
||||
|
||||
private Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte> GetRemainderDeltaKernel(Accelerator accelerator) =>
|
||||
_remainderDeltaKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>, byte>(ComputeRemainderDeltasKernel));
|
||||
_remainderDeltaKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>, byte>(DivisorByDivisorKernels.ComputeRemainderDeltasKernel));
|
||||
|
||||
private Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<byte>, byte, byte> GetRemainderScanKernel(Accelerator accelerator) =>
|
||||
_remainderScanKernelCache.GetOrAdd(
|
||||
accelerator,
|
||||
acc => acc.LoadStreamKernel<Index1D, ArrayView<byte>, ArrayView<byte>, byte, byte>(AccumulateRemaindersKernel));
|
||||
acc => acc.LoadStreamKernel<Index1D, ArrayView<byte>, ArrayView<byte>, byte, byte>(DivisorByDivisorKernels.AccumulateRemaindersKernel));
|
||||
|
||||
private Action<Index1D, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>, byte, ArrayView<byte>> GetCandidateMaskKernel(Accelerator accelerator) =>
|
||||
_candidateMaskKernelCache.GetOrAdd(
|
||||
accelerator,
|
||||
acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>, byte, ArrayView<byte>>(EvaluateCandidateMaskKernel));
|
||||
acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>, ArrayView<byte>, byte, ArrayView<byte>>(DivisorByDivisorKernels.EvaluateCandidateMaskKernel));
|
||||
|
||||
public int GpuBatchSize
|
||||
{
|
||||
@@ -535,20 +535,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
return Math.Min((1UL << (int)(prime - 1UL)) - 1UL, divisorLimit);
|
||||
}
|
||||
|
||||
private static void CheckKernel(Index1D index, ArrayView<MontgomeryDivisorData> divisors, ArrayView<ulong> exponents, ArrayView<byte> hits)
|
||||
{
|
||||
MontgomeryDivisorData divisor = divisors[index];
|
||||
ulong modulus = divisor.Modulus;
|
||||
if (modulus <= 1UL || (modulus & 1UL) == 0UL)
|
||||
{
|
||||
hits[index] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong exponent = exponents[index];
|
||||
hits[index] = ULongExtensions.Pow2MontgomeryModWindowedKernel(exponent, divisor, keepMontgomery: false) == 1UL ? (byte)1 : (byte)0;
|
||||
}
|
||||
|
||||
public sealed class DivisorScanSession : IMersenneNumberDivisorByDivisorTester.IDivisorScanSession
|
||||
{
|
||||
private readonly MersenneNumberDivisorByDivisorGpuTester _owner;
|
||||
@@ -636,137 +622,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
_owner._sessionPool.Add(this);
|
||||
}
|
||||
}
|
||||
private static void ComputeRemainderDeltasKernel(Index1D index, ArrayView<ulong> gaps, ArrayView<byte> deltas, byte modulus)
|
||||
{
|
||||
deltas[index] = (byte)(gaps[index] % modulus);
|
||||
}
|
||||
|
||||
private static void AccumulateRemaindersKernel(
|
||||
Index1D index,
|
||||
ArrayView<byte> deltas,
|
||||
ArrayView<byte> remainders,
|
||||
byte baseRemainder,
|
||||
byte modulus)
|
||||
{
|
||||
int globalIndex = index;
|
||||
int length = (int)remainders.Length;
|
||||
if (globalIndex >= length)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
int localIndex = Group.IdxX;
|
||||
int groupSize = Group.Dimension.X;
|
||||
var shared = SharedMemory.GetDynamic<int>();
|
||||
|
||||
shared[localIndex] = deltas[globalIndex];
|
||||
Group.Barrier();
|
||||
|
||||
int offset = 1;
|
||||
int mod = modulus;
|
||||
while (offset < groupSize)
|
||||
{
|
||||
int addend = 0;
|
||||
if (localIndex >= offset)
|
||||
{
|
||||
addend = shared[localIndex - offset];
|
||||
}
|
||||
|
||||
Group.Barrier();
|
||||
|
||||
if (localIndex >= offset)
|
||||
{
|
||||
int sum = shared[localIndex] + addend;
|
||||
if (sum >= mod)
|
||||
{
|
||||
sum %= mod;
|
||||
}
|
||||
|
||||
shared[localIndex] = sum;
|
||||
}
|
||||
|
||||
Group.Barrier();
|
||||
|
||||
offset <<= 1;
|
||||
}
|
||||
|
||||
if (globalIndex == 0)
|
||||
{
|
||||
remainders[0] = baseRemainder;
|
||||
return;
|
||||
}
|
||||
|
||||
int remainder = baseRemainder + shared[localIndex];
|
||||
if (remainder >= mod)
|
||||
{
|
||||
remainder %= mod;
|
||||
}
|
||||
|
||||
remainders[globalIndex] = (byte)remainder;
|
||||
}
|
||||
|
||||
private static void EvaluateCandidateMaskKernel(
|
||||
Index1D index,
|
||||
ArrayView<byte> remainder10,
|
||||
ArrayView<byte> remainder8,
|
||||
ArrayView<byte> remainder5,
|
||||
ArrayView<byte> remainder3,
|
||||
byte lastIsSevenFlag,
|
||||
ArrayView<byte> mask)
|
||||
{
|
||||
int globalIndex = index;
|
||||
int length = (int)mask.Length;
|
||||
if (globalIndex >= length)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
byte value10 = remainder10[globalIndex];
|
||||
bool accept10;
|
||||
if (lastIsSevenFlag != 0)
|
||||
{
|
||||
accept10 = value10 == 3 || value10 == 7 || value10 == 9;
|
||||
}
|
||||
else
|
||||
{
|
||||
accept10 = value10 == 1 || value10 == 3 || value10 == 7 || value10 == 9;
|
||||
}
|
||||
|
||||
if (!accept10)
|
||||
{
|
||||
mask[globalIndex] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
byte value8 = remainder8[globalIndex];
|
||||
if (value8 != 1 && value8 != 7)
|
||||
{
|
||||
mask[globalIndex] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (remainder3[globalIndex] == 0 || remainder5[globalIndex] == 0)
|
||||
{
|
||||
mask[globalIndex] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
mask[globalIndex] = 1;
|
||||
}
|
||||
|
||||
private static void ComputeMontgomeryExponentKernel(Index1D index, MontgomeryDivisorData divisor, ArrayView<ulong> exponents, ArrayView<ulong> results)
|
||||
{
|
||||
ulong modulus = divisor.Modulus;
|
||||
if (modulus <= 1UL || (modulus & 1UL) == 0UL)
|
||||
{
|
||||
results[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong exponent = exponents[index];
|
||||
|
||||
results[index] = ULongExtensions.Pow2MontgomeryModWindowedKernel(exponent, divisor, keepMontgomery: true);
|
||||
}
|
||||
|
||||
public ulong DivisorLimit
|
||||
{
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using ILGPU;
|
||||
using ILGPU.Algorithms;
|
||||
using ILGPU.Runtime;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
@@ -14,7 +13,7 @@ public sealed class MersenneNumberDivisorGpuTester
|
||||
private readonly ConcurrentDictionary<Accelerator, MemoryBuffer1D<byte, Stride1D.Dense>> _resultBuffers = new();
|
||||
|
||||
private Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
|
||||
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(Kernel));
|
||||
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(DivisorKernels.Kernel));
|
||||
|
||||
public static void BuildDivisorCandidates()
|
||||
{
|
||||
@@ -131,47 +130,5 @@ public sealed class MersenneNumberDivisorGpuTester
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void Kernel(Index1D _, ulong exponent, ReadOnlyGpuUInt128 divisor, ArrayView<byte> result)
|
||||
{
|
||||
if (divisor.High == 0UL && divisor.Low <= 1UL)
|
||||
{
|
||||
result[0] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 baseVal;
|
||||
int x = divisor.High == 0UL
|
||||
? 63 - XMath.LeadingZeroCount(divisor.Low)
|
||||
: 127 - XMath.LeadingZeroCount(divisor.High);
|
||||
ReadOnlyGpuUInt128 readOnlyMod = divisor;
|
||||
if (x == 0)
|
||||
{
|
||||
// TODO: Swap Pow2Mod for the ProcessEightBitWindows helper once Pow2Minus1Mod adopts it;
|
||||
// GpuPow2ModBenchmarks showed the windowed kernel cutting large divisors from ~51 µs to ~21 µs.
|
||||
baseVal = GpuUInt128.Pow2Mod(exponent, in readOnlyMod);
|
||||
result[0] = baseVal.High == 0UL && baseVal.Low == 1UL ? (byte)1 : (byte)0;
|
||||
return;
|
||||
}
|
||||
|
||||
ulong ux = (ulong)x;
|
||||
if (x >= 64)
|
||||
{
|
||||
baseVal = new GpuUInt128(1UL << (x - 64), 0UL);
|
||||
}
|
||||
else
|
||||
{
|
||||
baseVal = new GpuUInt128(0UL, 1UL << x);
|
||||
}
|
||||
|
||||
GpuUInt128 pow = baseVal;
|
||||
pow.ModPow(exponent / ux, in readOnlyMod);
|
||||
// TODO: Replace this trailing Pow2Mod call with the ProcessEightBitWindows variant once the shared
|
||||
// windowed helper lands so mixed-radix decompositions benefit from the same GPU speedup.
|
||||
var part2 = GpuUInt128.Pow2Mod(exponent % ux, in readOnlyMod);
|
||||
GpuUInt128 product = pow;
|
||||
ReadOnlyGpuUInt128 part2ReadOnly = part2.AsReadOnly();
|
||||
product.MulMod(in part2ReadOnly, in readOnlyMod);
|
||||
result[0] = product.High == 0UL && product.Low == 1UL ? (byte)1 : (byte)0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.CompilerServices;
|
||||
using ILGPU;
|
||||
using ILGPU.Runtime;
|
||||
using UInt128 = System.UInt128;
|
||||
@@ -18,23 +17,23 @@ public class MersenneNumberLucasLehmerGpuTester
|
||||
|
||||
private Action<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>> GetKernel(Accelerator accelerator)
|
||||
{
|
||||
return KernelCache.GetOrAdd(accelerator, accel => accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(Kernel));
|
||||
return KernelCache.GetOrAdd(accelerator, accel => accel.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<GpuUInt128>>(LucasLehmerKernels.Kernel));
|
||||
}
|
||||
|
||||
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetAddSmallKernel(Accelerator accelerator) =>
|
||||
AddSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(AddSmallKernel));
|
||||
AddSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.AddSmallKernel));
|
||||
|
||||
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetSubSmallKernel(Accelerator accelerator) =>
|
||||
SubSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(SubtractSmallKernel));
|
||||
SubSmallKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.SubtractSmallKernel));
|
||||
|
||||
private Action<Index1D, ulong, ArrayView<GpuUInt128>> GetReduceKernel(Accelerator accelerator) =>
|
||||
ReduceKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(ReduceModMersenneKernel));
|
||||
ReduceKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ArrayView<GpuUInt128>>(LucasLehmerKernels.ReduceModMersenneKernel));
|
||||
|
||||
private Action<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>> GetIsZeroKernel(Accelerator accelerator) =>
|
||||
IsZeroKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(IsZeroKernel));
|
||||
IsZeroKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<byte>>(LucasLehmerKernels.IsZeroKernel));
|
||||
|
||||
private Action<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>> GetBatchKernel(Accelerator accelerator) =>
|
||||
BatchKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(KernelBatch));
|
||||
BatchKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>>(LucasLehmerKernels.KernelBatch));
|
||||
|
||||
// Configurable LL slice size to keep kernels short. Default 32.
|
||||
public int SliceSize = 32;
|
||||
@@ -466,6 +465,13 @@ public class MersenneNumberLucasLehmerGpuTester
|
||||
return true;
|
||||
}
|
||||
|
||||
private static ulong MulMod(ulong a, ulong b, ulong modulus)
|
||||
{
|
||||
// TODO: Swap this UInt128 `%` reduction for the GPU-compatible MulMod helper once it adopts the faster
|
||||
// inline UInt128 path benchmarked in MulMod64Benchmarks so host/GPU parity avoids BigInteger-style fallbacks.
|
||||
return (ulong)(((UInt128)a * b) % modulus);
|
||||
}
|
||||
|
||||
private static ulong ModPow(ulong value, ulong exponent, ulong modulus)
|
||||
{
|
||||
ulong result = 1UL;
|
||||
@@ -485,453 +491,5 @@ public class MersenneNumberLucasLehmerGpuTester
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ulong MulMod(ulong a, ulong b, ulong modulus)
|
||||
{
|
||||
// TODO: Swap this UInt128 `%` reduction for the GPU-compatible MulMod helper once it adopts the faster
|
||||
// inline UInt128 path benchmarked in MulMod64Benchmarks so host/GPU parity avoids BigInteger-style fallbacks.
|
||||
return (ulong)(((UInt128)a * b) % modulus);
|
||||
}
|
||||
|
||||
private static void AddSmallKernel(Index1D index, ulong add, ArrayView<GpuUInt128> value)
|
||||
{
|
||||
ulong carry = add;
|
||||
for (int i = 0; i < value.Length && carry != 0UL; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong low = limb.Low + carry;
|
||||
ulong carryOut = low < carry ? 1UL : 0UL;
|
||||
ulong high = limb.High + carryOut;
|
||||
carry = high < limb.High ? 1UL : 0UL;
|
||||
value[i] = new GpuUInt128(high, low);
|
||||
}
|
||||
}
|
||||
|
||||
private static void SubtractSmallKernel(Index1D index, ulong subtract, ArrayView<GpuUInt128> value)
|
||||
{
|
||||
ulong borrow = subtract;
|
||||
for (int i = 0; i < value.Length && borrow != 0UL; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong low = limb.Low;
|
||||
ulong high = limb.High;
|
||||
if (low >= borrow)
|
||||
{
|
||||
low -= borrow;
|
||||
borrow = 0UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
low = unchecked(low - borrow);
|
||||
borrow = 1UL;
|
||||
if (high != 0UL)
|
||||
{
|
||||
high--;
|
||||
borrow = 0UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
high = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
value[i] = new GpuUInt128(high, low);
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReduceModMersenneKernel(Index1D index, ulong exponent, ArrayView<GpuUInt128> value)
|
||||
{
|
||||
int limbCount = (int)((exponent + 127UL) / 128UL);
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
if (limb.IsZero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int target = i - limbCount;
|
||||
var sum = value[target];
|
||||
var original = sum;
|
||||
sum.Add(limb);
|
||||
value[target] = sum;
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
if (sum.CompareTo(original) < 0)
|
||||
{
|
||||
int j = target + 1;
|
||||
while (j < limbCount)
|
||||
{
|
||||
var next = value[j];
|
||||
var nextOriginal = next;
|
||||
next.Add(1UL);
|
||||
value[j] = next;
|
||||
if (next.CompareTo(nextOriginal) >= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int topBits = (int)exponent.Mod128();
|
||||
if (topBits == 0)
|
||||
{
|
||||
topBits = 128;
|
||||
}
|
||||
|
||||
int topIndex = limbCount - 1;
|
||||
ulong topLowMask = topBits >= 64 ? ulong.MaxValue : (1UL << topBits) - 1UL;
|
||||
ulong topHighMask = topBits > 64 ? (1UL << (topBits - 64)) - 1UL : 0UL;
|
||||
var top = value[topIndex];
|
||||
var carryBits = top >> topBits;
|
||||
top = new(top.High & topHighMask, top.Low & topLowMask);
|
||||
value[topIndex] = top;
|
||||
|
||||
if (!carryBits.IsZero)
|
||||
{
|
||||
int j = 0;
|
||||
while (j < limbCount && !carryBits.IsZero)
|
||||
{
|
||||
var cur = value[j];
|
||||
var curOriginal = cur;
|
||||
cur.Add(carryBits);
|
||||
value[j] = cur;
|
||||
carryBits = cur.CompareTo(curOriginal) < 0 ? new GpuUInt128(0UL, 1UL) : new GpuUInt128(0UL, 0UL);
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
bool geq = true;
|
||||
for (int i = topIndex; i >= 0; i--)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong modHigh = i == topIndex ? topHighMask : ulong.MaxValue;
|
||||
ulong modLow = i == topIndex ? topLowMask : ulong.MaxValue;
|
||||
if (limb.High > modHigh || (limb.High == modHigh && limb.Low > modLow))
|
||||
{
|
||||
geq = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (limb.High < modHigh || (limb.High == modHigh && limb.Low < modLow))
|
||||
{
|
||||
geq = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (geq)
|
||||
{
|
||||
ulong borrow = 0UL;
|
||||
for (int i = 0; i < limbCount - 1; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
ulong newLow = limb.Low - ulong.MaxValue - borrow;
|
||||
borrow = limb.Low < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
ulong newHigh = limb.High - ulong.MaxValue - borrow;
|
||||
borrow = limb.High < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
value[i] = new GpuUInt128(newHigh, newLow);
|
||||
}
|
||||
|
||||
var topLimb = value[topIndex];
|
||||
ulong newTopLow = topLimb.Low - topLowMask - borrow;
|
||||
borrow = topLimb.Low < topLowMask + borrow ? 1UL : 0UL;
|
||||
ulong newTopHigh = topLimb.High - topHighMask - borrow;
|
||||
value[topIndex] = new GpuUInt128(newTopHigh, newTopLow);
|
||||
}
|
||||
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
}
|
||||
}
|
||||
|
||||
private static void IsZeroKernel(Index1D index, ArrayView<GpuUInt128> value, ArrayView<byte> result)
|
||||
{
|
||||
byte isZero = 1;
|
||||
for (int i = 0; i < value.Length; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
if (limb.High != 0UL || limb.Low != 0UL)
|
||||
{
|
||||
isZero = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result[0] = isZero;
|
||||
}
|
||||
|
||||
private static void ReduceModMersenne(Span<GpuUInt128> value, ulong exponent)
|
||||
{
|
||||
int limbCount = (int)((exponent + 127UL) / 128UL);
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
var limb = value[i];
|
||||
if (limb.IsZero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int target = i - limbCount;
|
||||
UInt128 original = value[target];
|
||||
UInt128 sum = original + (UInt128)limb;
|
||||
bool carry = sum < original;
|
||||
value[target] = new GpuUInt128(sum);
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
if (carry)
|
||||
{
|
||||
int j = target + 1;
|
||||
while (j < limbCount)
|
||||
{
|
||||
UInt128 next = value[j];
|
||||
UInt128 nextSum = next + 1UL;
|
||||
value[j] = new GpuUInt128(nextSum);
|
||||
if (nextSum != 0UL)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int topBits = (int)exponent.Mod128();
|
||||
if (topBits == 0)
|
||||
{
|
||||
topBits = 128;
|
||||
}
|
||||
|
||||
int topIndex = limbCount - 1;
|
||||
ulong topLowMask = topBits >= 64 ? ulong.MaxValue : (1UL << topBits) - 1UL;
|
||||
ulong topHighMask = topBits > 64 ? (1UL << (topBits - 64)) - 1UL : 0UL;
|
||||
UInt128 mask = ((UInt128)topHighMask << 64) | topLowMask;
|
||||
UInt128 top = value[topIndex];
|
||||
UInt128 carryBits = top >> topBits;
|
||||
top &= mask;
|
||||
value[topIndex] = new GpuUInt128(top);
|
||||
if (carryBits != 0UL)
|
||||
{
|
||||
int j = 0;
|
||||
UInt128 carry = carryBits;
|
||||
while (j < limbCount && carry != 0UL)
|
||||
{
|
||||
UInt128 cur = value[j];
|
||||
UInt128 sum = cur + carry;
|
||||
bool overflow = sum < cur;
|
||||
value[j] = new GpuUInt128(sum);
|
||||
carry = overflow ? 1UL : 0UL;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
bool geq = true;
|
||||
for (int i = topIndex; i >= 0; i--)
|
||||
{
|
||||
UInt128 limb = value[i];
|
||||
UInt128 modLimb = i == topIndex ? mask : UInt128.MaxValue;
|
||||
if (limb > modLimb)
|
||||
{
|
||||
geq = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (limb < modLimb)
|
||||
{
|
||||
geq = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (geq)
|
||||
{
|
||||
ulong borrow = 0UL;
|
||||
for (int i = 0; i < limbCount - 1; i++)
|
||||
{
|
||||
ulong low = value[i].Low;
|
||||
ulong high = value[i].High;
|
||||
ulong newLow = low - ulong.MaxValue - borrow;
|
||||
borrow = low < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
ulong newHigh = high - ulong.MaxValue - borrow;
|
||||
borrow = high < ulong.MaxValue + borrow ? 1UL : 0UL;
|
||||
value[i] = new GpuUInt128(newHigh, newLow);
|
||||
}
|
||||
|
||||
ulong topLow = value[topIndex].Low;
|
||||
ulong topHigh = value[topIndex].High;
|
||||
ulong newTopLow = topLow - topLowMask - borrow;
|
||||
borrow = topLow < topLowMask + borrow ? 1UL : 0UL;
|
||||
ulong newTopHigh = topHigh - topHighMask - borrow;
|
||||
value[topIndex] = new GpuUInt128(newTopHigh, newTopLow);
|
||||
}
|
||||
|
||||
for (int i = limbCount; i < value.Length; i++)
|
||||
{
|
||||
value[i] = new GpuUInt128(0UL, 0UL);
|
||||
}
|
||||
}
|
||||
|
||||
private static void KernelBatch(Index1D index, ArrayView<ulong> exponents, ArrayView<GpuUInt128> moduli, ArrayView<GpuUInt128> states)
|
||||
{
|
||||
int idx = index.X;
|
||||
ulong exponent = exponents[idx];
|
||||
GpuUInt128 modulus = moduli[idx];
|
||||
var s = new GpuUInt128(0UL, 4UL);
|
||||
ulong limit = exponent - 2UL;
|
||||
for (ulong i = 0UL; i < limit; i++)
|
||||
{
|
||||
s = SquareModMersenne128(s, exponent);
|
||||
s.SubMod(2UL, modulus);
|
||||
}
|
||||
|
||||
states[idx] = s;
|
||||
}
|
||||
|
||||
private static void Kernel(Index1D index, ulong exponent, GpuUInt128 modulus, ArrayView<GpuUInt128> state)
|
||||
{
|
||||
// Lucas–Lehmer iteration in the field GF(2^p-1).
|
||||
// For p < 128, use a Mersenne-specific squaring + reduction to avoid
|
||||
// the generic 128-bit long-division-like reduction.
|
||||
var s = new GpuUInt128(0UL, 4UL);
|
||||
ulong i = 0UL, limit = exponent - 2UL;
|
||||
if (exponent < 128UL)
|
||||
{
|
||||
for (; i < limit; i++)
|
||||
{
|
||||
s = SquareModMersenne128(s, exponent);
|
||||
s.SubMod(2UL, modulus);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (; i < limit; i++)
|
||||
{
|
||||
s.SquareMod(modulus);
|
||||
s.SubMod(2UL, modulus);
|
||||
}
|
||||
}
|
||||
|
||||
state[0] = s;
|
||||
}
|
||||
|
||||
// Squares a 128-bit value and reduces modulo M_p = 2^p - 1 using
|
||||
// Mersenne folding. Valid for 0 < p < 128 and input s in [0, M_p].
|
||||
private static GpuUInt128 SquareModMersenne128(GpuUInt128 s, ulong p)
|
||||
{
|
||||
// Full square: (H*2^64 + L)^2 = H^2*2^128 + 2HL*2^64 + L^2
|
||||
SquareFull128(s, out var q3, out var q2, out var q1, out var q0);
|
||||
|
||||
// v = low128 + high128 (fold by 128 bits once)
|
||||
var vHigh = new GpuUInt128(q3, q2);
|
||||
var vLow = new GpuUInt128(q1, q0);
|
||||
vLow.Add(vHigh);
|
||||
|
||||
// Mask to keep only p low bits and fold the carry bits (v >> p)
|
||||
int topBits = (int)p.Mod128();
|
||||
if (topBits == 0)
|
||||
{
|
||||
topBits = 128;
|
||||
}
|
||||
|
||||
ulong maskLow = topBits >= 64 ? ulong.MaxValue : (1UL << topBits) - 1UL;
|
||||
ulong maskHigh = topBits > 64 ? (1UL << (topBits - 64)) - 1UL : 0UL;
|
||||
|
||||
var carryBits = vLow >> topBits;
|
||||
vLow = new(vLow.High & maskHigh, vLow.Low & maskLow);
|
||||
|
||||
// Propagate carryBits back into the masked value (single-limb fold)
|
||||
GpuUInt128 before;
|
||||
while (!carryBits.IsZero)
|
||||
{
|
||||
before = vLow;
|
||||
vLow.Add(carryBits);
|
||||
carryBits = vLow.CompareTo(before) < 0 ? GpuUInt128.One : GpuUInt128.Zero;
|
||||
}
|
||||
|
||||
// Final correction if v >= modulus (mask)
|
||||
if (vLow.High > maskHigh || (vLow.High == maskHigh && vLow.Low >= maskLow))
|
||||
{
|
||||
ulong borrow = 0UL;
|
||||
ulong newLow = vLow.Low - maskLow;
|
||||
borrow = vLow.Low < maskLow ? 1UL : 0UL;
|
||||
ulong newHigh = vLow.High - maskHigh - borrow;
|
||||
vLow = new GpuUInt128(newHigh, newLow);
|
||||
}
|
||||
|
||||
return vLow;
|
||||
}
|
||||
|
||||
// Helper: 128-bit square -> 256-bit product as four 64-bit limbs (q3..q0)
|
||||
private static void SquareFull128(GpuUInt128 value, out ulong q3, out ulong q2, out ulong q1, out ulong q0)
|
||||
{
|
||||
ulong L = value.Low;
|
||||
ulong H = value.High;
|
||||
Mul64Parts(L, L, out var hLL, out var lLL); // L^2
|
||||
Mul64Parts(H, H, out var hHH, out var lHH); // H^2
|
||||
Mul64Parts(L, H, out var hLH, out var lLH); // L*H
|
||||
|
||||
// double LH
|
||||
ulong dLH_low = lLH << 1;
|
||||
ulong carry = (lLH >> 63) & 1UL;
|
||||
ulong dLH_high = (hLH << 1) | carry;
|
||||
|
||||
q0 = lLL;
|
||||
ulong sum1 = hLL + dLH_low;
|
||||
ulong c1 = sum1 < hLL ? 1UL : 0UL;
|
||||
q1 = sum1;
|
||||
|
||||
ulong sum2 = lHH + dLH_high;
|
||||
ulong c2 = sum2 < lHH ? 1UL : 0UL;
|
||||
sum2 += c1;
|
||||
if (sum2 < c1)
|
||||
{
|
||||
c2++;
|
||||
}
|
||||
q2 = sum2;
|
||||
q3 = hHH + c2;
|
||||
}
|
||||
|
||||
// 64x64 -> 128 multiply (high,low)
|
||||
[MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul64Parts(ulong a, ulong b, out ulong high, out ulong low)
|
||||
{
|
||||
ulong a0 = (uint)a;
|
||||
ulong a1 = a >> 32;
|
||||
ulong b0 = (uint)b;
|
||||
ulong b1 = b >> 32;
|
||||
|
||||
ulong lo = a0 * b0;
|
||||
ulong mid1 = a1 * b0;
|
||||
ulong mid2 = a0 * b1;
|
||||
ulong hi = a1 * b1;
|
||||
|
||||
ulong carry = (lo >> 32) + (uint)mid1 + (uint)mid2;
|
||||
low = (lo & 0xFFFFFFFFUL) | (carry << 32);
|
||||
hi += (mid1 >> 32) + (mid2 >> 32) + (carry >> 32);
|
||||
high = hi;
|
||||
}
|
||||
|
||||
[MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong ModPow64(ulong value, ulong exponent, ulong modulus)
|
||||
{
|
||||
ulong result = 1UL;
|
||||
ulong baseValue = value % modulus;
|
||||
ulong exp = exponent;
|
||||
|
||||
while (exp != 0UL)
|
||||
{
|
||||
if ((exp & 1UL) != 0UL)
|
||||
{
|
||||
result = (ulong)(((UInt128)result * baseValue) % modulus);
|
||||
}
|
||||
|
||||
baseValue = (ulong)(((UInt128)baseValue * baseValue) % modulus);
|
||||
exp >>= 1;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,6 +228,26 @@ public static class NttGpuMath
|
||||
return 0UL - inv;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul64Parts(ulong a, ulong b, out ulong high, out ulong low)
|
||||
{
|
||||
// 64x64 -> 128 using 32-bit limbs
|
||||
ulong a0 = (uint)a;
|
||||
ulong a1 = a >> 32;
|
||||
ulong b0 = (uint)b;
|
||||
ulong b1 = b >> 32;
|
||||
|
||||
ulong lo = a0 * b0;
|
||||
ulong mid1 = a1 * b0;
|
||||
ulong mid2 = a0 * b1;
|
||||
ulong hi = a1 * b1;
|
||||
|
||||
ulong carry = (lo >> 32) + (uint)mid1 + (uint)mid2;
|
||||
low = (lo & 0xFFFFFFFFUL) | (carry << 32);
|
||||
hi += (mid1 >> 32) + (mid2 >> 32) + (carry >> 32);
|
||||
high = hi;
|
||||
}
|
||||
|
||||
private readonly record struct SquareCacheKey(int Length, GpuUInt128 Modulus, GpuUInt128 PrimitiveRoot);
|
||||
|
||||
private static readonly ConcurrentDictionary<Accelerator, ConcurrentDictionary<SquareCacheKey, SquareCacheEntry>> SquareCache = new();
|
||||
@@ -357,43 +377,43 @@ public static class NttGpuMath
|
||||
}
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, int> GetBitReverseKernel(Accelerator accelerator) =>
|
||||
BitReverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(BitReverseKernel));
|
||||
BitReverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int>(NttTransformKernels.BitReverseKernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128> GetMulKernel(Accelerator accelerator) =>
|
||||
MulKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(MulKernel));
|
||||
MulKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, GpuUInt128>(NttPointwiseKernels.MulKernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128> GetStageKernel(Accelerator accelerator) =>
|
||||
StageKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(StageKernel));
|
||||
StageKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, GpuUInt128>(NttButterflyKernels.StageKernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128> GetScaleKernel(Accelerator accelerator) =>
|
||||
ScaleKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(ScaleKernel));
|
||||
ScaleKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, GpuUInt128>(NttPointwiseKernels.ScaleKernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong> GetStageMontKernel(Accelerator accelerator) =>
|
||||
StageMontKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(StageMontKernel));
|
||||
StageMontKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.StageMontKernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetStageBarrett128Kernel(Accelerator accelerator) =>
|
||||
StageBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(StageBarrett128Kernel));
|
||||
StageBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, int, int, int, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.StageBarrett128Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong> GetScaleBarrett128Kernel(Accelerator accelerator) =>
|
||||
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(ScaleBarrett128Kernel));
|
||||
ScaleBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, GpuUInt128, ulong, ulong, ulong, ulong>(NttBarrettKernels.ScaleBarrett128Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong> GetSquareBarrett128Kernel(Accelerator accelerator) =>
|
||||
SquareBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(SquareBarrett128Kernel));
|
||||
SquareBarrett128KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong, ulong>(NttBarrettKernels.SquareBarrett128Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetToMont64Kernel(Accelerator accelerator) =>
|
||||
ToMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(ToMont64Kernel));
|
||||
ToMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ToMont64Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetFromMont64Kernel(Accelerator accelerator) =>
|
||||
FromMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(FromMont64Kernel));
|
||||
FromMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.FromMont64Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong> GetSquareMont64Kernel(Accelerator accelerator) =>
|
||||
SquareMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(SquareMont64Kernel));
|
||||
SquareMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong>(NttMontgomeryKernels.SquareMont64Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong> GetScaleMont64Kernel(Accelerator accelerator) =>
|
||||
ScaleMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(ScaleMont64Kernel));
|
||||
ScaleMont64KernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ulong, ulong, ulong>(NttMontgomeryKernels.ScaleMont64Kernel));
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128> GetForwardKernel(Accelerator accelerator) =>
|
||||
ForwardKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(ForwardKernel));
|
||||
ForwardKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128>(NttTransformKernels.ForwardKernel));
|
||||
|
||||
// TODO(NTT-OPT): Introduce stage-wise Cooley–Tukey kernels and a
|
||||
// per-(accelerator,length,modulus,root) twiddle cache. The plan:
|
||||
@@ -402,421 +422,19 @@ public static class NttGpuMath
|
||||
// - Provide accessors like GetStageKernel(...) and GetTwiddleBuffer(...).
|
||||
|
||||
private static Action<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128> GetInverseKernel(Accelerator accelerator) =>
|
||||
InverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(InverseKernel));
|
||||
InverseKernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ArrayView<GpuUInt128>, ArrayView<GpuUInt128>, int, GpuUInt128, GpuUInt128, GpuUInt128>(NttTransformKernels.InverseKernel));
|
||||
|
||||
// TODO(NTT-OPT): Mirror the forward stage-wise design for the inverse
|
||||
// transform. Precompute inverse twiddles and a normalization factor
|
||||
// (n^-1 mod m). Launch one short kernel per stage to avoid TDR.
|
||||
|
||||
private static void MulKernel(Index1D index, ArrayView<GpuUInt128> a, ArrayView<GpuUInt128> b, GpuUInt128 modulus)
|
||||
{
|
||||
var val = a[index];
|
||||
val.MulMod(b[index], modulus);
|
||||
a[index] = val;
|
||||
}
|
||||
|
||||
private static void StageKernel(Index1D index, ArrayView<GpuUInt128> data, int len, int half, int stageOffset, ArrayView<GpuUInt128> twiddles, GpuUInt128 modulus)
|
||||
{
|
||||
// TODO(NTT-OPT): Consider shared-memory tiling to improve locality:
|
||||
// load a block of size `len` into shared memory, do butterflies, write back.
|
||||
// Requires explicit grouped kernels and chosen group size.
|
||||
int t = index.X;
|
||||
// TODO: Replace this `%` with a bitmask when `half` is a power of two so the stage index math matches the faster
|
||||
// residue strategy from the GPU residue benchmarks.
|
||||
int j = t % half;
|
||||
int block = t / half;
|
||||
int k = block * len;
|
||||
int i1 = k + j;
|
||||
int i2 = i1 + half;
|
||||
var u = data[i1];
|
||||
var v = data[i2];
|
||||
var w = twiddles[stageOffset + j];
|
||||
v.MulMod(w, modulus);
|
||||
var sum = new GpuUInt128(u);
|
||||
sum.AddMod(v, modulus);
|
||||
u.SubMod(v, modulus);
|
||||
data[i1] = sum;
|
||||
data[i2] = u;
|
||||
}
|
||||
|
||||
private static void ScaleKernel(Index1D index, ArrayView<GpuUInt128> data, GpuUInt128 scale, GpuUInt128 modulus)
|
||||
{
|
||||
var v = data[index];
|
||||
v.MulMod(scale, modulus);
|
||||
data[index] = v;
|
||||
}
|
||||
|
||||
// Barrett-based scaling: v = v * scale mod n for 128-bit n
|
||||
private static void ScaleBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, GpuUInt128 scale, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
|
||||
{
|
||||
var v = data[index];
|
||||
// 128x128 -> 256
|
||||
Mul128Full(v.High, v.Low, scale.High, scale.Low, out var z3, out var z2, out var z1, out var z0);
|
||||
// q approx
|
||||
Mul3x2Top(z3, z2, z1, muHigh, muLow, out var qHigh, out var qLow);
|
||||
// q*n
|
||||
Mul128Full(qHigh, qLow, modHigh, modLow, out var qq3, out var qq2, out var qq1, out var qq0);
|
||||
// r = z - q*n
|
||||
Sub256(z3, z2, z1, z0, qq3, qq2, qq1, qq0, out var r3, out var r2, out var r1, out var r0);
|
||||
ulong resHigh = r1;
|
||||
ulong resLow = r0;
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
}
|
||||
}
|
||||
|
||||
data[index] = new GpuUInt128(resHigh, resLow);
|
||||
}
|
||||
|
||||
// Barrett-based squaring: v = v^2 mod n for 128-bit n
|
||||
private static void SquareBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
|
||||
{
|
||||
var v = data[index];
|
||||
Mul128Full(v.High, v.Low, v.High, v.Low, out var z3, out var z2, out var z1, out var z0);
|
||||
Mul3x2Top(z3, z2, z1, muHigh, muLow, out var qHigh, out var qLow);
|
||||
Mul128Full(qHigh, qLow, modHigh, modLow, out var qq3, out var qq2, out var qq1, out var qq0);
|
||||
Sub256(z3, z2, z1, z0, qq3, qq2, qq1, qq0, out var r3, out var r2, out var r1, out var r0);
|
||||
ulong resHigh = r1;
|
||||
ulong resLow = r0;
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
}
|
||||
}
|
||||
data[index] = new GpuUInt128(resHigh, resLow);
|
||||
}
|
||||
|
||||
// Barrett reduction based stage for 128-bit moduli (no '%').
|
||||
private static void StageBarrett128Kernel(Index1D index, ArrayView<GpuUInt128> data, int len, int half, int stageOffset, ArrayView<GpuUInt128> twiddles, ulong modHigh, ulong modLow, ulong muHigh, ulong muLow)
|
||||
{
|
||||
int t = index.X;
|
||||
// TODO: Use the bitmask-based remainder helper here as well to remove `%` from the butterfly stage and align with the
|
||||
// optimized kernels highlighted in the GPU pow2mod benchmarks.
|
||||
int j = t % half;
|
||||
int block = t / half;
|
||||
int k = block * len;
|
||||
int i1 = k + j;
|
||||
int i2 = i1 + half;
|
||||
var u = data[i1];
|
||||
var v = data[i2];
|
||||
var w = twiddles[stageOffset + j];
|
||||
|
||||
// Compute v * w (128x128) -> 256-bit z3..z0
|
||||
Mul128Full(v.High, v.Low, w.High, w.Low, out var z3, out var z2, out var z1, out var z0);
|
||||
|
||||
// t = floor(z / b) where b=2^64 => take (z3,z2,z1)
|
||||
// Compute q = floor((t * mu) / b^3). Only need top two limbs of 5-limb product.
|
||||
Mul3x2Top(z3, z2, z1, muHigh, muLow, out var qHigh, out var qLow);
|
||||
|
||||
// q * n (128x128) -> 256-bit qq3..qq0
|
||||
Mul128Full(qHigh, qLow, modHigh, modLow, out var qq3, out var qq2, out var qq1, out var qq0);
|
||||
|
||||
// r = z - q*n (256-bit)
|
||||
Sub256(z3, z2, z1, z0, qq3, qq2, qq1, qq0, out var r3, out var r2, out var r1, out var r0);
|
||||
|
||||
// Reduce r to [0, n) by at most two subtractions
|
||||
// Discard higher limbs (should be zero if r < 2n), keep low 128-bit
|
||||
ulong resHigh = r1; // r1 is limb1 after subtraction (since lower two limbs are r1 (high) and r0 (low))
|
||||
ulong resLow = r0;
|
||||
|
||||
// while (res >= n) res -= n; (at most twice)
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
if (Geq128(resHigh, resLow, modHigh, modLow))
|
||||
{
|
||||
Sub128(ref resHigh, ref resLow, modHigh, modLow);
|
||||
}
|
||||
}
|
||||
|
||||
var sum = new GpuUInt128(u);
|
||||
var mulred = new GpuUInt128(resHigh, resLow);
|
||||
sum.AddMod(mulred, new GpuUInt128(modHigh, modLow));
|
||||
u.SubMod(mulred, new GpuUInt128(modHigh, modLow));
|
||||
data[i1] = sum;
|
||||
data[i2] = u;
|
||||
}
|
||||
|
||||
// TODO(MOD-OPT): Stage kernel using 64-bit Montgomery multiplication. Requires data and twiddles in Montgomery domain.
|
||||
private static void StageMontKernel(Index1D index, ArrayView<GpuUInt128> data, int len, int half, int stageOffset, ArrayView<GpuUInt128> twiddlesMont, ulong modulus, ulong nPrime)
|
||||
{
|
||||
int t = index.X;
|
||||
// TODO: Apply the bitmask remainder helper to this stage too so every butterfly path drops the slower `%` operation.
|
||||
int j = t % half;
|
||||
int block = t / half;
|
||||
int k = block * len;
|
||||
int i1 = k + j;
|
||||
int i2 = i1 + half;
|
||||
ulong u = data[i1].Low;
|
||||
ulong v = data[i2].Low;
|
||||
ulong wR = twiddlesMont[stageOffset + j].Low;
|
||||
v = MontMul64(v, wR, modulus, nPrime);
|
||||
ulong sum = u + v;
|
||||
if (sum >= modulus)
|
||||
{
|
||||
sum -= modulus;
|
||||
}
|
||||
ulong diff = u >= v ? u - v : unchecked(u + modulus - v);
|
||||
data[i1] = new GpuUInt128(0UL, sum);
|
||||
data[i2] = new GpuUInt128(0UL, diff);
|
||||
}
|
||||
|
||||
// Computes Montgomery multiplication for 64-bit operands in Montgomery domain.
|
||||
// Returns a*b*R^{-1} mod modulus, where R=2^64 and nPrime = -modulus^{-1} mod 2^64.
|
||||
private static ulong MontMul64(ulong aR, ulong bR, ulong modulus, ulong nPrime)
|
||||
{
|
||||
// t = aR * bR (128-bit)
|
||||
ulong tLow, tHigh;
|
||||
Mul64Parts(aR, bR, out tHigh, out tLow);
|
||||
// m = (tLow * nPrime) mod 2^64 -> just low 64 bits of the product
|
||||
ulong mLow, mHigh;
|
||||
Mul64Parts(tLow, nPrime, out mHigh, out mLow);
|
||||
// u = (t + m*modulus) >> 64
|
||||
ulong mmLow, mmHigh;
|
||||
Mul64Parts(mLow, modulus, out mmHigh, out mmLow);
|
||||
// add tLow + mmLow, keep carry
|
||||
ulong carry = 0UL;
|
||||
ulong low = tLow + mmLow;
|
||||
if (low < tLow)
|
||||
{
|
||||
carry = 1UL;
|
||||
}
|
||||
// high = tHigh + mmHigh + carry
|
||||
ulong high = tHigh + mmHigh + carry;
|
||||
ulong u = high; // this is (t + m*n) >> 64
|
||||
if (u >= modulus)
|
||||
{
|
||||
u -= modulus;
|
||||
}
|
||||
return u;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul64Parts(ulong a, ulong b, out ulong high, out ulong low)
|
||||
{
|
||||
// 64x64 -> 128 using 32-bit limbs
|
||||
ulong a0 = (uint)a;
|
||||
ulong a1 = a >> 32;
|
||||
ulong b0 = (uint)b;
|
||||
ulong b1 = b >> 32;
|
||||
|
||||
ulong lo = a0 * b0;
|
||||
ulong mid1 = a1 * b0;
|
||||
ulong mid2 = a0 * b1;
|
||||
ulong hi = a1 * b1;
|
||||
|
||||
ulong carry = (lo >> 32) + (uint)mid1 + (uint)mid2;
|
||||
low = (lo & 0xFFFFFFFFUL) | (carry << 32);
|
||||
hi += (mid1 >> 32) + (mid2 >> 32) + (carry >> 32);
|
||||
high = hi;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul128Full(ulong aHigh, ulong aLow, ulong bHigh, ulong bLow, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
{
|
||||
// Full 128x128 -> 256 multiply using 64-bit limbs
|
||||
Mul64Parts(aLow, bLow, out var h0, out var l0);
|
||||
Mul64Parts(aLow, bHigh, out var h1, out var l1);
|
||||
Mul64Parts(aHigh, bLow, out var h2, out var l2);
|
||||
Mul64Parts(aHigh, bHigh, out var h3, out var l3);
|
||||
|
||||
p0 = l0;
|
||||
ulong carry = 0UL;
|
||||
ulong sum = h0;
|
||||
sum += l1; if (sum < l1) carry++;
|
||||
sum += l2; if (sum < l2) carry++;
|
||||
p1 = sum;
|
||||
|
||||
sum = h1;
|
||||
sum += h2; ulong carry2 = sum < h2 ? 1UL : 0UL;
|
||||
sum += l3; if (sum < l3) carry2++;
|
||||
sum += carry; if (sum < carry) carry2++;
|
||||
p2 = sum;
|
||||
p3 = h3 + carry2;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Mul3x2Top(ulong t2, ulong t1, ulong t0, ulong muHigh, ulong muLow, out ulong top1, out ulong top0)
|
||||
{
|
||||
// Compute top two limbs of ( (t2,t1,t0) * (muHigh,muLow) ) after shifting right by 192 bits.
|
||||
// We build full 5 limbs and return p4 (top1) and p3 (top0).
|
||||
ulong p0 = 0, p1 = 0, p2 = 0, p3 = 0, p4 = 0;
|
||||
|
||||
void Add128To(ref ulong lo, ref ulong hi, ulong addLo, ulong addHi)
|
||||
{
|
||||
ulong s = lo + addLo;
|
||||
ulong c = s < lo ? 1UL : 0UL;
|
||||
lo = s;
|
||||
ulong s2 = hi + addHi + c;
|
||||
hi = s2;
|
||||
}
|
||||
|
||||
// t0 * muLow -> contributes to p0,p1
|
||||
Mul64Parts(t0, muLow, out var h, out var l);
|
||||
Add128To(ref p0, ref p1, l, h);
|
||||
|
||||
// t0 * muHigh -> to p1,p2
|
||||
Mul64Parts(t0, muHigh, out h, out l);
|
||||
Add128To(ref p1, ref p2, l, h);
|
||||
|
||||
// t1 * muLow -> to p1,p2
|
||||
Mul64Parts(t1, muLow, out h, out l);
|
||||
Add128To(ref p1, ref p2, l, h);
|
||||
|
||||
// t1 * muHigh -> to p2,p3
|
||||
Mul64Parts(t1, muHigh, out h, out l);
|
||||
Add128To(ref p2, ref p3, l, h);
|
||||
|
||||
// t2 * muLow -> to p2,p3
|
||||
Mul64Parts(t2, muLow, out h, out l);
|
||||
Add128To(ref p2, ref p3, l, h);
|
||||
|
||||
// t2 * muHigh -> to p3,p4
|
||||
Mul64Parts(t2, muHigh, out h, out l);
|
||||
Add128To(ref p3, ref p4, l, h);
|
||||
|
||||
top1 = p4; top0 = p3;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Sub256(ulong a3, ulong a2, ulong a1, ulong a0, ulong b3, ulong b2, ulong b1, ulong b0, out ulong r3, out ulong r2, out ulong r1, out ulong r0)
|
||||
{
|
||||
ulong borrow = 0;
|
||||
r0 = SubWithBorrow(a0, b0, ref borrow);
|
||||
r1 = SubWithBorrow(a1, b1, ref borrow);
|
||||
r2 = SubWithBorrow(a2, b2, ref borrow);
|
||||
r3 = SubWithBorrow(a3, b3, ref borrow);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong SubWithBorrow(ulong a, ulong b, ref ulong borrow)
|
||||
{
|
||||
ulong res = a - b - borrow;
|
||||
borrow = ((a ^ b) & (a ^ res) & 0x8000_0000_0000_0000UL) != 0 ? 1UL : (a < b + borrow ? 1UL : 0UL);
|
||||
return res;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool Geq128(ulong aHigh, ulong aLow, ulong bHigh, ulong bLow)
|
||||
{
|
||||
return aHigh > bHigh || (aHigh == bHigh && aLow >= bLow);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Sub128(ref ulong aHigh, ref ulong aLow, ulong bHigh, ulong bLow)
|
||||
{
|
||||
ulong borrow = 0;
|
||||
ulong low = aLow - bLow;
|
||||
borrow = aLow < bLow ? 1UL : 0UL;
|
||||
ulong high = aHigh - bHigh - borrow;
|
||||
aHigh = high; aLow = low;
|
||||
}
|
||||
|
||||
private static void ToMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong r2)
|
||||
{
|
||||
ulong a = data[index].Low;
|
||||
ulong v = MontMul64(a, r2, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
private static void FromMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
|
||||
{
|
||||
ulong aR = data[index].Low;
|
||||
ulong v = MontMul64(aR, 1UL, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
private static void SquareMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime)
|
||||
{
|
||||
ulong aR = data[index].Low;
|
||||
ulong v = MontMul64(aR, aR, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
private static void ScaleMont64Kernel(Index1D index, ArrayView<GpuUInt128> data, ulong modulus, ulong nPrime, ulong scaleMont)
|
||||
{
|
||||
ulong aR = data[index].Low;
|
||||
ulong v = MontMul64(aR, scaleMont, modulus, nPrime);
|
||||
data[index] = new GpuUInt128(0UL, v);
|
||||
}
|
||||
|
||||
private static void ForwardKernel(Index1D index, ArrayView<GpuUInt128> input, ArrayView<GpuUInt128> output, int length, GpuUInt128 modulus, GpuUInt128 root)
|
||||
{
|
||||
// TODO(NTT-OPT): Replace this reference O(n^2) kernel with a stage-wise
|
||||
// Cooley–Tukey butterfly kernel (O(n log n)). This version uses a per-
|
||||
// element ModPow in the hot loop which is far too slow and causes long
|
||||
// single-kernel runtimes. After twiddle precomputation, each butterfly
|
||||
// should perform: (u, v) -> (u+v*w, u-v*w) with a single MulMod.
|
||||
var sum = new GpuUInt128(0UL, 0UL);
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
var term = input[i];
|
||||
var power = new GpuUInt128(root);
|
||||
power.ModPow((ulong)index.X * (ulong)i, modulus);
|
||||
term.MulMod(power, modulus);
|
||||
sum.AddMod(term, modulus);
|
||||
}
|
||||
|
||||
output[index] = sum;
|
||||
}
|
||||
|
||||
private static void InverseKernel(Index1D index, ArrayView<GpuUInt128> input, ArrayView<GpuUInt128> output, int length, GpuUInt128 modulus, GpuUInt128 rootInv, GpuUInt128 nInv)
|
||||
{
|
||||
// TODO(NTT-OPT): Replace this O(n^2) inverse with stage-wise butterflies
|
||||
// using precomputed inverse twiddles. Apply final scaling by nInv after
|
||||
// all stages, preferably in a small dedicated kernel.
|
||||
var sum = new GpuUInt128(0UL, 0UL);
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
var term = input[i];
|
||||
var power = new GpuUInt128(rootInv);
|
||||
power.ModPow((ulong)index.X * (ulong)i, modulus);
|
||||
term.MulMod(power, modulus);
|
||||
sum.AddMod(term, modulus);
|
||||
}
|
||||
|
||||
sum.MulMod(nInv, modulus);
|
||||
output[index] = sum;
|
||||
}
|
||||
|
||||
private static void BitReverseKernel(Index1D index, ArrayView<GpuUInt128> data, int bits)
|
||||
{
|
||||
int i = index.X;
|
||||
int j = ReverseBits(i, bits);
|
||||
if (i < j)
|
||||
{
|
||||
var tmp = data[i];
|
||||
data[i] = data[j];
|
||||
data[j] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
private static int ReverseBits(int value, int bits)
|
||||
{
|
||||
int result = 0;
|
||||
for (int i = 0; i < bits; i++)
|
||||
{
|
||||
result = (result << 1) | (value & 1);
|
||||
value >>= 1;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void BitReverse(Span<GpuUInt128> values)
|
||||
{
|
||||
int n = values.Length;
|
||||
int bits = (int)Math.Log2(n);
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
int j = ReverseBits(i, bits);
|
||||
int j = NttTransformKernels.ReverseBits(i, bits);
|
||||
if (i < j)
|
||||
{
|
||||
(values[j], values[i]) = (values[i], values[j]);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -847,7 +847,7 @@ public class MersenneDivisorCycles
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<
|
||||
Index1D,
|
||||
ArrayView1D<ulong, Stride1D.Dense>,
|
||||
ArrayView1D<ulong, Stride1D.Dense>>(GpuDivisorCycleKernel);
|
||||
ArrayView1D<ulong, Stride1D.Dense>>(DivisorCycleKernels.GpuDivisorCycleKernel);
|
||||
|
||||
|
||||
using Stream outputStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, BufferSize10M, useAsync: true);
|
||||
@@ -941,93 +941,8 @@ public class MersenneDivisorCycles
|
||||
|
||||
}
|
||||
|
||||
const byte ByteZero = 0;
|
||||
const byte ByteOne = 1;
|
||||
|
||||
// GPU kernel for divisor cycle calculation
|
||||
static void GpuDivisorCycleKernel(
|
||||
Index1D index,
|
||||
ArrayView1D<ulong, Stride1D.Dense> divisors,
|
||||
ArrayView1D<ulong, Stride1D.Dense> outCycles)
|
||||
{
|
||||
int i = index.X;
|
||||
outCycles[i] = CalculateCycleLengthGpu(divisors[i]);
|
||||
}
|
||||
|
||||
// GPU-friendly version of cycle length calculation
|
||||
/// <summary>
|
||||
/// GPU-friendly cycle calculator that unrolls sixteen doubling steps; it wins the 8,388,607 benchmark and stays within ~2% of the octo loop at divisor 131,071.
|
||||
/// </summary>
|
||||
public static ulong CalculateCycleLengthGpu(ulong divisor)
|
||||
{
|
||||
if ((divisor & (divisor - 1UL)) == 0UL)
|
||||
return 1UL;
|
||||
|
||||
ulong order = 1UL;
|
||||
ulong pow = 2UL;
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
|
||||
if (GpuStep(ref pow, divisor, ref order))
|
||||
return order;
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool GpuStep(ref ulong pow, ulong divisor, ref ulong order)
|
||||
{
|
||||
pow += pow;
|
||||
if (pow >= divisor)
|
||||
pow -= divisor;
|
||||
|
||||
order++;
|
||||
return pow == 1UL;
|
||||
}
|
||||
|
||||
public static ulong CalculateCycleLengthGpu(ulong divisor) => DivisorCycleKernels.CalculateCycleLengthGpu(divisor);
|
||||
|
||||
public static ulong CalculateCycleLength(ulong divisor, in MontgomeryDivisorData divisorData, bool skipPrimeOrderHeuristic = false)
|
||||
{
|
||||
|
||||
@@ -169,7 +169,7 @@ public sealed class PrimeTester(bool useInternal = false)
|
||||
{
|
||||
_accel = accelerator;
|
||||
// Compile once per accelerator and upload primes once.
|
||||
Kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<byte>>(SmallPrimeSieveKernel);
|
||||
Kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<uint>, ArrayView<byte>>(PrimeTesterKernels.SmallPrimeSieveKernel);
|
||||
|
||||
var primes = PrimesGenerator.SmallPrimes;
|
||||
DevicePrimes = accelerator.Allocate1D<uint>(primes.Length);
|
||||
@@ -266,56 +266,6 @@ public sealed class PrimeTester(bool useInternal = false)
|
||||
GpuKernelState.Clear(accelerator);
|
||||
}
|
||||
|
||||
// GPU kernel: small-prime sieve only. Returns 1 if passes sieve (probable prime), 0 otherwise.
|
||||
private static void SmallPrimeSieveKernel(Index1D index, ArrayView<ulong> numbers, ArrayView<uint> smallPrimes, ArrayView<byte> results)
|
||||
{
|
||||
ulong n = numbers[index];
|
||||
if (n <= 3UL)
|
||||
{
|
||||
results[index] = (byte)(n >= 2UL ? 1 : 0);
|
||||
return;
|
||||
}
|
||||
|
||||
if ((n & 1UL) == 0UL || (n > 5UL && (n % 5UL) == 0UL))
|
||||
{
|
||||
// TODO: Replace the `% 5` branch with the GPU Mod5 helper once the sieve kernel can
|
||||
// reuse the benchmarked bitmask reduction instead of modulo on every candidate.
|
||||
results[index] = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (n.Mod10() == 1UL)
|
||||
{
|
||||
// Early reject special GCD heuristic with floor(log2 n)
|
||||
ulong m = 63UL - (ulong)ILGPU.Algorithms.XMath.LeadingZeroCount(n);
|
||||
if (BinaryGcdGpu(n, m) != 1UL)
|
||||
{
|
||||
results[index] = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
long len = smallPrimes.Length;
|
||||
for (int i = 0; i < len; i++)
|
||||
{
|
||||
ulong p = smallPrimes[i];
|
||||
if (p * p > n)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if ((n % p) == 0UL)
|
||||
{
|
||||
// TODO: Route this modulo through the shared divisor-cycle cache once exposed to GPU kernels
|
||||
// so batched sieves avoid per-prime `%` operations that profiling showed expensive.
|
||||
results[index] = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
results[index] = 1;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal static bool SharesFactorWithMaxExponent(ulong n)
|
||||
{
|
||||
@@ -333,7 +283,7 @@ public sealed class PrimeTester(bool useInternal = false)
|
||||
// and divisor-cycle staging instead of allocating new device buffers per call.
|
||||
var gpu = GpuContextPool.Rent();
|
||||
var accelerator = gpu.Accelerator;
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>>(SharesFactorKernel);
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<ulong>, ArrayView<byte>>(PrimeTesterKernels.SharesFactorKernel);
|
||||
|
||||
int length = values.Length;
|
||||
var inputBuffer = accelerator.Allocate1D<ulong>(length);
|
||||
@@ -384,45 +334,5 @@ public sealed class PrimeTester(bool useInternal = false)
|
||||
|
||||
return u << shift;
|
||||
}
|
||||
|
||||
private static void SharesFactorKernel(Index1D index, ArrayView<ulong> numbers, ArrayView<byte> results)
|
||||
{
|
||||
ulong n = numbers[index];
|
||||
ulong m = 63UL - (ulong)XMath.LeadingZeroCount(n);
|
||||
ulong gcd = BinaryGcdGpu(n, m);
|
||||
results[index] = gcd == 1UL ? (byte)0 : (byte)1;
|
||||
}
|
||||
|
||||
private static ulong BinaryGcdGpu(ulong u, ulong v)
|
||||
{
|
||||
// TODO: Replace this inline GPU binary GCD with the kernel extracted from
|
||||
// GpuUInt128BinaryGcdBenchmarks via GpuKernelPool so device callers reuse the
|
||||
// fully unrolled ladder instead of this branchy fallback.
|
||||
if (u == 0UL)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
|
||||
if (v == 0UL)
|
||||
{
|
||||
return u;
|
||||
}
|
||||
|
||||
int shift = XMath.TrailingZeroCount(u | v);
|
||||
u >>= XMath.TrailingZeroCount(u);
|
||||
|
||||
do
|
||||
{
|
||||
v >>= XMath.TrailingZeroCount(v);
|
||||
if (u > v)
|
||||
{
|
||||
(u, v) = (v, u);
|
||||
}
|
||||
|
||||
v -= u;
|
||||
}
|
||||
while (v != 0UL);
|
||||
|
||||
return u << shift;
|
||||
}
|
||||
}
|
||||
@@ -536,7 +536,7 @@ public static class ULongExtensions
|
||||
|
||||
var kernel = KernelCache.GetOrAdd(accelerator, static accel =>
|
||||
{
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, byte, ArrayView1D<ulong, Stride1D.Dense>>(Pow2MontgomeryKernel);
|
||||
var loaded = accel.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, byte, ArrayView1D<ulong, Stride1D.Dense>>(Pow2MontgomeryKernels.Pow2MontgomeryKernel);
|
||||
var launcher = KernelUtil.GetKernel(loaded);
|
||||
return launcher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, byte, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||
});
|
||||
@@ -573,11 +573,6 @@ public static class ULongExtensions
|
||||
}
|
||||
}
|
||||
|
||||
private static void Pow2MontgomeryKernel(Index1D index, ArrayView1D<ulong, Stride1D.Dense> exponents, MontgomeryDivisorData divisor, byte keepMontgomery, ArrayView1D<ulong, Stride1D.Dense> results)
|
||||
{
|
||||
results[index] = Pow2MontgomeryModWindowedKernel(exponents[index], divisor, keepMontgomery != 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static ulong Pow2MontgomeryModSingleBit(ulong exponent, in MontgomeryDivisorData divisor, bool keepMontgomery)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#r "nuget: ILGPU, 1.5.3"
|
||||
#r "nuget: PeterO.Numbers, 1.8.1"
|
||||
#r "nuget: Open.Numeric.Primes, 4.0.4"
|
||||
#load "../PerfectNumbers.Core/Gpu/Kernels/NumericProbeKernels.cs"
|
||||
|
||||
using System;
|
||||
using System.Numerics;
|
||||
@@ -9,6 +10,7 @@ using ILGPU.Runtime;
|
||||
using ILGPU.Runtime.CPU;
|
||||
using PeterO.Numbers;
|
||||
using Open.Numeric.Primes;
|
||||
using PerfectNumbers.Core.Gpu;
|
||||
|
||||
var context = Context.CreateDefault();
|
||||
var accelerator = context.CreateCPUAccelerator(0);
|
||||
@@ -53,7 +55,7 @@ static ProbeResult ProbeBigInteger(Accelerator accelerator)
|
||||
{
|
||||
try
|
||||
{
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(BigIntegerKernel);
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(NumericProbeKernels.BigIntegerKernel);
|
||||
using var buffer = accelerator.Allocate1D<int>(1);
|
||||
var extent = new Index1D((int)buffer.Length);
|
||||
kernel(extent, buffer.View);
|
||||
@@ -72,7 +74,7 @@ static ProbeResult ProbeEInteger(Accelerator accelerator)
|
||||
{
|
||||
try
|
||||
{
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(EIntegerKernel);
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(NumericProbeKernels.EIntegerKernel);
|
||||
using var buffer = accelerator.Allocate1D<int>(1);
|
||||
kernel(new Index1D((int)buffer.Length), buffer.View);
|
||||
accelerator.Synchronize();
|
||||
@@ -92,7 +94,7 @@ static ProbeResult ProbeERational(Accelerator accelerator)
|
||||
|
||||
try
|
||||
{
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(ERationalKernel);
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(NumericProbeKernels.ERationalKernel);
|
||||
using var buffer = accelerator.Allocate1D<int>(1);
|
||||
kernel(new Index1D((int)buffer.Length), buffer.View);
|
||||
accelerator.Synchronize();
|
||||
@@ -119,7 +121,7 @@ static ProbeResult ProbeOpenNumericPrimesIsPrime(Accelerator accelerator)
|
||||
{
|
||||
try
|
||||
{
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(OpenNumericPrimesIsPrimeKernel);
|
||||
var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<int, Stride1D.Dense>>(NumericProbeKernels.OpenNumericPrimesIsPrimeKernel);
|
||||
using var buffer = accelerator.Allocate1D<int>(4);
|
||||
kernel(new Index1D((int)buffer.Length), buffer.View);
|
||||
accelerator.Synchronize();
|
||||
@@ -133,47 +135,6 @@ static ProbeResult ProbeOpenNumericPrimesIsPrime(Accelerator accelerator)
|
||||
}
|
||||
}
|
||||
|
||||
static void OpenNumericPrimesIsPrimeKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
ulong value = (ulong)(index + 2);
|
||||
bool isPrime = Prime.Numbers.IsPrime(value);
|
||||
data[index] = isPrime ? 1 : 0;
|
||||
}
|
||||
|
||||
static void BigIntegerKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
BigInteger value = new BigInteger(data[index]);
|
||||
value += BigInteger.One;
|
||||
value -= BigInteger.One;
|
||||
value *= new BigInteger(2);
|
||||
value /= new BigInteger(3);
|
||||
value %= new BigInteger(5);
|
||||
int comparison = value.CompareTo(BigInteger.Zero);
|
||||
data[index] = comparison;
|
||||
}
|
||||
|
||||
static void EIntegerKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
EInteger value = EInteger.FromInt32(data[index]);
|
||||
value = value + EInteger.One;
|
||||
value = value - EInteger.One;
|
||||
value = value * EInteger.FromInt32(2);
|
||||
value = value.Divide(EInteger.FromInt32(3));
|
||||
value = value.Remainder(EInteger.FromInt32(5));
|
||||
int comparison = value.CompareTo(EInteger.Zero);
|
||||
data[index] = comparison;
|
||||
}
|
||||
|
||||
static void ERationalKernel(Index1D index, ArrayView1D<int, Stride1D.Dense> data)
|
||||
{
|
||||
ERational value = ERational.FromInt32(data[index]);
|
||||
ERational adjusted = (value + ERational.FromInt32(1)) - ERational.FromInt32(1);
|
||||
ERational product = adjusted * ERational.FromInt32(2);
|
||||
ERational quotient = product / ERational.FromInt32(3);
|
||||
int comparison = quotient.CompareTo(ERational.FromInt32(0));
|
||||
data[index] = comparison;
|
||||
}
|
||||
|
||||
static bool HasERationalRemainder()
|
||||
{
|
||||
foreach (var method in typeof(ERational).GetMethods())
|
||||
|
||||
Reference in new issue
Block a user