Make by-divisor GPU kernels reuse read-only operands
This commit is contained in:
Marek Lemańczyk committed
1 parent
cd1debe9d6
commit
6918bd28ff
8 files changed
+594
-250
No files matched your search
@@ -21,7 +21,8 @@ public class MersenneNumberDivisorGpuTesterTests
|
||||
public void IsDivisible_returns_expected(ulong exponent, ulong divisor, bool expected)
|
||||
{
|
||||
var tester = new MersenneNumberDivisorGpuTester();
|
||||
tester.IsDivisible(exponent, divisor).Should().Be(expected);
|
||||
ReadOnlyGpuUInt128 divisorValue = new ReadOnlyGpuUInt128(divisor);
|
||||
tester.IsDivisible(exponent, in divisorValue).Should().Be(expected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -30,8 +31,10 @@ public class MersenneNumberDivisorGpuTesterTests
|
||||
{
|
||||
var tester = new MersenneNumberDivisorGpuTester();
|
||||
UInt128 divisor = (UInt128.One << 65) - UInt128.One;
|
||||
tester.IsDivisible(65UL, divisor).Should().BeTrue();
|
||||
tester.IsDivisible(65UL, divisor + 2).Should().BeFalse();
|
||||
ReadOnlyGpuUInt128 divisorValue = new ReadOnlyGpuUInt128(divisor);
|
||||
tester.IsDivisible(65UL, in divisorValue).Should().BeTrue();
|
||||
divisorValue = new ReadOnlyGpuUInt128(divisor + 2); // Reusing divisorValue for the shifted candidate.
|
||||
tester.IsDivisible(65UL, in divisorValue).Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Numerics;
|
||||
using PerfectNumbers.Core;
|
||||
|
||||
@@ -188,14 +189,32 @@ public sealed class MersenneNumberDivisorByDivisorCpuTester : IMersenneNumberDiv
|
||||
return false;
|
||||
}
|
||||
|
||||
Dictionary<ulong, MersenneDivisorCycles.FactorCacheEntry>? factorCache = null;
|
||||
DivisorCycleCache cycleCache = DivisorCycleCache.Shared;
|
||||
|
||||
while (divisor <= limit)
|
||||
{
|
||||
ulong candidate = (ulong)divisor;
|
||||
processedCount++;
|
||||
lastProcessed = candidate;
|
||||
|
||||
MontgomeryDivisorData divisorData = MontgomeryDivisorData.FromModulus(candidate);
|
||||
if (MersenneDivisorCycles.CycleEqualsExponentForMersenneCandidate(candidate, divisorData, prime))
|
||||
MontgomeryDivisorData divisorData = MontgomeryDivisorDataCache.Get(candidate);
|
||||
ulong divisorCycle = 0UL;
|
||||
|
||||
if (candidate <= PerfectNumberConstants.MaxQForDivisorCycles)
|
||||
{
|
||||
divisorCycle = cycleCache.GetCycleLength(candidate);
|
||||
}
|
||||
else
|
||||
{
|
||||
factorCache ??= new Dictionary<ulong, MersenneDivisorCycles.FactorCacheEntry>(8);
|
||||
if (!MersenneDivisorCycles.TryCalculateCycleLengthForExponent(candidate, prime, divisorData, factorCache, out divisorCycle) || divisorCycle == 0UL)
|
||||
{
|
||||
divisorCycle = cycleCache.GetCycleLength(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
if (divisorCycle != 0UL && CheckDivisor(prime, divisorCycle, divisorData) != 0)
|
||||
{
|
||||
processedAll = true;
|
||||
return true;
|
||||
@@ -208,6 +227,12 @@ public sealed class MersenneNumberDivisorByDivisorCpuTester : IMersenneNumberDiv
|
||||
return false;
|
||||
}
|
||||
|
||||
private static byte CheckDivisor(ulong prime, ulong divisorCycle, in MontgomeryDivisorData divisorData)
|
||||
{
|
||||
ulong residue = prime.Pow2MontgomeryModWithCycle(divisorCycle, divisorData);
|
||||
return residue == 1UL ? (byte)1 : (byte)0;
|
||||
}
|
||||
|
||||
private void UpdateStatusUnsafe(ulong lastProcessed, ulong processedCount)
|
||||
{
|
||||
if (processedCount == 0UL)
|
||||
@@ -286,11 +311,48 @@ public sealed class MersenneNumberDivisorByDivisorCpuTester : IMersenneNumberDiv
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < length; i++)
|
||||
MontgomeryDivisorData cachedData = divisorData;
|
||||
if (cachedData.Modulus != divisor)
|
||||
{
|
||||
ulong prime = primes[i];
|
||||
bool divides = MersenneDivisorCycles.CycleEqualsExponentForMersenneCandidate(divisor, divisorData, prime);
|
||||
hits[i] = divides ? (byte)1 : (byte)0;
|
||||
cachedData = MontgomeryDivisorDataCache.Get(divisor);
|
||||
}
|
||||
|
||||
if (divisorCycle == 0UL)
|
||||
{
|
||||
divisorCycle = DivisorCycleCache.Shared.GetCycleLength(divisor);
|
||||
if (divisorCycle == 0UL)
|
||||
{
|
||||
hits.Clear();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep these remainder steppers in place so future updates continue reusing the previously computed residues.
|
||||
// They are critical for avoiding repeated full Montgomery exponentiation work when scanning divisors.
|
||||
var exponentStepper = new ExponentRemainderStepper(cachedData);
|
||||
if (!exponentStepper.IsValidModulus)
|
||||
{
|
||||
hits.Clear();
|
||||
return;
|
||||
}
|
||||
|
||||
var cycleStepper = new CycleRemainderStepper(divisorCycle);
|
||||
|
||||
ulong remainder = cycleStepper.Initialize(primes[0]);
|
||||
hits[0] = remainder == 0UL
|
||||
? (exponentStepper.ComputeNextIsUnity(primes[0]) ? (byte)1 : (byte)0)
|
||||
: (byte)0;
|
||||
|
||||
for (int i = 1; i < length; i++)
|
||||
{
|
||||
remainder = cycleStepper.ComputeNext(primes[i]);
|
||||
if (remainder != 0UL)
|
||||
{
|
||||
hits[i] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
hits[i] = exponentStepper.ComputeNextIsUnity(primes[i]) ? (byte)1 : (byte)0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -247,6 +247,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
@@ -271,13 +272,13 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Replace these Pow2Mod calls with the ProcessEightBitWindows helper when the shared windowed
|
||||
// scalar implementation lands; benchmarks showed the windowed kernel trimming per-divisor runtime by ~2.4×.
|
||||
if (GpuUInt128.Pow2Mod(phi64, q) != GpuUInt128.One)
|
||||
if (GpuUInt128.Pow2Mod(phi64, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, q) - GpuUInt128.One;
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
@@ -285,7 +286,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
}
|
||||
|
||||
ulong div = FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, q) - GpuUInt128.One;
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
@@ -298,6 +299,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
private static void OrderKernelScan(Index1D index, ulong exponent, ulong divMul, ArrayView<GpuUInt128> qs, ArrayView<ulong> orders)
|
||||
{
|
||||
GpuUInt128 q = qs[index];
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
GpuUInt128 phi = q - GpuUInt128.One;
|
||||
if (phi.High != 0UL)
|
||||
{
|
||||
@@ -308,14 +310,14 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Once the ProcessEightBitWindows helper is available, switch this order kernel to that faster
|
||||
// Pow2Mod variant so cycle checks inherit the same gains observed in GpuPow2ModBenchmarks.
|
||||
GpuUInt128 pow = GpuUInt128.Pow2Mod(phi64, q);
|
||||
GpuUInt128 pow = GpuUInt128.Pow2Mod(phi64, in readOnlyQ);
|
||||
if (pow != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, q) - GpuUInt128.One;
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
@@ -323,7 +325,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
}
|
||||
|
||||
ulong div = FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, q) - GpuUInt128.One;
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
@@ -410,6 +412,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(kStart);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
ulong cycle = smallCycles[(int)q.Low];
|
||||
@@ -422,7 +425,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
}
|
||||
// TODO: Swap Pow2Minus1Mod for the eight-bit window helper once the scalar version switches;
|
||||
// benchmarks show the windowed variant cuts large-divisor scans from ~51 µs to ~21 µs.
|
||||
if (GpuUInt128.Pow2Minus1Mod(exponent, q) != GpuUInt128.Zero)
|
||||
if (GpuUInt128.Pow2Minus1Mod(exponent, in readOnlyQ) != GpuUInt128.Zero)
|
||||
{
|
||||
orders[index] = 0UL;
|
||||
return;
|
||||
@@ -559,6 +562,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
@@ -578,19 +582,19 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
ulong phi64 = phi.Low;
|
||||
// TODO: Upgrade this pow2mod order kernel to the ProcessEightBitWindows helper once available so GPU residue
|
||||
// scans avoid the single-bit ladder that benchmarks found to be 2.3× slower on large exponents.
|
||||
if (GpuUInt128.Pow2Mod(phi64, q) != GpuUInt128.One)
|
||||
if (GpuUInt128.Pow2Mod(phi64, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, q) - GpuUInt128.One;
|
||||
GpuUInt128 halfPow = GpuUInt128.Pow2Mod(phi64 >> 1, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(halfPow, q) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong div = FastDiv64Gpu(phi64, exponent, divMul);
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, q) - GpuUInt128.One;
|
||||
GpuUInt128 divPow = GpuUInt128.Pow2Mod(div, in readOnlyQ) - GpuUInt128.One;
|
||||
if (GpuUInt128.BinaryGcd(divPow, q) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
@@ -645,6 +649,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
GpuUInt128 q = twoP;
|
||||
q.Mul64(k);
|
||||
q.Add(GpuUInt128.One);
|
||||
ReadOnlyGpuUInt128 readOnlyQ = q.AsReadOnly();
|
||||
// Small-cycles in-kernel early rejection from device table
|
||||
if (q.High == 0UL && q.Low < (ulong)smallCycles.Length)
|
||||
{
|
||||
@@ -656,7 +661,7 @@ public ref struct GpuKernelLease(IDisposable limiter, GpuContextLease gpu, Kerne
|
||||
}
|
||||
// TODO: Replace this Pow2Mod check with the ProcessEightBitWindows helper once Pow2Minus1Mod adopts it;
|
||||
// residue order scans will then benefit from the same 2× speedup the benchmarked windowed kernel delivered.
|
||||
if (GpuUInt128.Pow2Mod(exponent, q) != GpuUInt128.One)
|
||||
if (GpuUInt128.Pow2Mod(exponent, in readOnlyQ) != GpuUInt128.One)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -81,23 +81,31 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator GpuUInt128(ulong value) => new(value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator GpuUInt128(ReadOnlyGpuUInt128 value) => new(value.High, value.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator UInt128(GpuUInt128 value) =>
|
||||
((UInt128)value.High << 64) | value.Low;
|
||||
|
||||
// TODO: The operators modifying the state of left is intuitive and can easily lead to bugs. Change this back to return a new instance, but replace where possible with method calls, if we want to modify the state of the instance.
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly ReadOnlyGpuUInt128 AsReadOnly() => new(this);
|
||||
|
||||
// Operators must return fresh instances so callers never rely on accidental in-place mutation. Prefer the mutating helpers when state updates are required.
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 operator +(GpuUInt128 left, GpuUInt128 right)
|
||||
{
|
||||
left.Add(right);
|
||||
return left;
|
||||
GpuUInt128 result = new(left.High, left.Low);
|
||||
result.Add(right);
|
||||
return result;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 operator -(GpuUInt128 left, GpuUInt128 right)
|
||||
{
|
||||
left.Sub(right);
|
||||
return left;
|
||||
GpuUInt128 result = new(left.High, left.Low);
|
||||
result.Sub(right);
|
||||
return result;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
@@ -108,63 +116,63 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
public static GpuUInt128 operator *(GpuUInt128 left, GpuUInt128 right)
|
||||
{
|
||||
Multiply(left, right, out var high, out var low);
|
||||
left.High = high;
|
||||
left.Low = low;
|
||||
return left;
|
||||
return new GpuUInt128(high, low);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 operator <<(GpuUInt128 value, int shift)
|
||||
{
|
||||
shift &= 127;
|
||||
GpuUInt128 result = new(value.High, value.Low);
|
||||
if (shift == 0)
|
||||
{
|
||||
return value;
|
||||
return result;
|
||||
}
|
||||
|
||||
ulong high = value.High;
|
||||
ulong low = value.Low;
|
||||
ulong high = result.High;
|
||||
ulong low = result.Low;
|
||||
|
||||
int shiftedValue;
|
||||
if (shift >= 64)
|
||||
{
|
||||
shiftedValue = shift - 64;
|
||||
value.High = low << shiftedValue;
|
||||
value.Low = 0UL;
|
||||
return value;
|
||||
result.High = low << shiftedValue;
|
||||
result.Low = 0UL;
|
||||
return result;
|
||||
}
|
||||
|
||||
shiftedValue = 64 - shift;
|
||||
value.High = (high << shift) | (low >> shiftedValue);
|
||||
value.Low = low << shift;
|
||||
return value;
|
||||
result.High = (high << shift) | (low >> shiftedValue);
|
||||
result.Low = low << shift;
|
||||
return result;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 operator >>(GpuUInt128 value, int shift)
|
||||
{
|
||||
shift &= 127;
|
||||
GpuUInt128 result = new(value.High, value.Low);
|
||||
if (shift == 0)
|
||||
{
|
||||
return value;
|
||||
return result;
|
||||
}
|
||||
|
||||
ulong high = value.High;
|
||||
ulong low = value.Low;
|
||||
ulong high = result.High;
|
||||
ulong low = result.Low;
|
||||
|
||||
int shiftedValue;
|
||||
if (shift >= 64)
|
||||
{
|
||||
shiftedValue = shift - 64;
|
||||
value.Low = shiftedValue == 0 ? high : high >> shiftedValue;
|
||||
value.High = 0UL;
|
||||
return value;
|
||||
result.Low = shiftedValue == 0 ? high : high >> shiftedValue;
|
||||
result.High = 0UL;
|
||||
return result;
|
||||
}
|
||||
|
||||
shiftedValue = 64 - shift;
|
||||
value.Low = (low >> shift) | (high << shiftedValue);
|
||||
value.High = high >> shift;
|
||||
return value;
|
||||
result.Low = (low >> shift) | (high << shiftedValue);
|
||||
result.High = high >> shift;
|
||||
return result;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
@@ -188,8 +196,9 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 operator ++(GpuUInt128 value)
|
||||
{
|
||||
value.Add(1UL);
|
||||
return value;
|
||||
GpuUInt128 result = new(value.High, value.Low);
|
||||
result.Add(1UL);
|
||||
return result;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
@@ -218,6 +227,33 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
return 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly int CompareTo(in ReadOnlyGpuUInt128 other)
|
||||
{
|
||||
if (High < other.High)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (High > other.High)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
ulong otherLow = other.Low;
|
||||
if (Low < otherLow)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (Low > otherLow)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly bool Equals(GpuUInt128 other) => High == other.High && Low == other.Low;
|
||||
|
||||
@@ -228,15 +264,21 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
public override readonly int GetHashCode() => HashCode.Combine(High, Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Add(GpuUInt128 other)
|
||||
public void Add(in ReadOnlyGpuUInt128 other) => AddInternal(other.High, other.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Add(in GpuUInt128 other) => AddInternal(other.High, other.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void AddInternal(ulong otherHigh, ulong otherLow)
|
||||
{
|
||||
ulong originalLow = Low;
|
||||
// We're reusing originalLow for carry to avoid an extra local.
|
||||
ulong low = originalLow + other.Low;
|
||||
ulong low = originalLow + otherLow;
|
||||
originalLow = low < originalLow ? 1UL : 0UL;
|
||||
Low = low;
|
||||
// We're reusing low for the final sum to avoid an extra local and to keep max stack frame at 3 instead of 4 with one statement.
|
||||
low = High + other.High;
|
||||
low = High + otherHigh;
|
||||
High = low + originalLow;
|
||||
}
|
||||
|
||||
@@ -249,7 +291,43 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(GpuUInt128 value, GpuUInt128 modulus)
|
||||
public void AddMod(in ReadOnlyGpuUInt128 value, in ReadOnlyGpuUInt128 modulus) => AddModInternal(value.High, value.Low, in modulus);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(in GpuUInt128 value, in ReadOnlyGpuUInt128 modulus) => AddModInternal(value.High, value.Low, in modulus);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(in ReadOnlyGpuUInt128 value, in GpuUInt128 modulus)
|
||||
{
|
||||
ReadOnlyGpuUInt128 readOnlyMod = modulus.AsReadOnly();
|
||||
AddModInternal(value.High, value.Low, in readOnlyMod);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(in GpuUInt128 value, in GpuUInt128 modulus)
|
||||
{
|
||||
ReadOnlyGpuUInt128 readOnlyMod = modulus.AsReadOnly();
|
||||
AddModInternal(value.High, value.Low, in readOnlyMod);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void AddModInternal(ulong valueHigh, ulong valueLow, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
AddMod(valueLow, modulus.Low);
|
||||
return;
|
||||
}
|
||||
|
||||
AddInternal(valueHigh, valueLow);
|
||||
if (CompareTo(in modulus) >= 0)
|
||||
{
|
||||
Sub(in modulus);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(ulong value, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
@@ -258,30 +336,14 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
Add(value);
|
||||
if (CompareTo(modulus) >= 0)
|
||||
if (CompareTo(in modulus) >= 0)
|
||||
{
|
||||
Sub(modulus);
|
||||
Sub(in modulus);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(ulong value, GpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
AddMod(value, modulus.Low);
|
||||
return;
|
||||
}
|
||||
|
||||
Add(value);
|
||||
if (CompareTo(modulus) >= 0)
|
||||
{
|
||||
Sub(modulus);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 Pow2Minus1Mod(ulong exponent, GpuUInt128 modulus)
|
||||
public static GpuUInt128 Pow2Minus1Mod(ulong exponent, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
// This should never happen in production code
|
||||
// if (modulus.IsZero)
|
||||
@@ -300,7 +362,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void Pow2Minus1ModBatch(GpuUInt128 modulus, ReadOnlySpan<ulong> exponents, Span<GpuUInt128> results)
|
||||
public static void Pow2Minus1ModBatch(in ReadOnlyGpuUInt128 modulus, ReadOnlySpan<ulong> exponents, Span<GpuUInt128> results)
|
||||
{
|
||||
int count = exponents.Length;
|
||||
for (int i = 0; i < count; i++)
|
||||
@@ -310,7 +372,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(GpuUInt128 value, ulong modulus)
|
||||
public void AddMod(in ReadOnlyGpuUInt128 value, ulong modulus)
|
||||
{
|
||||
// TODO: Pre-reduce the operands via the Montgomery ladder used in MulMod64Benchmarks so the GPU
|
||||
// compatible shim stops paying for `%` on every call; the InlineUInt128 helper ran 6–82× faster on
|
||||
@@ -327,6 +389,9 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
Low = sum;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(in GpuUInt128 value, ulong modulus) => AddMod(value.Low, modulus);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AddMod(ulong value, ulong modulus)
|
||||
{
|
||||
@@ -346,11 +411,17 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Sub(GpuUInt128 other)
|
||||
public void Sub(in ReadOnlyGpuUInt128 other) => SubInternal(other.High, other.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Sub(in GpuUInt128 other) => SubInternal(other.High, other.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void SubInternal(ulong otherHigh, ulong otherLow)
|
||||
{
|
||||
ulong borrow = Low < other.Low ? 1UL : 0UL;
|
||||
High = High - other.High - borrow;
|
||||
Low -= other.Low;
|
||||
ulong borrow = Low < otherLow ? 1UL : 0UL;
|
||||
High = High - otherHigh - borrow;
|
||||
Low -= otherLow;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
@@ -387,20 +458,27 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Mul64(GpuUInt128 other)
|
||||
public void Mul64(in ReadOnlyGpuUInt128 other) => Mul64Internal(other.High, other.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Mul64(in GpuUInt128 other) => Mul64Internal(other.High, other.Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void Mul64Internal(ulong otherHigh, ulong otherLow)
|
||||
{
|
||||
// Multiply this.Low (assumed 64-bit value) by full 128-bit other
|
||||
ulong operand = Low;
|
||||
Low = operand * other.Low;
|
||||
ulong highProduct = operand * other.High;
|
||||
High = highProduct + MulHigh(operand, other.Low);
|
||||
Low = operand * otherLow;
|
||||
ulong highProduct = operand * otherHigh;
|
||||
High = highProduct + MulHigh(operand, otherLow);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MulMod(in GpuUInt128 value, in GpuUInt128 modulus)
|
||||
// Kept by-value to coexist with the ref readonly overload below while still signaling read-only intent.
|
||||
public void MulMod(ReadOnlyGpuUInt128 value, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
GpuUInt128 multiplicand = this;
|
||||
GpuUInt128 multiplier = value;
|
||||
GpuUInt128 multiplier = value.ToMutable();
|
||||
|
||||
High = 0UL;
|
||||
Low = 0UL;
|
||||
@@ -423,7 +501,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 Pow2Mod(ulong exponent, GpuUInt128 modulus)
|
||||
public static GpuUInt128 Pow2Mod(ulong exponent, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
// This should never happen in production code.
|
||||
// if (modulus.IsZero || modulus == One)
|
||||
@@ -499,7 +577,18 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
return result;
|
||||
}
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static int TrailingZeroCount(GpuUInt128 value)
|
||||
public static int TrailingZeroCount(in ReadOnlyGpuUInt128 value)
|
||||
{
|
||||
if (value.Low != 0UL)
|
||||
{
|
||||
return BitOperations.TrailingZeroCount(value.Low);
|
||||
}
|
||||
|
||||
return 64 + BitOperations.TrailingZeroCount(value.High);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static int TrailingZeroCount(in GpuUInt128 value)
|
||||
{
|
||||
if (value.Low != 0UL)
|
||||
{
|
||||
@@ -554,21 +643,66 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SubMod(GpuUInt128 value, GpuUInt128 modulus)
|
||||
public void SubMod(in ReadOnlyGpuUInt128 value, in ReadOnlyGpuUInt128 modulus) => SubModInternal(value.High, value.Low, in modulus);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SubMod(in GpuUInt128 value, in ReadOnlyGpuUInt128 modulus) => SubModInternal(value.High, value.Low, in modulus);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SubMod(in ReadOnlyGpuUInt128 value, in GpuUInt128 modulus)
|
||||
{
|
||||
if (CompareTo(value) >= 0)
|
||||
{
|
||||
Sub(value);
|
||||
}
|
||||
else
|
||||
{
|
||||
Add(modulus);
|
||||
Sub(value);
|
||||
}
|
||||
ReadOnlyGpuUInt128 readOnlyMod = modulus.AsReadOnly();
|
||||
SubModInternal(value.High, value.Low, in readOnlyMod);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SubMod(ulong value, GpuUInt128 modulus)
|
||||
public void SubMod(in GpuUInt128 value, in GpuUInt128 modulus)
|
||||
{
|
||||
ReadOnlyGpuUInt128 readOnlyMod = modulus.AsReadOnly();
|
||||
SubModInternal(value.High, value.Low, in readOnlyMod);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void SubModInternal(ulong valueHigh, ulong valueLow, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (CompareToRaw(valueHigh, valueLow) >= 0)
|
||||
{
|
||||
SubInternal(valueHigh, valueLow);
|
||||
return;
|
||||
}
|
||||
|
||||
Add(in modulus);
|
||||
SubInternal(valueHigh, valueLow);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private readonly int CompareToRaw(ulong otherHigh, ulong otherLow)
|
||||
{
|
||||
if (High < otherHigh)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (High > otherHigh)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (Low < otherLow)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (Low > otherLow)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SubMod(ulong value, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
ulong low = Low;
|
||||
ulong high = High;
|
||||
@@ -584,9 +718,20 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Mul(GpuUInt128 other)
|
||||
public void Mul(in ReadOnlyGpuUInt128 other)
|
||||
{
|
||||
Multiply(this, other, out var high, out var low);
|
||||
ReadOnlyGpuUInt128 readOnlyThis = AsReadOnly();
|
||||
Multiply(readOnlyThis, other, out var high, out var low);
|
||||
High = high;
|
||||
Low = low;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Mul(in GpuUInt128 other)
|
||||
{
|
||||
ReadOnlyGpuUInt128 readOnlyThis = AsReadOnly();
|
||||
ReadOnlyGpuUInt128 readOnlyOther = other.AsReadOnly();
|
||||
Multiply(readOnlyThis, readOnlyOther, out var high, out var low);
|
||||
High = high;
|
||||
Low = low;
|
||||
}
|
||||
@@ -600,14 +745,15 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static GpuUInt128 DivideExact(GpuUInt128 dividend, GpuUInt128 divisor)
|
||||
public static GpuUInt128 DivideExact(in ReadOnlyGpuUInt128 dividend, in ReadOnlyGpuUInt128 divisor)
|
||||
{
|
||||
int divisorBits = divisor.GetBitLength();
|
||||
int shift = dividend.GetBitLength() - divisorBits;
|
||||
|
||||
GpuUInt128 quotient = Zero;
|
||||
GpuUInt128 remainder = dividend;
|
||||
GpuUInt128 shiftedDivisor = divisor << shift;
|
||||
GpuUInt128 shiftedDivisor = divisor;
|
||||
shiftedDivisor.ShiftLeft(shift);
|
||||
GpuUInt128 quotientBit = One << shift;
|
||||
|
||||
while (true)
|
||||
@@ -637,7 +783,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Xor(GpuUInt128 other)
|
||||
public void Xor(in ReadOnlyGpuUInt128 other)
|
||||
{
|
||||
High ^= other.High;
|
||||
Low ^= other.Low;
|
||||
@@ -697,7 +843,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
High = high >> shift;
|
||||
}
|
||||
|
||||
public void MulMod(GpuUInt128 other, GpuUInt128 modulus)
|
||||
public void MulMod(in ReadOnlyGpuUInt128 other, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
@@ -728,7 +874,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
/// method is intended for validation only and should not be used inside
|
||||
/// GPU kernels.
|
||||
/// </summary>
|
||||
internal void MulModBigInteger(GpuUInt128 other, GpuUInt128 modulus)
|
||||
internal void MulModBigInteger(in ReadOnlyGpuUInt128 other, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
var left = (BigInteger)(UInt128)this;
|
||||
var right = (BigInteger)(UInt128)other;
|
||||
@@ -743,7 +889,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
/// repeated subtractions and becomes extremely slow for large remainders.
|
||||
/// Kept for future optimization work.
|
||||
/// </summary>
|
||||
internal void MulModByLimb(GpuUInt128 other, GpuUInt128 modulus)
|
||||
internal void MulModByLimb(in ReadOnlyGpuUInt128 other, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
// TODO: Relocate this limb-based reducer to the benchmark project once the production
|
||||
// pipeline switches to the faster allocating legacy path demonstrated in the benchmarks.
|
||||
@@ -772,7 +918,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MulMod(ulong value, GpuUInt128 modulus)
|
||||
public void MulMod(ulong value, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
@@ -786,7 +932,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly ulong MulMod(GpuUInt128 other, ulong modulus)
|
||||
public readonly ulong MulMod(in ReadOnlyGpuUInt128 other, ulong modulus)
|
||||
{
|
||||
ulong modulusLocal = modulus;
|
||||
ulong a = Low % modulusLocal;
|
||||
@@ -870,7 +1016,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void ReduceProductBitwise(ulong p3, ulong p2, ulong p1, ulong p0, GpuUInt128 modulus)
|
||||
private void ReduceProductBitwise(ulong p3, ulong p2, ulong p1, ulong p0, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
GpuUInt128 remainder = new();
|
||||
int bit;
|
||||
@@ -960,7 +1106,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly ulong MulModMontgomery64(GpuUInt128 other, ulong modulus, ulong nPrime, ulong r2)
|
||||
public readonly ulong MulModMontgomery64(in ReadOnlyGpuUInt128 other, ulong modulus, ulong nPrime, ulong r2)
|
||||
{
|
||||
// TODO: Retire this struct-based Montgomery path from production after adopting the extension
|
||||
// helper, which benchmarks 6-7× faster across dense and near-modulus operands.
|
||||
@@ -999,7 +1145,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SquareMod(GpuUInt128 modulus)
|
||||
public void SquareMod(in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
@@ -1058,7 +1204,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
Low = remainder.Low;
|
||||
}
|
||||
|
||||
public void ModPow(GpuUInt128 exponent, GpuUInt128 modulus)
|
||||
public void ModPow(GpuUInt128 exponent, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
// TODO: Swap this copy-heavy path for the pooled base/exponent ladder used in
|
||||
// GpuUInt128MulModBenchmarks so the GPU shim keeps reusing buffers instead of
|
||||
@@ -1084,7 +1230,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
Low = result.Low;
|
||||
}
|
||||
|
||||
public void ModPow(ulong exponent, GpuUInt128 modulus)
|
||||
public void ModPow(ulong exponent, in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
// TODO: Share the pooled ladder state from GpuUInt128MulModBenchmarks here as well so
|
||||
// Lucas–Lehmer batches avoid constructing throwaway temporaries on every exponent.
|
||||
@@ -1118,7 +1264,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ModInv(GpuUInt128 modulus)
|
||||
public void ModInv(in ReadOnlyGpuUInt128 modulus)
|
||||
{
|
||||
if (modulus.High == 0UL)
|
||||
{
|
||||
@@ -1136,7 +1282,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
((UInt128)this).ToString();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void MultiplyFull(GpuUInt128 left, GpuUInt128 right, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
private static void MultiplyFull(in ReadOnlyGpuUInt128 left, in ReadOnlyGpuUInt128 right, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
{
|
||||
// TODO: Reuse variables to reduce register pressure following the fused-limb layout
|
||||
// from GpuUInt128MulModByLimbBenchmarks so the multiply helper matches the fastest
|
||||
@@ -1182,7 +1328,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void MultiplyFull(GpuUInt128 left, ulong right, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
private static void MultiplyFull(in ReadOnlyGpuUInt128 left, ulong right, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
{
|
||||
var (highLow, lowLow) = Mul64(left.Low, right);
|
||||
var (highHigh, lowHigh) = Mul64(left.High, right);
|
||||
@@ -1199,7 +1345,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void Multiply(GpuUInt128 left, GpuUInt128 right, out ulong high, out ulong low)
|
||||
private static void Multiply(in ReadOnlyGpuUInt128 left, in ReadOnlyGpuUInt128 right, out ulong high, out ulong low)
|
||||
{
|
||||
// TODO: Reuse variables to reduce register pressure.
|
||||
var (h0, l0) = Mul64(left.Low, right.Low);
|
||||
@@ -1234,7 +1380,7 @@ public struct GpuUInt128 : IComparable<GpuUInt128>, IEquatable<GpuUInt128>
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SquareFull(GpuUInt128 value, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
private static void SquareFull(in ReadOnlyGpuUInt128 value, out ulong p3, out ulong p2, out ulong p1, out ulong p0)
|
||||
{
|
||||
// TODO: Reuse variables to reduce register pressure.
|
||||
// Compute (H*2^64 + L)^2 = H^2*2^128 + 2*H*L*2^64 + L^2
|
||||
|
||||
@@ -115,6 +115,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
resources.Exponents,
|
||||
resources.Hits,
|
||||
resources.DivisorData,
|
||||
resources.CycleCandidates,
|
||||
resources.CycleLengths,
|
||||
resources.CycleCapacity,
|
||||
out lastProcessed,
|
||||
out coveredRange,
|
||||
out processedCount
|
||||
@@ -178,6 +181,9 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
ulong[] exponents,
|
||||
byte[] hits,
|
||||
MontgomeryDivisorData[] divisorData,
|
||||
ulong[] cycleCandidates,
|
||||
ulong[] cycleLengths,
|
||||
int cycleCapacity,
|
||||
out ulong lastProcessed,
|
||||
out bool coveredRange,
|
||||
out ulong processedCount)
|
||||
@@ -196,13 +202,20 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
ArrayView1D<ulong, Stride1D.Dense> exponentView;
|
||||
ArrayView1D<byte, Stride1D.Dense> hitsView;
|
||||
|
||||
int chunkCapacity = Math.Min(batchCapacity, (int)accelerator.MaxNumThreadsPerGroup);
|
||||
if (chunkCapacity <= 0)
|
||||
cycleCapacity = Math.Max(1, cycleCapacity);
|
||||
|
||||
int chunkCapacity = Math.Min(batchCapacity, cycleCapacity);
|
||||
int maxThreadsPerGroup = (int)accelerator.MaxNumThreadsPerGroup;
|
||||
if (maxThreadsPerGroup > 0)
|
||||
{
|
||||
chunkCapacity = batchCapacity;
|
||||
chunkCapacity = Math.Min(chunkCapacity, maxThreadsPerGroup);
|
||||
}
|
||||
|
||||
if (chunkCapacity <= 0)
|
||||
{
|
||||
chunkCapacity = cycleCapacity;
|
||||
}
|
||||
|
||||
ulong[] candidateDivisors = ArrayPool<ulong>.Shared.Rent(chunkCapacity);
|
||||
ulong[] filteredDivisors = ArrayPool<ulong>.Shared.Rent(chunkCapacity);
|
||||
ulong[] divisorGaps = ArrayPool<ulong>.Shared.Rent(chunkCapacity);
|
||||
byte[] remainder10Buffer = ArrayPool<byte>.Shared.Rent(chunkCapacity);
|
||||
@@ -246,6 +259,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
byte remainder5 = (byte)((ulong)(currentDivisor128 % 5UL));
|
||||
byte remainder3 = (byte)((ulong)(currentDivisor128 % 3UL));
|
||||
bool lastIsSeven = (prime & 3UL) == 3UL;
|
||||
DivisorCycleCache cycleCache = DivisorCycleCache.Shared;
|
||||
|
||||
try
|
||||
{
|
||||
@@ -263,7 +277,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
break;
|
||||
}
|
||||
|
||||
Span<ulong> candidateSpan = candidateDivisors.AsSpan(0, chunkCount);
|
||||
Span<ulong> candidateSpan = cycleCandidates.AsSpan(0, chunkCount);
|
||||
Span<ulong> gapSpan = divisorGaps.AsSpan(0, chunkCount);
|
||||
Span<byte> rem10Span = remainder10Buffer.AsSpan(0, chunkCount);
|
||||
Span<byte> rem8Span = remainder8Buffer.AsSpan(0, chunkCount);
|
||||
@@ -350,40 +364,46 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
continue;
|
||||
}
|
||||
|
||||
int batchSize = 0;
|
||||
Span<ulong> cycleSpan = cycleLengths.AsSpan(0, filteredCount);
|
||||
cycleCache.GetCycleLengths(filteredDivisorsSpan[..filteredCount], cycleSpan);
|
||||
|
||||
int admissibleCount = 0;
|
||||
for (int i = 0; i < filteredCount; i++)
|
||||
{
|
||||
ulong divisorValue = filteredDivisorsSpan[i];
|
||||
divisors[batchSize] = divisorValue;
|
||||
exponents[batchSize] = prime;
|
||||
batchSize++;
|
||||
if (cycleSpan[i] == prime)
|
||||
{
|
||||
filteredDivisorsSpan[admissibleCount++] = filteredDivisorsSpan[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (batchSize == 0)
|
||||
if (admissibleCount == 0)
|
||||
{
|
||||
processedAll = currentK > maxK;
|
||||
continue;
|
||||
}
|
||||
|
||||
divisorDataSpan = divisorData.AsSpan(0, batchSize);
|
||||
divisorSpan = divisors.AsSpan(0, batchSize);
|
||||
exponentSpan = exponents.AsSpan(0, batchSize);
|
||||
hitsSpan = hits.AsSpan(0, batchSize);
|
||||
divisorView = divisorsBuffer.View.SubView(0, batchSize);
|
||||
exponentView = exponentBuffer.View.SubView(0, batchSize);
|
||||
hitsView = hitsBuffer.View.SubView(0, batchSize);
|
||||
divisorDataSpan = divisorData.AsSpan(0, admissibleCount);
|
||||
divisorSpan = divisors.AsSpan(0, admissibleCount);
|
||||
exponentSpan = exponents.AsSpan(0, admissibleCount);
|
||||
hitsSpan = hits.AsSpan(0, admissibleCount);
|
||||
divisorView = divisorsBuffer.View.SubView(0, admissibleCount);
|
||||
exponentView = exponentBuffer.View.SubView(0, admissibleCount);
|
||||
hitsView = hitsBuffer.View.SubView(0, admissibleCount);
|
||||
|
||||
for (int i = 0; i < batchSize; i++)
|
||||
for (int i = 0; i < admissibleCount; i++)
|
||||
{
|
||||
divisorDataSpan[i] = MontgomeryDivisorData.FromModulus(divisorSpan[i]);
|
||||
ulong divisorValue = filteredDivisorsSpan[i];
|
||||
divisorSpan[i] = divisorValue;
|
||||
exponentSpan[i] = prime;
|
||||
divisorDataSpan[i] = MontgomeryDivisorData.FromModulus(divisorValue);
|
||||
}
|
||||
|
||||
divisorView.CopyFromCPU(ref MemoryMarshal.GetReference(divisorDataSpan), batchSize);
|
||||
exponentView.CopyFromCPU(ref MemoryMarshal.GetReference(exponentSpan), batchSize);
|
||||
kernel(batchSize, divisorView, exponentView, hitsView);
|
||||
hitsView.CopyToCPU(ref MemoryMarshal.GetReference(hitsSpan), batchSize);
|
||||
divisorView.CopyFromCPU(ref MemoryMarshal.GetReference(divisorDataSpan), admissibleCount);
|
||||
exponentView.CopyFromCPU(ref MemoryMarshal.GetReference(exponentSpan), admissibleCount);
|
||||
kernel(admissibleCount, divisorView, exponentView, hitsView);
|
||||
hitsView.CopyToCPU(ref MemoryMarshal.GetReference(hitsSpan), admissibleCount);
|
||||
|
||||
for (int i = 0; i < batchSize; i++)
|
||||
for (int i = 0; i < admissibleCount; i++)
|
||||
{
|
||||
if (hitsSpan[i] != 0)
|
||||
{
|
||||
@@ -395,7 +415,7 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
|
||||
if (!composite)
|
||||
{
|
||||
lastProcessed = divisorSpan[batchSize - 1];
|
||||
lastProcessed = divisorSpan[admissibleCount - 1];
|
||||
}
|
||||
|
||||
if (composite)
|
||||
@@ -408,7 +428,6 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<ulong>.Shared.Return(candidateDivisors, clearArray: false);
|
||||
ArrayPool<ulong>.Shared.Return(filteredDivisors, clearArray: false);
|
||||
ArrayPool<ulong>.Shared.Return(divisorGaps, clearArray: false);
|
||||
ArrayPool<byte>.Shared.Return(remainder10Buffer, clearArray: false);
|
||||
@@ -712,19 +731,30 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
{
|
||||
internal BatchResources(MersenneNumberDivisorByDivisorGpuTester owner, Accelerator accelerator, int capacity)
|
||||
{
|
||||
DivisorsBuffer = accelerator.Allocate1D<MontgomeryDivisorData>(capacity);
|
||||
ExponentBuffer = accelerator.Allocate1D<ulong>(capacity);
|
||||
HitsBuffer = accelerator.Allocate1D<byte>(capacity);
|
||||
DivisorDeltaBuffer = accelerator.Allocate1D<ulong>(capacity);
|
||||
RemainderDeltaBuffer = accelerator.Allocate1D<byte>(capacity);
|
||||
RemainderBuffer = accelerator.Allocate1D<byte>(capacity);
|
||||
Divisors = ArrayPool<ulong>.Shared.Rent(capacity);
|
||||
Exponents = ArrayPool<ulong>.Shared.Rent(capacity);
|
||||
Hits = ArrayPool<byte>.Shared.Rent(capacity);
|
||||
DivisorData = ArrayPool<MontgomeryDivisorData>.Shared.Rent(capacity);
|
||||
int cycleCapacity = DivisorCycleCache.Shared.PreferredBatchSize;
|
||||
if (cycleCapacity <= 0)
|
||||
{
|
||||
cycleCapacity = 1;
|
||||
}
|
||||
|
||||
int actualCapacity = Math.Max(capacity, cycleCapacity);
|
||||
|
||||
DivisorsBuffer = accelerator.Allocate1D<MontgomeryDivisorData>(actualCapacity);
|
||||
ExponentBuffer = accelerator.Allocate1D<ulong>(actualCapacity);
|
||||
HitsBuffer = accelerator.Allocate1D<byte>(actualCapacity);
|
||||
DivisorDeltaBuffer = accelerator.Allocate1D<ulong>(actualCapacity);
|
||||
RemainderDeltaBuffer = accelerator.Allocate1D<byte>(actualCapacity);
|
||||
RemainderBuffer = accelerator.Allocate1D<byte>(actualCapacity);
|
||||
Divisors = ArrayPool<ulong>.Shared.Rent(actualCapacity);
|
||||
Exponents = ArrayPool<ulong>.Shared.Rent(actualCapacity);
|
||||
Hits = ArrayPool<byte>.Shared.Rent(actualCapacity);
|
||||
DivisorData = ArrayPool<MontgomeryDivisorData>.Shared.Rent(actualCapacity);
|
||||
CycleCandidates = ArrayPool<ulong>.Shared.Rent(cycleCapacity);
|
||||
CycleLengths = ArrayPool<ulong>.Shared.Rent(cycleCapacity);
|
||||
RemainderDeltaKernel = owner.GetRemainderDeltaKernel(accelerator);
|
||||
RemainderScanKernel = owner.GetRemainderScanKernel(accelerator);
|
||||
Capacity = capacity;
|
||||
Capacity = actualCapacity;
|
||||
CycleCapacity = cycleCapacity;
|
||||
}
|
||||
|
||||
internal readonly MemoryBuffer1D<MontgomeryDivisorData, Stride1D.Dense> DivisorsBuffer;
|
||||
@@ -747,12 +777,18 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
|
||||
internal readonly MontgomeryDivisorData[] DivisorData;
|
||||
|
||||
internal readonly ulong[] CycleCandidates;
|
||||
|
||||
internal readonly ulong[] CycleLengths;
|
||||
|
||||
internal readonly Action<Index1D, ArrayView<ulong>, ArrayView<byte>, byte> RemainderDeltaKernel;
|
||||
|
||||
internal readonly Action<KernelConfig, Index1D, ArrayView<byte>, ArrayView<byte>, byte, byte> RemainderScanKernel;
|
||||
|
||||
internal readonly int Capacity;
|
||||
|
||||
internal readonly int CycleCapacity;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
DivisorsBuffer.Dispose();
|
||||
@@ -765,6 +801,8 @@ public sealed class MersenneNumberDivisorByDivisorGpuTester : IMersenneNumberDiv
|
||||
ArrayPool<ulong>.Shared.Return(Exponents, clearArray: false);
|
||||
ArrayPool<byte>.Shared.Return(Hits, clearArray: false);
|
||||
ArrayPool<MontgomeryDivisorData>.Shared.Return(DivisorData, clearArray: false);
|
||||
ArrayPool<ulong>.Shared.Return(CycleCandidates, clearArray: false);
|
||||
ArrayPool<ulong>.Shared.Return(CycleLengths, clearArray: false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,11 +10,11 @@ namespace PerfectNumbers.Core.Gpu;
|
||||
/// </summary>
|
||||
public sealed class MersenneNumberDivisorGpuTester
|
||||
{
|
||||
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, GpuUInt128, ArrayView<byte>>> _kernelCache = new();
|
||||
private readonly ConcurrentDictionary<Accelerator, Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>> _kernelCache = new();
|
||||
private readonly ConcurrentDictionary<Accelerator, MemoryBuffer1D<byte, Stride1D.Dense>> _resultBuffers = new();
|
||||
|
||||
private Action<Index1D, ulong, GpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
|
||||
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, GpuUInt128, ArrayView<byte>>(Kernel));
|
||||
private Action<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>> GetKernel(Accelerator accelerator) =>
|
||||
_kernelCache.GetOrAdd(accelerator, acc => acc.LoadAutoGroupedStreamKernel<Index1D, ulong, ReadOnlyGpuUInt128, ArrayView<byte>>(Kernel));
|
||||
|
||||
public static void BuildDivisorCandidates()
|
||||
{
|
||||
@@ -36,106 +36,102 @@ public sealed class MersenneNumberDivisorGpuTester
|
||||
_divisorCandidates = count == 0 ? [] : list[..count];
|
||||
}
|
||||
|
||||
public bool IsDivisible(ulong exponent, UInt128 divisor)
|
||||
{
|
||||
var gpu = GpuContextPool.RentPreferred(preferCpu: false);
|
||||
var accelerator = gpu.Accelerator;
|
||||
var kernel = GetKernel(accelerator);
|
||||
var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D<byte>(1));
|
||||
resultBuffer.MemSetToZero();
|
||||
kernel(1, exponent, (GpuUInt128)divisor, resultBuffer.View);
|
||||
accelerator.Synchronize();
|
||||
Span<byte> result = stackalloc byte[1];
|
||||
resultBuffer.View.CopyToCPU(ref result[0], 1);
|
||||
bool divisible = result[0] != 0;
|
||||
result[0] = 0;
|
||||
resultBuffer.View.CopyFromCPU(ref result[0], 1);
|
||||
accelerator.Synchronize();
|
||||
gpu.Dispose();
|
||||
return divisible;
|
||||
}
|
||||
public bool IsDivisible(ulong exponent, in ReadOnlyGpuUInt128 divisor)
|
||||
{
|
||||
var gpu = GpuContextPool.RentPreferred(preferCpu: false);
|
||||
var accelerator = gpu.Accelerator;
|
||||
var kernel = GetKernel(accelerator);
|
||||
var resultBuffer = _resultBuffers.GetOrAdd(accelerator, acc => acc.Allocate1D<byte>(1));
|
||||
resultBuffer.MemSetToZero();
|
||||
kernel(1, exponent, divisor, resultBuffer.View);
|
||||
accelerator.Synchronize();
|
||||
Span<byte> result = stackalloc byte[1];
|
||||
resultBuffer.View.CopyToCPU(ref result[0], 1);
|
||||
bool divisible = result[0] != 0;
|
||||
result[0] = 0;
|
||||
resultBuffer.View.CopyFromCPU(ref result[0], 1);
|
||||
accelerator.Synchronize();
|
||||
gpu.Dispose();
|
||||
return divisible;
|
||||
}
|
||||
|
||||
private static (ulong divisor, uint cycle)[]? _divisorCandidates = Array.Empty<(ulong divisor, uint cycle)>();
|
||||
|
||||
public bool IsPrime(ulong p, UInt128 d, ulong divisorCyclesSearchLimit, out bool divisorsExhausted)
|
||||
{
|
||||
if (d != UInt128.Zero)
|
||||
{
|
||||
if (IsDivisible(p, d))
|
||||
{
|
||||
divisorsExhausted = true;
|
||||
return false;
|
||||
}
|
||||
public bool IsPrime(ulong p, UInt128 d, ulong divisorCyclesSearchLimit, out bool divisorsExhausted)
|
||||
{
|
||||
ReadOnlyGpuUInt128 readOnlyDivisor;
|
||||
|
||||
divisorsExhausted = false;
|
||||
return true;
|
||||
}
|
||||
if (d != UInt128.Zero)
|
||||
{
|
||||
readOnlyDivisor = new ReadOnlyGpuUInt128(d);
|
||||
if (IsDivisible(p, in readOnlyDivisor))
|
||||
{
|
||||
divisorsExhausted = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (_divisorCandidates is { Length: > 0 } candidates)
|
||||
divisorsExhausted = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (_divisorCandidates is { Length: > 0 } candidates)
|
||||
{
|
||||
int candidateCount = candidates.Length;
|
||||
for (int index = 0; index < candidateCount; index++)
|
||||
{
|
||||
(ulong candidateDivisor, uint cycle) = candidates[index];
|
||||
if (p % cycle != 0UL)
|
||||
{
|
||||
int len = candidates.Length;
|
||||
uint cycle;
|
||||
for (int k = 0; k < len; k++)
|
||||
{
|
||||
(ulong dSmall, cycle) = candidates[k];
|
||||
// TODO: Replace this `%` check with the shared divisor-cycle divisibility helper so GPU scans
|
||||
// reuse the cached cycle remainders instead of recomputing modulo for every candidate.
|
||||
if (p % cycle != 0UL)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsDivisible(p, dSmall))
|
||||
{
|
||||
divisorsExhausted = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
readOnlyDivisor = new ReadOnlyGpuUInt128(candidateDivisor); // Reusing readOnlyDivisor for candidate divisors.
|
||||
if (IsDivisible(p, in readOnlyDivisor))
|
||||
{
|
||||
divisorsExhausted = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UInt128 kMul2;
|
||||
var divisorCycles = MersenneDivisorCycles.Shared;
|
||||
UInt128 maxK2 = UInt128.MaxValue / ((UInt128)p << 1);
|
||||
ulong limit = divisorCyclesSearchLimit;
|
||||
if ((UInt128)limit > maxK2)
|
||||
{
|
||||
limit = (ulong)maxK2;
|
||||
}
|
||||
UInt128 kMul2;
|
||||
var divisorCycles = MersenneDivisorCycles.Shared;
|
||||
UInt128 maxK2 = UInt128.MaxValue / ((UInt128)p << 1);
|
||||
ulong limit = divisorCyclesSearchLimit;
|
||||
if ((UInt128)limit > maxK2)
|
||||
{
|
||||
limit = (ulong)maxK2;
|
||||
}
|
||||
|
||||
ulong k2;
|
||||
for (k2 = 1UL; k2 <= limit; k2++)
|
||||
{
|
||||
kMul2 = (UInt128)k2 << 1;
|
||||
UInt128 candidate = checked(kMul2 * p);
|
||||
d = checked(candidate + UInt128.One);
|
||||
if (p < 64UL && d == ((UInt128)1 << (int)p) - UInt128.One)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for (ulong k2 = 1UL; k2 <= limit; k2++)
|
||||
{
|
||||
kMul2 = (UInt128)k2 << 1;
|
||||
UInt128 candidate = checked(kMul2 * p);
|
||||
d = checked(candidate + UInt128.One);
|
||||
if (p < 64UL && d == ((UInt128)1 << (int)p) - UInt128.One)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
UInt128 cycle128 = MersenneDivisorCycles.GetCycle(d);
|
||||
// TODO: When this lookup falls outside the preloaded snapshot, invoke the single-cycle GPU helper
|
||||
// instead of allowing GetCycle to allocate an entire block; the divisor-cycle benchmarks showed the
|
||||
// one-off computation keeping memory flat while still delivering the optimized stepping data.
|
||||
// TODO: Feed this check through the ProcessEightBitWindows-based residue tracker so we avoid the slow
|
||||
// UInt128 modulo on the CPU fallback highlighted by the Pow2Montgomery benchmarks.
|
||||
if ((UInt128)p % cycle128 != UInt128.Zero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
UInt128 cycle128 = MersenneDivisorCycles.GetCycle(d);
|
||||
if ((UInt128)p % cycle128 != UInt128.Zero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsDivisible(p, d))
|
||||
{
|
||||
divisorsExhausted = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
readOnlyDivisor = new ReadOnlyGpuUInt128(d); // Reusing readOnlyDivisor for generated divisors.
|
||||
if (IsDivisible(p, in readOnlyDivisor))
|
||||
{
|
||||
divisorsExhausted = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
divisorsExhausted = (UInt128)divisorCyclesSearchLimit >= maxK2;
|
||||
return true;
|
||||
}
|
||||
divisorsExhausted = (UInt128)divisorCyclesSearchLimit >= maxK2;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void Kernel(Index1D _, ulong exponent, GpuUInt128 divisor, ArrayView<byte> result)
|
||||
private static void Kernel(Index1D _, ulong exponent, ReadOnlyGpuUInt128 divisor, ArrayView<byte> result)
|
||||
{
|
||||
if (divisor.High == 0UL && divisor.Low <= 1UL)
|
||||
{
|
||||
@@ -143,16 +139,16 @@ public sealed class MersenneNumberDivisorGpuTester
|
||||
return;
|
||||
}
|
||||
|
||||
GpuUInt128 baseVal, mod;
|
||||
GpuUInt128 baseVal;
|
||||
int x = divisor.High == 0UL
|
||||
? 63 - XMath.LeadingZeroCount(divisor.Low)
|
||||
: 127 - XMath.LeadingZeroCount(divisor.High);
|
||||
mod = divisor;
|
||||
ReadOnlyGpuUInt128 readOnlyMod = divisor;
|
||||
if (x == 0)
|
||||
{
|
||||
// TODO: Swap Pow2Mod for the ProcessEightBitWindows helper once Pow2Minus1Mod adopts it;
|
||||
// GpuPow2ModBenchmarks showed the windowed kernel cutting large divisors from ~51 µs to ~21 µs.
|
||||
baseVal = GpuUInt128.Pow2Mod(exponent, mod);
|
||||
baseVal = GpuUInt128.Pow2Mod(exponent, in readOnlyMod);
|
||||
result[0] = baseVal.High == 0UL && baseVal.Low == 1UL ? (byte)1 : (byte)0;
|
||||
return;
|
||||
}
|
||||
@@ -168,12 +164,13 @@ public sealed class MersenneNumberDivisorGpuTester
|
||||
}
|
||||
|
||||
GpuUInt128 pow = baseVal;
|
||||
pow.ModPow(exponent / ux, mod);
|
||||
pow.ModPow(exponent / ux, in readOnlyMod);
|
||||
// TODO: Replace this trailing Pow2Mod call with the ProcessEightBitWindows variant once the shared
|
||||
// windowed helper lands so mixed-radix decompositions benefit from the same GPU speedup.
|
||||
var part2 = GpuUInt128.Pow2Mod(exponent % ux, mod);
|
||||
var part2 = GpuUInt128.Pow2Mod(exponent % ux, in readOnlyMod);
|
||||
GpuUInt128 product = pow;
|
||||
product.MulMod(part2, mod);
|
||||
ReadOnlyGpuUInt128 part2ReadOnly = part2.AsReadOnly();
|
||||
product.MulMod(in part2ReadOnly, in readOnlyMod);
|
||||
result[0] = product.High == 0UL && product.Low == 1UL ? (byte)1 : (byte)0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
using System;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace PerfectNumbers.Core.Gpu;
|
||||
|
||||
public readonly struct ReadOnlyGpuUInt128 : IEquatable<ReadOnlyGpuUInt128>
|
||||
{
|
||||
public readonly ulong High;
|
||||
public readonly ulong Low;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlyGpuUInt128(ulong high, ulong low)
|
||||
{
|
||||
High = high;
|
||||
Low = low;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlyGpuUInt128(ulong low)
|
||||
: this(0UL, low)
|
||||
{
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlyGpuUInt128(UInt128 value)
|
||||
{
|
||||
High = (ulong)(value >> 64);
|
||||
Low = (ulong)value;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlyGpuUInt128(GpuUInt128 value)
|
||||
{
|
||||
High = value.High;
|
||||
Low = value.Low;
|
||||
}
|
||||
|
||||
public bool IsZero
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => High == 0UL && Low == 0UL;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int GetBitLength()
|
||||
{
|
||||
if (High != 0UL)
|
||||
{
|
||||
return 64 + (64 - BitOperations.LeadingZeroCount(High));
|
||||
}
|
||||
|
||||
if (Low == 0UL)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 64 - BitOperations.LeadingZeroCount(Low);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public GpuUInt128 ToMutable() => new(High, Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool Equals(ReadOnlyGpuUInt128 other) => High == other.High && Low == other.Low;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool Equals(object? obj) => obj is ReadOnlyGpuUInt128 other && Equals(other);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override int GetHashCode() => HashCode.Combine(High, Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator ReadOnlyGpuUInt128(ulong value) => new(value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator ReadOnlyGpuUInt128(UInt128 value) => new(value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator ReadOnlyGpuUInt128(GpuUInt128 value) => new(value);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static implicit operator UInt128(ReadOnlyGpuUInt128 value) => ((UInt128)value.High << 64) | value.Low;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlyGpuUInt128 WithHigh(ulong high) => new(high, Low);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlyGpuUInt128 WithLow(ulong low) => new(High, low);
|
||||
}
|
||||
@@ -1,2 +1,5 @@
|
||||
- implement full routing for CPU or GPU execution based on the (...)-device=cpu|gpu switches. The current --mersenne=bydivisor GPU flow appears to run every stage on the GPU and ignore switches such as --primes-device for the primality check.
|
||||
- investigate why the --mersenne=bydivisor GPU path runs out of host memory. Exceptions must remain visible.
|
||||
- Reimplement the GPU cycle heuristics for MersenneNumberIncrementalGpuTester and re-enable Scan_handles_various_prime_exponents once the device path is fixed.
|
||||
- Extend the GPU numeric type probe to cover proposed custom types such as GpuUInt256 and GpuUInt512 once their definitions are available so we can plan the wide-order heuristics on the device.
|
||||
- document the GPU by-divisor regressions introduced in commit cd1debe ("Various micro- and medium- optimizations"): the refactor dropped divisor-cycle caching, remainder stepper reuse, and honoring the cached cycle parameter so we can plan which pieces to restore.
|
||||
Reference in new issue
Block a user