Formatting
This commit is contained in:
mlemanczyk committed
1 parent
4baca1ba70
commit
43cd067e09
1 file changed
+395
-395
@@ -9,399 +9,399 @@ namespace PerfectNumbers.Core;
|
|||||||
|
|
||||||
public static partial class ULongExtensions
|
public static partial class ULongExtensions
|
||||||
{
|
{
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong MulHighGpuCompatible(this ulong x, ulong y)
|
public static ulong MulHighGpuCompatible(this ulong x, ulong y)
|
||||||
{
|
{
|
||||||
GpuUInt128 product = new(x);
|
GpuUInt128 product = new(x);
|
||||||
GpuUInt128.Mul64(ref product, 0UL, y);
|
GpuUInt128.Mul64(ref product, 0UL, y);
|
||||||
return product.High;
|
return product.High;
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong MulMod64Gpu(this ulong a, ulong b, ulong modulus)
|
public static ulong MulMod64Gpu(this ulong a, ulong b, ulong modulus)
|
||||||
{
|
{
|
||||||
// TODO: Remove this GPU-compatible shim from production once callers migrate to MulMod64,
|
// TODO: Remove this GPU-compatible shim from production once callers migrate to MulMod64,
|
||||||
// which the benchmarks show is roughly 6-7× faster on dense 64-bit inputs.
|
// which the benchmarks show is roughly 6-7× faster on dense 64-bit inputs.
|
||||||
GpuUInt128 state = new(a % modulus);
|
GpuUInt128 state = new(a % modulus);
|
||||||
return state.MulMod(b, modulus);
|
return state.MulMod(b, modulus);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong MulMod64GpuDeferred(this ulong a, ulong b, ulong modulus)
|
public static ulong MulMod64GpuDeferred(this ulong a, ulong b, ulong modulus)
|
||||||
{
|
{
|
||||||
// TODO: Move this deferred helper to the benchmark suite; the baseline MulMod64 avoids the
|
// TODO: Move this deferred helper to the benchmark suite; the baseline MulMod64 avoids the
|
||||||
// 5-40× slowdown seen across real-world operand distributions.
|
// 5-40× slowdown seen across real-world operand distributions.
|
||||||
GpuUInt128 state = new(a);
|
GpuUInt128 state = new(a);
|
||||||
return state.MulModWithNativeModulo(b, modulus);
|
return state.MulModWithNativeModulo(b, modulus);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong Pow2ModWindowedGpu(this ulong exponent, ulong modulus)
|
public static ulong Pow2ModWindowedGpu(this ulong exponent, ulong modulus)
|
||||||
{
|
{
|
||||||
// EvenPerfectBitScanner never routes moduli ≤ 1 through this path; keep the guard commented out so the hot loop stays branch-free.
|
// EvenPerfectBitScanner never routes moduli ≤ 1 through this path; keep the guard commented out so the hot loop stays branch-free.
|
||||||
// if (modulus <= 1UL)
|
// if (modulus <= 1UL)
|
||||||
// {
|
// {
|
||||||
// return 0UL;
|
// return 0UL;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// EvenPerfectBitScanner always works with positive exponents here; leave the defensive check commented for future reference.
|
// EvenPerfectBitScanner always works with positive exponents here; leave the defensive check commented for future reference.
|
||||||
// if (exponent == 0UL)
|
// if (exponent == 0UL)
|
||||||
// {
|
// {
|
||||||
// return 1UL % modulus;
|
// return 1UL % modulus;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
int bitLength = GetPortableBitLengthGpu(exponent);
|
int bitLength = GetPortableBitLengthGpu(exponent);
|
||||||
int windowSize = GetWindowSizeGpu(bitLength);
|
int windowSize = GetWindowSizeGpu(bitLength);
|
||||||
int oddPowerCount = 1 << (windowSize - 1);
|
int oddPowerCount = 1 << (windowSize - 1);
|
||||||
|
|
||||||
// oddPowerCount is always ≥ 1 for the production workloads handled by EvenPerfectBitScanner.
|
// oddPowerCount is always ≥ 1 for the production workloads handled by EvenPerfectBitScanner.
|
||||||
// if (oddPowerCount <= 0)
|
// if (oddPowerCount <= 0)
|
||||||
// {
|
// {
|
||||||
// return 1UL;
|
// return 1UL;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
ulong[] oddPowersArray = ThreadStaticPools.UlongPool.Rent(oddPowerCount);
|
ulong[] oddPowersArray = ThreadStaticPools.UlongPool.Rent(oddPowerCount);
|
||||||
Span<ulong> oddPowers = oddPowersArray.AsSpan(0, oddPowerCount);
|
Span<ulong> oddPowers = oddPowersArray.AsSpan(0, oddPowerCount);
|
||||||
|
|
||||||
InitializeStandardOddPowers(modulus, oddPowers);
|
InitializeStandardOddPowers(modulus, oddPowers);
|
||||||
|
|
||||||
ulong result = 1UL;
|
ulong result = 1UL;
|
||||||
int index = bitLength - 1;
|
int index = bitLength - 1;
|
||||||
|
|
||||||
while (index >= 0)
|
while (index >= 0)
|
||||||
{
|
{
|
||||||
ulong currentBit = (exponent >> index) & 1UL;
|
ulong currentBit = (exponent >> index) & 1UL;
|
||||||
ulong squared = result.MulMod64Gpu(result, modulus);
|
ulong squared = result.MulMod64Gpu(result, modulus);
|
||||||
result = currentBit == 0UL ? squared : result;
|
result = currentBit == 0UL ? squared : result;
|
||||||
index = currentBit == 0UL ? index - 1 : index;
|
index = currentBit == 0UL ? index - 1 : index;
|
||||||
if (currentBit == 0UL)
|
if (currentBit == 0UL)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
int windowStart = index - windowSize + 1;
|
int windowStart = index - windowSize + 1;
|
||||||
// EvenPerfectBitScanner guarantees windowStart stays non-negative for production exponents, so keep the guard commented out to avoid extra branching.
|
// EvenPerfectBitScanner guarantees windowStart stays non-negative for production exponents, so keep the guard commented out to avoid extra branching.
|
||||||
// if (windowStart < 0)
|
// if (windowStart < 0)
|
||||||
// {
|
// {
|
||||||
// windowStart = 0;
|
// windowStart = 0;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
windowStart = GetNextSetBitIndexGpu(exponent, windowStart);
|
windowStart = GetNextSetBitIndexGpu(exponent, windowStart);
|
||||||
|
|
||||||
int windowLength = index - windowStart + 1;
|
int windowLength = index - windowStart + 1;
|
||||||
for (int square = 0; square < windowLength; square++)
|
for (int square = 0; square < windowLength; square++)
|
||||||
{
|
{
|
||||||
result = result.MulMod64Gpu(result, modulus);
|
result = result.MulMod64Gpu(result, modulus);
|
||||||
}
|
}
|
||||||
|
|
||||||
ulong mask = (1UL << windowLength) - 1UL;
|
ulong mask = (1UL << windowLength) - 1UL;
|
||||||
ulong windowValue = (exponent >> windowStart) & mask;
|
ulong windowValue = (exponent >> windowStart) & mask;
|
||||||
int tableIndex = (int)((windowValue - 1UL) >> 1);
|
int tableIndex = (int)((windowValue - 1UL) >> 1);
|
||||||
ulong multiplier = oddPowers[tableIndex];
|
ulong multiplier = oddPowers[tableIndex];
|
||||||
result = result.MulMod64Gpu(multiplier, modulus);
|
result = result.MulMod64Gpu(multiplier, modulus);
|
||||||
|
|
||||||
index = windowStart - 1;
|
index = windowStart - 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
ThreadStaticPools.UlongPool.Return(oddPowersArray);
|
ThreadStaticPools.UlongPool.Return(oddPowersArray);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
internal static ulong Pow2MontgomeryModWindowedGpuKeepMontgomery(in MontgomeryDivisorData divisor, ulong exponent)
|
internal static ulong Pow2MontgomeryModWindowedGpuKeepMontgomery(in MontgomeryDivisorData divisor, ulong exponent)
|
||||||
{
|
{
|
||||||
return Pow2MontgomeryModWindowedGpuMontgomeryResult(divisor, exponent);
|
return Pow2MontgomeryModWindowedGpuMontgomeryResult(divisor, exponent);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
internal static ulong Pow2MontgomeryModWindowedGpuConvertToStandard(in MontgomeryDivisorData divisor, ulong exponent)
|
internal static ulong Pow2MontgomeryModWindowedGpuConvertToStandard(in MontgomeryDivisorData divisor, ulong exponent)
|
||||||
{
|
{
|
||||||
ulong modulus = divisor.Modulus;
|
ulong modulus = divisor.Modulus;
|
||||||
ulong nPrime = divisor.NPrime;
|
ulong nPrime = divisor.NPrime;
|
||||||
ulong montgomeryResult = Pow2MontgomeryModWindowedGpuMontgomeryResult(divisor, exponent);
|
ulong montgomeryResult = Pow2MontgomeryModWindowedGpuMontgomeryResult(divisor, exponent);
|
||||||
return MontgomeryMultiply(montgomeryResult, 1UL, modulus, nPrime);
|
return MontgomeryMultiply(montgomeryResult, 1UL, modulus, nPrime);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static ulong Pow2MontgomeryModWindowedGpuMontgomeryResult(in MontgomeryDivisorData divisor, ulong exponent)
|
private static ulong Pow2MontgomeryModWindowedGpuMontgomeryResult(in MontgomeryDivisorData divisor, ulong exponent)
|
||||||
{
|
{
|
||||||
ulong modulus = divisor.Modulus;
|
ulong modulus = divisor.Modulus;
|
||||||
// Reuse the optimized MontgomeryMultiply extension highlighted in MontgomeryMultiplyBenchmarks so the GPU path matches the fastest CPU reduction.
|
// Reuse the optimized MontgomeryMultiply extension highlighted in MontgomeryMultiplyBenchmarks so the GPU path matches the fastest CPU reduction.
|
||||||
// We shouldn't hit it in production code. If we do, we're doing something wrong.
|
// We shouldn't hit it in production code. If we do, we're doing something wrong.
|
||||||
// if (exponent == 0UL)
|
// if (exponent == 0UL)
|
||||||
// {
|
// {
|
||||||
// return divisor.MontgomeryOne;
|
// return divisor.MontgomeryOne;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// We barely ever hit it in production code. It's e.g. 772 calls out of billions
|
// We barely ever hit it in production code. It's e.g. 772 calls out of billions
|
||||||
// if (exponent <= Pow2WindowFallbackThreshold)
|
// if (exponent <= Pow2WindowFallbackThreshold)
|
||||||
// {
|
// {
|
||||||
// return Pow2MontgomeryModSingleBit(exponent, divisor);
|
// return Pow2MontgomeryModSingleBit(exponent, divisor);
|
||||||
// }
|
// }
|
||||||
|
|
||||||
int bitLength = GetPortableBitLengthGpu(exponent);
|
int bitLength = GetPortableBitLengthGpu(exponent);
|
||||||
int windowSize = GetWindowSizeGpu(bitLength);
|
int windowSize = GetWindowSizeGpu(bitLength);
|
||||||
ulong result = divisor.MontgomeryOne;
|
ulong result = divisor.MontgomeryOne;
|
||||||
ulong nPrime = divisor.NPrime;
|
ulong nPrime = divisor.NPrime;
|
||||||
|
|
||||||
int index = bitLength - 1;
|
int index = bitLength - 1;
|
||||||
while (index >= 0)
|
while (index >= 0)
|
||||||
{
|
{
|
||||||
ulong currentBit = (exponent >> index) & 1UL;
|
ulong currentBit = (exponent >> index) & 1UL;
|
||||||
ulong squared = MontgomeryMultiply(result, result, modulus, nPrime);
|
ulong squared = MontgomeryMultiply(result, result, modulus, nPrime);
|
||||||
bool processWindow = currentBit != 0UL;
|
bool processWindow = currentBit != 0UL;
|
||||||
|
|
||||||
result = processWindow ? result : squared;
|
result = processWindow ? result : squared;
|
||||||
index -= (int)(currentBit ^ 1UL);
|
index -= (int)(currentBit ^ 1UL);
|
||||||
|
|
||||||
int windowStartCandidate = index - windowSize + 1;
|
int windowStartCandidate = index - windowSize + 1;
|
||||||
// if (windowStartCandidate < 0)
|
// if (windowStartCandidate < 0)
|
||||||
// {
|
// {
|
||||||
// windowStartCandidate = 0;
|
// windowStartCandidate = 0;
|
||||||
// }
|
// }
|
||||||
// Clamp the negative offset without branching so the GPU loop stays divergence-free.
|
// Clamp the negative offset without branching so the GPU loop stays divergence-free.
|
||||||
// This still handles windows that would otherwise extend past the most significant bit.
|
// This still handles windows that would otherwise extend past the most significant bit.
|
||||||
int negativeMask = windowStartCandidate >> 31;
|
int negativeMask = windowStartCandidate >> 31;
|
||||||
windowStartCandidate &= ~negativeMask;
|
windowStartCandidate &= ~negativeMask;
|
||||||
|
|
||||||
int windowStart = processWindow ? GetNextSetBitIndexGpu(exponent, windowStartCandidate) : windowStartCandidate;
|
int windowStart = processWindow ? GetNextSetBitIndexGpu(exponent, windowStartCandidate) : windowStartCandidate;
|
||||||
int windowLength = processWindow ? index - windowStart + 1 : 0;
|
int windowLength = processWindow ? index - windowStart + 1 : 0;
|
||||||
for (int square = 0; square < windowLength; square++)
|
for (int square = 0; square < windowLength; square++)
|
||||||
{
|
{
|
||||||
result = MontgomeryMultiply(result, result, modulus, nPrime);
|
result = MontgomeryMultiply(result, result, modulus, nPrime);
|
||||||
}
|
}
|
||||||
|
|
||||||
result = processWindow
|
result = processWindow
|
||||||
? MontgomeryMultiply(
|
? MontgomeryMultiply(
|
||||||
result,
|
result,
|
||||||
ComputeMontgomeryOddPowerGpu(
|
ComputeMontgomeryOddPowerGpu(
|
||||||
(exponent >> windowStart) & ((1UL << windowLength) - 1UL),
|
(exponent >> windowStart) & ((1UL << windowLength) - 1UL),
|
||||||
divisor,
|
divisor,
|
||||||
modulus,
|
modulus,
|
||||||
nPrime),
|
nPrime),
|
||||||
modulus,
|
modulus,
|
||||||
nPrime)
|
nPrime)
|
||||||
: result;
|
: result;
|
||||||
|
|
||||||
index = processWindow ? windowStart - 1 : index;
|
index = processWindow ? windowStart - 1 : index;
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong Pow2MontgomeryModWindowedGpu(this ulong exponent, in MontgomeryDivisorData divisor, bool keepMontgomery)
|
public static ulong Pow2MontgomeryModWindowedGpu(this ulong exponent, in MontgomeryDivisorData divisor, bool keepMontgomery)
|
||||||
{
|
{
|
||||||
// We shouldn't hit it in production code. If we do, we're doing something wrong.
|
// We shouldn't hit it in production code. If we do, we're doing something wrong.
|
||||||
// ulong modulus = divisor.Modulus;
|
// ulong modulus = divisor.Modulus;
|
||||||
// if (exponent == 0UL)
|
// if (exponent == 0UL)
|
||||||
// {
|
// {
|
||||||
// return divisor.MontgomeryOne;
|
// return divisor.MontgomeryOne;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// We barely ever hit it in production code. It's e.g. 772 calls out of billions
|
// We barely ever hit it in production code. It's e.g. 772 calls out of billions
|
||||||
// if (exponent <= Pow2WindowFallbackThreshold)
|
// if (exponent <= Pow2WindowFallbackThreshold)
|
||||||
// {
|
// {
|
||||||
// return Pow2MontgomeryModSingleBit(exponent, divisor);
|
// return Pow2MontgomeryModSingleBit(exponent, divisor);
|
||||||
// }
|
// }
|
||||||
|
|
||||||
return keepMontgomery
|
return keepMontgomery
|
||||||
? Pow2MontgomeryGpuExecutor.ExecuteKeep(exponent, divisor)
|
? Pow2MontgomeryGpuExecutor.ExecuteKeep(exponent, divisor)
|
||||||
: Pow2MontgomeryGpuExecutor.ExecuteConvert(exponent, divisor);
|
: Pow2MontgomeryGpuExecutor.ExecuteConvert(exponent, divisor);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static void InitializeStandardOddPowers(ulong modulus, Span<ulong> oddPowers)
|
private static void InitializeStandardOddPowers(ulong modulus, Span<ulong> oddPowers)
|
||||||
{
|
{
|
||||||
// oddPowers spans at least one entry for production workloads; keep the guard commented out to avoid redundant checks in the hot loop.
|
// oddPowers spans at least one entry for production workloads; keep the guard commented out to avoid redundant checks in the hot loop.
|
||||||
// if (oddPowers.IsEmpty)
|
// if (oddPowers.IsEmpty)
|
||||||
// {
|
// {
|
||||||
// return;
|
// return;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// EvenPerfectBitScanner feeds odd prime moduli (≥ 3) here, so the base value stays within range without a modulo reduction.
|
// EvenPerfectBitScanner feeds odd prime moduli (≥ 3) here, so the base value stays within range without a modulo reduction.
|
||||||
ulong baseValue = 2UL;
|
ulong baseValue = 2UL;
|
||||||
oddPowers[0] = baseValue;
|
oddPowers[0] = baseValue;
|
||||||
if (oddPowers.Length == 1)
|
if (oddPowers.Length == 1)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
ulong square = baseValue.MulMod64Gpu(baseValue, modulus);
|
ulong square = baseValue.MulMod64Gpu(baseValue, modulus);
|
||||||
for (int i = 1; i < oddPowers.Length; i++)
|
for (int i = 1; i < oddPowers.Length; i++)
|
||||||
{
|
{
|
||||||
ulong previous = oddPowers[i - 1];
|
ulong previous = oddPowers[i - 1];
|
||||||
oddPowers[i] = previous.MulMod64Gpu(square, modulus);
|
oddPowers[i] = previous.MulMod64Gpu(square, modulus);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static ulong ComputeMontgomeryOddPowerGpu(ulong exponent, in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime)
|
private static ulong ComputeMontgomeryOddPowerGpu(ulong exponent, in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime)
|
||||||
{
|
{
|
||||||
ulong baseValue = divisor.MontgomeryTwo;
|
ulong baseValue = divisor.MontgomeryTwo;
|
||||||
ulong power = divisor.MontgomeryOne;
|
ulong power = divisor.MontgomeryOne;
|
||||||
ulong remaining = exponent;
|
ulong remaining = exponent;
|
||||||
|
|
||||||
while (remaining != 0UL)
|
while (remaining != 0UL)
|
||||||
{
|
{
|
||||||
ulong bit = remaining & 1UL;
|
ulong bit = remaining & 1UL;
|
||||||
ulong multiplied = MontgomeryMultiply(power, baseValue, modulus, nPrime);
|
ulong multiplied = MontgomeryMultiply(power, baseValue, modulus, nPrime);
|
||||||
ulong mask = (ulong)-(long)bit;
|
ulong mask = (ulong)-(long)bit;
|
||||||
power = (power & ~mask) | (multiplied & mask);
|
power = (power & ~mask) | (multiplied & mask);
|
||||||
|
|
||||||
remaining >>= 1;
|
remaining >>= 1;
|
||||||
if (remaining == 0UL)
|
if (remaining == 0UL)
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
baseValue = MontgomeryMultiply(baseValue, baseValue, modulus, nPrime);
|
baseValue = MontgomeryMultiply(baseValue, baseValue, modulus, nPrime);
|
||||||
}
|
}
|
||||||
|
|
||||||
return power;
|
return power;
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static int GetPortableBitLengthGpu(ulong value)
|
private static int GetPortableBitLengthGpu(ulong value)
|
||||||
{
|
{
|
||||||
// Keep this commented out. It will never happen in production code.
|
// Keep this commented out. It will never happen in production code.
|
||||||
// if (value == 0UL)
|
// if (value == 0UL)
|
||||||
// {
|
// {
|
||||||
// return 0;
|
// return 0;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
return 64 - XMath.LeadingZeroCount(value);
|
return 64 - XMath.LeadingZeroCount(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static int GetWindowSizeGpu(int n)
|
private static int GetWindowSizeGpu(int n)
|
||||||
{
|
{
|
||||||
int w = Pow2WindowSize;
|
int w = Pow2WindowSize;
|
||||||
|
|
||||||
w = (n <= 671) ? 7 : w;
|
w = (n <= 671) ? 7 : w;
|
||||||
w = (n <= 239) ? 6 : w;
|
w = (n <= 239) ? 6 : w;
|
||||||
w = (n <= 79) ? 5 : w;
|
w = (n <= 79) ? 5 : w;
|
||||||
w = (n <= 23) ? 4 : w;
|
w = (n <= 23) ? 4 : w;
|
||||||
|
|
||||||
int m = (n >= 1) ? n : 1;
|
int m = (n >= 1) ? n : 1;
|
||||||
w = (n <= 6) ? m : w;
|
w = (n <= 6) ? m : w;
|
||||||
|
|
||||||
return w;
|
return w;
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static int GetNextSetBitIndexGpu(ulong exponent, int startIndex)
|
private static int GetNextSetBitIndexGpu(ulong exponent, int startIndex)
|
||||||
{
|
{
|
||||||
ulong guard = (ulong)(((long)startIndex - 64) >> 63);
|
ulong guard = (ulong)(((long)startIndex - 64) >> 63);
|
||||||
int shift = startIndex & 63;
|
int shift = startIndex & 63;
|
||||||
ulong mask = (~0UL << shift) & guard;
|
ulong mask = (~0UL << shift) & guard;
|
||||||
ulong masked = exponent & mask;
|
ulong masked = exponent & mask;
|
||||||
|
|
||||||
return XMath.TrailingZeroCount(masked);
|
return XMath.TrailingZeroCount(masked);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong Pow2MontgomeryModWithCycleGpu(this ulong exponent, ulong cycleLength, in MontgomeryDivisorData divisor)
|
public static ulong Pow2MontgomeryModWithCycleGpu(this ulong exponent, ulong cycleLength, in MontgomeryDivisorData divisor)
|
||||||
{
|
{
|
||||||
ulong rotationCount = exponent % cycleLength;
|
ulong rotationCount = exponent % cycleLength;
|
||||||
return Pow2MontgomeryGpuExecutor.ExecuteConvert(rotationCount, divisor);
|
return Pow2MontgomeryGpuExecutor.ExecuteConvert(rotationCount, divisor);
|
||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
public static ulong Pow2MontgomeryModFromCycleRemainderGpu(this ulong reducedExponent, in MontgomeryDivisorData divisor)
|
public static ulong Pow2MontgomeryModFromCycleRemainderGpu(this ulong reducedExponent, in MontgomeryDivisorData divisor)
|
||||||
{
|
{
|
||||||
return Pow2MontgomeryGpuExecutor.ExecuteConvert(reducedExponent, divisor);
|
return Pow2MontgomeryGpuExecutor.ExecuteConvert(reducedExponent, divisor);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class Pow2MontgomeryGpuExecutor
|
private static class Pow2MontgomeryGpuExecutor
|
||||||
{
|
{
|
||||||
private static readonly ConcurrentDictionary<Accelerator, Pow2MontgomeryKernelGroup> KernelCache = new();
|
private static readonly ConcurrentDictionary<Accelerator, Pow2MontgomeryKernelGroup> KernelCache = new();
|
||||||
|
|
||||||
public static ulong ExecuteKeep(ulong exponent, in MontgomeryDivisorData divisor)
|
public static ulong ExecuteKeep(ulong exponent, in MontgomeryDivisorData divisor)
|
||||||
{
|
{
|
||||||
return Execute(exponent, divisor, useKeepKernel: true);
|
return Execute(exponent, divisor, useKeepKernel: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ulong ExecuteConvert(ulong exponent, in MontgomeryDivisorData divisor)
|
public static ulong ExecuteConvert(ulong exponent, in MontgomeryDivisorData divisor)
|
||||||
{
|
{
|
||||||
return Execute(exponent, divisor, useKeepKernel: false);
|
return Execute(exponent, divisor, useKeepKernel: false);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static ulong Execute(ulong exponent, in MontgomeryDivisorData divisor, bool useKeepKernel)
|
private static ulong Execute(ulong exponent, in MontgomeryDivisorData divisor, bool useKeepKernel)
|
||||||
{
|
{
|
||||||
ulong result = 0UL;
|
ulong result = 0UL;
|
||||||
|
|
||||||
GpuKernelLease lease = GpuKernelPool.GetKernel(useGpuOrder: true);
|
GpuKernelLease lease = GpuKernelPool.GetKernel(useGpuOrder: true);
|
||||||
var execution = lease.EnterExecutionScope();
|
var execution = lease.EnterExecutionScope();
|
||||||
|
|
||||||
Accelerator accelerator = lease.Accelerator;
|
Accelerator accelerator = lease.Accelerator;
|
||||||
// Keep this commented out. It should never happen in production code.
|
// Keep this commented out. It should never happen in production code.
|
||||||
// if (accelerator.AcceleratorType == AcceleratorType.CPU)
|
// if (accelerator.AcceleratorType == AcceleratorType.CPU)
|
||||||
// {
|
// {
|
||||||
// return 0UL;
|
// return 0UL;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
var kernelGroup = KernelCache.GetOrAdd(accelerator, static accel => Pow2MontgomeryKernelGroup.Create(accel));
|
var kernelGroup = KernelCache.GetOrAdd(accelerator, static accel => Pow2MontgomeryKernelGroup.Create(accel));
|
||||||
|
|
||||||
var exponentBuffer = accelerator.Allocate1D<ulong>(1);
|
var exponentBuffer = accelerator.Allocate1D<ulong>(1);
|
||||||
var resultBuffer = accelerator.Allocate1D<ulong>(1);
|
var resultBuffer = accelerator.Allocate1D<ulong>(1);
|
||||||
|
|
||||||
exponentBuffer.View.CopyFromCPU(ref exponent, 1);
|
exponentBuffer.View.CopyFromCPU(ref exponent, 1);
|
||||||
// We don't need to worry about any left-overs here.
|
// We don't need to worry about any left-overs here.
|
||||||
// resultBuffer.MemSetToZero();
|
// resultBuffer.MemSetToZero();
|
||||||
|
|
||||||
var kernel = useKeepKernel ? kernelGroup.KeepMontgomery : kernelGroup.ConvertToStandard;
|
var kernel = useKeepKernel ? kernelGroup.KeepMontgomery : kernelGroup.ConvertToStandard;
|
||||||
AcceleratorStream stream = lease.Stream;
|
AcceleratorStream stream = lease.Stream;
|
||||||
kernel(stream, 1, exponentBuffer.View, divisor, resultBuffer.View);
|
kernel(stream, 1, exponentBuffer.View, divisor, resultBuffer.View);
|
||||||
stream.Synchronize();
|
stream.Synchronize();
|
||||||
|
|
||||||
resultBuffer.View.CopyToCPU(ref result, 1);
|
resultBuffer.View.CopyToCPU(ref result, 1);
|
||||||
|
|
||||||
resultBuffer.Dispose();
|
resultBuffer.Dispose();
|
||||||
exponentBuffer.Dispose();
|
exponentBuffer.Dispose();
|
||||||
execution.Dispose();
|
execution.Dispose();
|
||||||
lease.Dispose();
|
lease.Dispose();
|
||||||
// Keep this commented. We don't want to catch any exceptions. All should crash the scanner.
|
// Keep this commented. We don't want to catch any exceptions. All should crash the scanner.
|
||||||
// catch (AcceleratorException)
|
// catch (AcceleratorException)
|
||||||
// {
|
// {
|
||||||
// return 0UL;
|
// return 0UL;
|
||||||
// }
|
// }
|
||||||
// catch (NotSupportedException)
|
// catch (NotSupportedException)
|
||||||
// {
|
// {
|
||||||
// return 0UL;
|
// return 0UL;
|
||||||
// }
|
// }
|
||||||
// Intentionally avoid exception handling here; any accelerator failure should crash the scanner.
|
// Intentionally avoid exception handling here; any accelerator failure should crash the scanner.
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class Pow2MontgomeryKernelGroup
|
private sealed class Pow2MontgomeryKernelGroup
|
||||||
{
|
{
|
||||||
internal Pow2MontgomeryKernelGroup(
|
internal Pow2MontgomeryKernelGroup(
|
||||||
Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> keepMontgomery,
|
Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> keepMontgomery,
|
||||||
Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> convertToStandard)
|
Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> convertToStandard)
|
||||||
{
|
{
|
||||||
KeepMontgomery = keepMontgomery;
|
KeepMontgomery = keepMontgomery;
|
||||||
ConvertToStandard = convertToStandard;
|
ConvertToStandard = convertToStandard;
|
||||||
}
|
}
|
||||||
|
|
||||||
internal Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> KeepMontgomery { get; }
|
internal Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> KeepMontgomery { get; }
|
||||||
|
|
||||||
internal Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> ConvertToStandard { get; }
|
internal Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>> ConvertToStandard { get; }
|
||||||
|
|
||||||
internal static Pow2MontgomeryKernelGroup Create(Accelerator accelerator)
|
internal static Pow2MontgomeryKernelGroup Create(Accelerator accelerator)
|
||||||
{
|
{
|
||||||
var keepKernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>(Pow2MontgomeryKernels.Pow2MontgomeryKernelKeepMontgomery);
|
var keepKernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>(Pow2MontgomeryKernels.Pow2MontgomeryKernelKeepMontgomery);
|
||||||
var keepLauncher = KernelUtil.GetKernel(keepKernel);
|
var keepLauncher = KernelUtil.GetKernel(keepKernel);
|
||||||
var keepDelegate = keepLauncher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>>();
|
var keepDelegate = keepLauncher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||||
|
|
||||||
var convertKernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>(Pow2MontgomeryKernels.Pow2MontgomeryKernelConvertToStandard);
|
var convertKernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>(Pow2MontgomeryKernels.Pow2MontgomeryKernelConvertToStandard);
|
||||||
var convertLauncher = KernelUtil.GetKernel(convertKernel);
|
var convertLauncher = KernelUtil.GetKernel(convertKernel);
|
||||||
var convertDelegate = convertLauncher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>>();
|
var convertDelegate = convertLauncher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||||
|
|
||||||
return new Pow2MontgomeryKernelGroup(keepDelegate, convertDelegate);
|
return new Pow2MontgomeryKernelGroup(keepDelegate, convertDelegate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user