Various optimizations
This commit is contained in:
1 parent
91dd373f43
commit
d30a915890
1 file changed
+45
-71
@@ -241,20 +241,20 @@ public static class ULongExtensions
|
|||||||
// faster, while GPU code can keep using GpuUInt128.MulHigh.
|
// faster, while GPU code can keep using GpuUInt128.MulHigh.
|
||||||
ulong xLow = (uint)x;
|
ulong xLow = (uint)x;
|
||||||
ulong xHigh = x >> 32;
|
ulong xHigh = x >> 32;
|
||||||
ulong yLow = (uint)y;
|
ulong result = (uint)y;
|
||||||
ulong yHigh = y >> 32;
|
ulong yHigh = y >> 32;
|
||||||
|
|
||||||
ulong w1 = xLow * yHigh;
|
ulong w1 = xLow * yHigh;
|
||||||
ulong w2 = xHigh * yLow;
|
ulong w2 = xHigh * result;
|
||||||
ulong w3 = xLow * yLow;
|
xLow *= result;
|
||||||
|
|
||||||
// Matching the layout used in GpuUInt128.MulHigh: introducing the
|
// Matching the layout used in GpuUInt128.MulHigh: introducing the
|
||||||
// intermediate result looks like one extra store, but it lets RyuJIT keep
|
// intermediate result looks like one extra store, but it lets RyuJIT keep
|
||||||
// the accumulated high word entirely in registers. Without this explicit
|
// the accumulated high word entirely in registers. Without this explicit
|
||||||
// local the JIT spills the partial sum, which is where the performance
|
// local the JIT spills the partial sum, which is where the performance
|
||||||
// regression in the benchmarks came from.
|
// regression in the benchmarks came from.
|
||||||
ulong result = (xHigh * yHigh) + (w1 >> 32) + (w2 >> 32);
|
result = (xHigh * yHigh) + (w1 >> 32) + (w2 >> 32);
|
||||||
result += ((w3 >> 32) + (uint)w1 + (uint)w2) >> 32;
|
result += ((xLow >> 32) + (uint)w1 + (uint)w2) >> 32;
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -316,9 +316,9 @@ public static class ULongExtensions
|
|||||||
{
|
{
|
||||||
ulong tLow = unchecked(a * b);
|
ulong tLow = unchecked(a * b);
|
||||||
ulong m = unchecked(tLow * nPrime);
|
ulong m = unchecked(tLow * nPrime);
|
||||||
ulong mTimesModulusLow = unchecked(m * modulus);
|
ulong result = unchecked(m * modulus);
|
||||||
|
|
||||||
ulong result = unchecked(a.MulHigh(b) + m.MulHigh(modulus) + (unchecked(tLow + mTimesModulusLow) < tLow ? 1UL : 0UL));
|
result = unchecked(a.MulHigh(b) + m.MulHigh(modulus) + (unchecked(tLow + result) < tLow ? 1UL : 0UL));
|
||||||
if (result >= modulus)
|
if (result >= modulus)
|
||||||
{
|
{
|
||||||
result -= modulus;
|
result -= modulus;
|
||||||
@@ -424,10 +424,11 @@ public static class ULongExtensions
|
|||||||
internal static ulong Pow2MontgomeryModWindowedKernel(ulong exponent, in MontgomeryDivisorData divisor, bool keepMontgomery)
|
internal static ulong Pow2MontgomeryModWindowedKernel(ulong exponent, in MontgomeryDivisorData divisor, bool keepMontgomery)
|
||||||
{
|
{
|
||||||
ulong modulus = divisor.Modulus;
|
ulong modulus = divisor.Modulus;
|
||||||
if (exponent == 0UL)
|
// This will never happen in production code
|
||||||
{
|
// if (exponent == 0UL)
|
||||||
return keepMontgomery ? divisor.MontgomeryOne : 1UL % modulus;
|
// {
|
||||||
}
|
// return keepMontgomery ? divisor.MontgomeryOne : 1UL % modulus;
|
||||||
|
// }
|
||||||
|
|
||||||
if (exponent <= Pow2WindowFallbackThreshold)
|
if (exponent <= Pow2WindowFallbackThreshold)
|
||||||
{
|
{
|
||||||
@@ -450,26 +451,27 @@ public static class ULongExtensions
|
|||||||
}
|
}
|
||||||
|
|
||||||
int windowStart = index - windowSize + 1;
|
int windowStart = index - windowSize + 1;
|
||||||
if (windowStart < 0)
|
// This can never happen in production code
|
||||||
{
|
// if (windowStart < 0)
|
||||||
windowStart = 0;
|
// {
|
||||||
}
|
// windowStart = 0;
|
||||||
|
// }
|
||||||
|
|
||||||
while (((exponent >> windowStart) & 1UL) == 0UL)
|
while (((exponent >> windowStart) & 1UL) == 0UL)
|
||||||
{
|
{
|
||||||
windowStart++;
|
windowStart++;
|
||||||
}
|
}
|
||||||
|
|
||||||
int windowLength = index - windowStart + 1;
|
bitLength = index - windowStart + 1;
|
||||||
for (int square = 0; square < windowLength; square++)
|
for (int square = 0; square < bitLength; square++)
|
||||||
{
|
{
|
||||||
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
result = result.MontgomeryMultiply(result, modulus, nPrime);
|
||||||
}
|
}
|
||||||
|
|
||||||
ulong mask = (1UL << windowLength) - 1UL;
|
ulong windowValue = (1UL << bitLength) - 1UL;
|
||||||
ulong windowValue = (exponent >> windowStart) & mask;
|
windowValue = (exponent >> windowStart) & windowValue;
|
||||||
ulong multiplier = ComputeMontgomeryOddPower(windowValue, divisor, modulus, nPrime);
|
windowValue = ComputeMontgomeryOddPower(windowValue, divisor, modulus, nPrime);
|
||||||
result = result.MontgomeryMultiply(multiplier, modulus, nPrime);
|
result = result.MontgomeryMultiply(windowValue, modulus, nPrime);
|
||||||
|
|
||||||
index = windowStart - 1;
|
index = windowStart - 1;
|
||||||
}
|
}
|
||||||
@@ -527,8 +529,6 @@ public static class ULongExtensions
|
|||||||
}
|
}
|
||||||
|
|
||||||
var execution = lease.EnterExecutionScope();
|
var execution = lease.EnterExecutionScope();
|
||||||
try
|
|
||||||
{
|
|
||||||
Accelerator accelerator = lease.Accelerator;
|
Accelerator accelerator = lease.Accelerator;
|
||||||
if (accelerator.AcceleratorType == AcceleratorType.CPU)
|
if (accelerator.AcceleratorType == AcceleratorType.CPU)
|
||||||
{
|
{
|
||||||
@@ -542,11 +542,12 @@ public static class ULongExtensions
|
|||||||
return launcher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, byte, ArrayView1D<ulong, Stride1D.Dense>>>();
|
return launcher.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ArrayView1D<ulong, Stride1D.Dense>, MontgomeryDivisorData, byte, ArrayView1D<ulong, Stride1D.Dense>>>();
|
||||||
});
|
});
|
||||||
|
|
||||||
using var exponentBuffer = accelerator.Allocate1D<ulong>(1);
|
var exponentBuffer = accelerator.Allocate1D<ulong>(1);
|
||||||
using var resultBuffer = accelerator.Allocate1D<ulong>(1);
|
var resultBuffer = accelerator.Allocate1D<ulong>(1);
|
||||||
|
|
||||||
exponentBuffer.View.CopyFromCPU(ref exponent, 1);
|
exponentBuffer.View.CopyFromCPU(ref exponent, 1);
|
||||||
resultBuffer.MemSetToZero();
|
// We could clear the buffer, but it'll be overwritten anyway
|
||||||
|
// resultBuffer.MemSetToZero();
|
||||||
|
|
||||||
byte keepFlag = keepMontgomery ? (byte)1 : (byte)0;
|
byte keepFlag = keepMontgomery ? (byte)1 : (byte)0;
|
||||||
AcceleratorStream stream = lease.Stream;
|
AcceleratorStream stream = lease.Stream;
|
||||||
@@ -554,21 +555,12 @@ public static class ULongExtensions
|
|||||||
stream.Synchronize();
|
stream.Synchronize();
|
||||||
|
|
||||||
resultBuffer.View.CopyToCPU(ref result, 1);
|
resultBuffer.View.CopyToCPU(ref result, 1);
|
||||||
return true;
|
resultBuffer.Dispose();
|
||||||
}
|
exponentBuffer.Dispose();
|
||||||
catch (AcceleratorException)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (NotSupportedException)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
execution.Dispose();
|
execution.Dispose();
|
||||||
lease.Dispose();
|
lease.Dispose();
|
||||||
}
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void Pow2MontgomeryKernel(Index1D index, ArrayView1D<ulong, Stride1D.Dense> exponents, MontgomeryDivisorData divisor, byte keepMontgomery, ArrayView1D<ulong, Stride1D.Dense> results)
|
private static void Pow2MontgomeryKernel(Index1D index, ArrayView1D<ulong, Stride1D.Dense> exponents, MontgomeryDivisorData divisor, byte keepMontgomery, ArrayView1D<ulong, Stride1D.Dense> results)
|
||||||
@@ -608,43 +600,25 @@ public static class ULongExtensions
|
|||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static int GetPortableBitLength(ulong value)
|
private static int GetPortableBitLength(ulong value)
|
||||||
{
|
{
|
||||||
if (value == 0UL)
|
// This will never happen in production code
|
||||||
{
|
// if (value == 0UL)
|
||||||
return 0;
|
// {
|
||||||
}
|
// return 0;
|
||||||
|
// }
|
||||||
|
|
||||||
return 64 - XMath.LeadingZeroCount(value);
|
return 64 - XMath.LeadingZeroCount(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int GetWindowSize(int bitLength)
|
private static int GetWindowSize(int bitLength) => bitLength switch
|
||||||
{
|
{
|
||||||
if (bitLength <= 6)
|
<= 6 => bitLength,// bitLength is always >= 1 in production code
|
||||||
{
|
// return Math.Max(bitLength, 1);
|
||||||
return Math.Max(bitLength, 1);
|
<= 23 => 4,
|
||||||
}
|
<= 79 => 5,
|
||||||
|
<= 239 => 6,
|
||||||
if (bitLength <= 23)
|
<= 671 => 7,
|
||||||
{
|
_ => Pow2WindowSize,
|
||||||
return 4;
|
};
|
||||||
}
|
|
||||||
|
|
||||||
if (bitLength <= 79)
|
|
||||||
{
|
|
||||||
return 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (bitLength <= 239)
|
|
||||||
{
|
|
||||||
return 6;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (bitLength <= 671)
|
|
||||||
{
|
|
||||||
return 7;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Pow2WindowSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private static void InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, Span<ulong> oddPowers)
|
private static void InitializeMontgomeryOddPowersCpu(in MontgomeryDivisorData divisor, ulong modulus, ulong nPrime, Span<ulong> oddPowers)
|
||||||
|
|||||||
Reference in new issue
Block a user