mlemanczyk/even-perfect-numbers-scanner
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BitContradiction GPU performance improvements. Local variables partially optimized to limit registry pressure.

Browse Source
This commit is contained in:
mlemanczyk committed 2026-01-26 15:02:59 +01:00
1 parent 45f21efa56
commit b31aab4d89
4 files changed
+265 -288
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PerfectNumbers.Core/Cpu/MersenneNumberDivisorByDivisorCpuTesterWithTemplate.cs
+53 -45
View File
@@ -109,11 +109,13 @@ public struct MersenneNumberDivisorByDivisorCpuTesterWithTemplate() : IMersenneN
int[] found = new int[maxThreadCount]; int[] found = new int[maxThreadCount];
ulong[] batchIndexWords = new ulong[inputCapacity]; ulong[] batchIndexWords = new ulong[inputCapacity];
int debugWordsPerSlot = BitContradictionKernels.DebugWordCountPerSlot;
using MemoryBuffer1D<ulong, Stride1D.Dense> debugBuffer = gpu.Accelerator.Allocate1D<ulong>(maxThreadCount * debugWordsPerSlot);
using MemoryBuffer1D<int, Stride1D.Dense> delta8Keys = gpu.Accelerator.Allocate1D<int>(maxThreadCount * BitContradictionKernels.DeltaCacheSlots); using MemoryBuffer1D<int, Stride1D.Dense> delta8Keys = gpu.Accelerator.Allocate1D<int>(maxThreadCount * BitContradictionKernels.DeltaCacheSlots);
using MemoryBuffer1D<int, Stride1D.Dense> delta8Cache = gpu.Accelerator.Allocate1D<int>(maxThreadCount * BitContradictionKernels.DeltaCacheSlots * 4194304); using MemoryBuffer1D<int, Stride1D.Dense> delta8Cache = gpu.Accelerator.Allocate1D<int>(maxThreadCount * BitContradictionKernels.DeltaCacheSlots * 4194304);
ArrayView1D<ulong, Stride1D.Dense> debugView = debugBuffer.View; #if DETAILED_LOG
int debugWordsPerSlot = BitContradictionKernels.DebugWordCountPerSlot;
using MemoryBuffer1D<ulong, Stride1D.Dense> debugBuffer = gpu.Accelerator.Allocate1D<ulong>(maxThreadCount * debugWordsPerSlot);
ArrayView1D<ulong, Stride1D.Dense> debugView = debugBuffer.View;
#endif
BigInteger currentBatch = (currentK - BigInteger.One) / batchSizeBig; BigInteger currentBatch = (currentK - BigInteger.One) / batchSizeBig;
currentK = (currentBatch * batchSizeBig) + BigInteger.One; currentK = (currentBatch * batchSizeBig) + BigInteger.One;
@@ -161,7 +163,11 @@ public struct MersenneNumberDivisorByDivisorCpuTesterWithTemplate() : IMersenneN
} }
gpu.InputView.CopyFromCPU(stream, batchIndexWords.AsSpan(0, threadCount * wordCount)); gpu.InputView.CopyFromCPU(stream, batchIndexWords.AsSpan(0, threadCount * wordCount));
kernel(stream, threadCount, prime, gpu.InputView, countQ, gpu.OutputIntView, debugView, delta8Keys.View, delta8Cache.View); kernel(stream, threadCount, prime, gpu.InputView, countQ, gpu.OutputIntView,
#if DETAILED_LOG
debugView,
#endif
delta8Keys.View, delta8Cache.View);
stream.Synchronize(); stream.Synchronize();
gpu.OutputIntView.CopyToCPU(stream, found.AsSpan(0, threadCount)); gpu.OutputIntView.CopyToCPU(stream, found.AsSpan(0, threadCount));
@@ -211,47 +217,49 @@ public struct MersenneNumberDivisorByDivisorCpuTesterWithTemplate() : IMersenneN
// if (!dividesExact) // if (!dividesExact)
// { // {
// throw new InvalidOperationException($"BitContradiction CPU/GPU false positive for p={prime}, q={candidateQ}, k={candidateK}, batch={batchIndex}"); // throw new InvalidOperationException($"BitContradiction CPU/GPU false positive for p={prime}, q={candidateQ}, k={candidateK}, batch={batchIndex}");
if (prime == 138000001UL && candidateQ == new BigInteger(276000003)) #if DETAILED_LOG
{ if (prime == 138000001UL)
ulong[] debugWords = new ulong[debugWordsPerSlot]; {
debugView.SubView(i * debugWordsPerSlot, debugWordsPerSlot).CopyToCPU(stream, debugWords); ulong[] debugWords = new ulong[debugWordsPerSlot];
BigInteger gpuQ = ReadBigIntegerFromWords(debugWords.AsSpan(0, 16)); debugView.SubView(i * debugWordsPerSlot, debugWordsPerSlot).CopyToCPU(stream, debugWords);
BigInteger gpuInv = ReadBigIntegerFromWords(debugWords.AsSpan(16, 16)); BigInteger gpuQ = ReadBigIntegerFromWords(debugWords.AsSpan(0, 16));
BigInteger gpuALow = ReadBigIntegerFromWords(debugWords.AsSpan(32, 16)); BigInteger gpuInv = ReadBigIntegerFromWords(debugWords.AsSpan(16, 16));
ulong gpuQBits = debugWords[48]; BigInteger gpuALow = ReadBigIntegerFromWords(debugWords.AsSpan(32, 16));
ulong gpuQWordCount = debugWords[49]; ulong gpuQBits = debugWords[48];
ulong gpuQMask0 = debugWords[50]; ulong gpuQWordCount = debugWords[49];
ulong gpuQMask1 = debugWords[51]; ulong gpuQMask0 = debugWords[50];
ulong gpuKnownOne0 = debugWords[52]; ulong gpuQMask1 = debugWords[51];
ulong gpuKnownZero0 = debugWords[53]; ulong gpuKnownOne0 = debugWords[52];
ulong gpuAOneWin0 = debugWords[54]; ulong gpuKnownZero0 = debugWords[53];
ulong gpuAZeroWin0 = debugWords[55]; ulong gpuAOneWin0 = debugWords[54];
long gpuCarryMin = unchecked((long)debugWords[56]); ulong gpuAZeroWin0 = debugWords[55];
long gpuCarryMax = unchecked((long)debugWords[57]); long gpuCarryMin = unchecked((long)debugWords[56]);
long gpuMaxAllowedA = unchecked((long)debugWords[58]); long gpuCarryMax = unchecked((long)debugWords[57]);
long gpuMaxFixed = unchecked((long)debugWords[59]); long gpuMaxAllowedA = unchecked((long)debugWords[58]);
long gpuWindowSize = unchecked((long)debugWords[60]); long gpuMaxFixed = unchecked((long)debugWords[59]);
long gpuAWordCount = unchecked((long)debugWords[61]); long gpuWindowSize = unchecked((long)debugWords[60]);
long gpuMaxKnownA = unchecked((long)debugWords[62]); long gpuAWordCount = unchecked((long)debugWords[61]);
long gpuMaxFixedPrefilter = unchecked((long)debugWords[63]); long gpuMaxKnownA = unchecked((long)debugWords[62]);
ulong preKnownOne0 = debugWords[64]; long gpuMaxFixedPrefilter = unchecked((long)debugWords[63]);
ulong preKnownZero0 = debugWords[65]; ulong preKnownOne0 = debugWords[64];
ulong preAOneWin0 = debugWords[66]; ulong preKnownZero0 = debugWords[65];
ulong preAZeroWin0 = debugWords[67]; ulong preAOneWin0 = debugWords[66];
long preCarryMin = unchecked((long)debugWords[68]); ulong preAZeroWin0 = debugWords[67];
long preCarryMax = unchecked((long)debugWords[69]); long preCarryMin = unchecked((long)debugWords[68]);
long preSegmentBase = unchecked((long)debugWords[70]); long preCarryMax = unchecked((long)debugWords[69]);
long preWindowColumn = unchecked((long)debugWords[71]); long preSegmentBase = unchecked((long)debugWords[70]);
long gpuOffsetCount = unchecked((long)debugWords[72]); long preWindowColumn = unchecked((long)debugWords[71]);
long gpuMaxOffset = unchecked((long)debugWords[73]); long gpuOffsetCount = unchecked((long)debugWords[72]);
long gpuFirstForced = unchecked((long)debugWords[74]); long gpuMaxOffset = unchecked((long)debugWords[73]);
long gpuFirstRemaining = unchecked((long)debugWords[75]); long gpuFirstForced = unchecked((long)debugWords[74]);
long gpuFirstQWordsEff = unchecked((long)debugWords[76]); long gpuFirstRemaining = unchecked((long)debugWords[75]);
ulong gpuFirstQLastEff = debugWords[77]; long gpuFirstQWordsEff = unchecked((long)debugWords[76]);
long gpuFirstCarryMin = unchecked((long)debugWords[78]); ulong gpuFirstQLastEff = debugWords[77];
long gpuFirstCarryMax = unchecked((long)debugWords[79]); long gpuFirstCarryMin = unchecked((long)debugWords[78]);
throw new InvalidOperationException($"BitContradiction GPU debug for p={prime}, q={candidateQ}, k={candidateK}, qBits={gpuQBits}, qWordCount={gpuQWordCount}, qMask0=0x{gpuQMask0:X16}, qMask1=0x{gpuQMask1:X16}, knownOne0=0x{gpuKnownOne0:X16}, knownZero0=0x{gpuKnownZero0:X16}, aOneWin0=0x{gpuAOneWin0:X16}, aZeroWin0=0x{gpuAZeroWin0:X16}, carryMin={gpuCarryMin}, carryMax={gpuCarryMax}, maxAllowedA={gpuMaxAllowedA}, maxFixed={gpuMaxFixed}, windowSize={gpuWindowSize}, aWordCount={gpuAWordCount}, maxKnownA={gpuMaxKnownA}, maxFixedPrefilter={gpuMaxFixedPrefilter}, preKnownOne0=0x{preKnownOne0:X16}, preKnownZero0=0x{preKnownZero0:X16}, preAOneWin0=0x{preAOneWin0:X16}, preAZeroWin0=0x{preAZeroWin0:X16}, preCarryMin={preCarryMin}, preCarryMax={preCarryMax}, preSegmentBase={preSegmentBase}, preWindowColumn={preWindowColumn}, offsetCount={gpuOffsetCount}, maxOffset={gpuMaxOffset}, firstForced={gpuFirstForced}, firstRemaining={gpuFirstRemaining}, firstQWordsEff={gpuFirstQWordsEff}, firstQLastEff=0x{gpuFirstQLastEff:X16}, firstCarryMin={gpuFirstCarryMin}, firstCarryMax={gpuFirstCarryMax}, gpuQ={gpuQ}, gpuInv={gpuInv}, gpuALow={gpuALow}"); long gpuFirstCarryMax = unchecked((long)debugWords[79]);
} throw new InvalidOperationException($"BitContradiction GPU debug for p={prime}, q={candidateQ}, k={candidateK}, qBits={gpuQBits}, qWordCount={gpuQWordCount}, qMask0=0x{gpuQMask0:X16}, qMask1=0x{gpuQMask1:X16}, knownOne0=0x{gpuKnownOne0:X16}, knownZero0=0x{gpuKnownZero0:X16}, aOneWin0=0x{gpuAOneWin0:X16}, aZeroWin0=0x{gpuAZeroWin0:X16}, carryMin={gpuCarryMin}, carryMax={gpuCarryMax}, maxAllowedA={gpuMaxAllowedA}, maxFixed={gpuMaxFixed}, windowSize={gpuWindowSize}, aWordCount={gpuAWordCount}, maxKnownA={gpuMaxKnownA}, maxFixedPrefilter={gpuMaxFixedPrefilter}, preKnownOne0=0x{preKnownOne0:X16}, preKnownZero0=0x{preKnownZero0:X16}, preAOneWin0=0x{preAOneWin0:X16}, preAZeroWin0=0x{preAZeroWin0:X16}, preCarryMin={preCarryMin}, preCarryMax={preCarryMax}, preSegmentBase={preSegmentBase}, preWindowColumn={preWindowColumn}, offsetCount={gpuOffsetCount}, maxOffset={gpuMaxOffset}, firstForced={gpuFirstForced}, firstRemaining={gpuFirstRemaining}, firstQWordsEff={gpuFirstQWordsEff}, firstQLastEff=0x{gpuFirstQLastEff:X16}, firstCarryMin={gpuFirstCarryMin}, firstCarryMax={gpuFirstCarryMax}, gpuQ={gpuQ}, gpuInv={gpuInv}, gpuALow={gpuALow}");
}
#endif
foundDivisor = candidateQ; foundDivisor = candidateQ;
AcceleratorStreamPool.Return(gpu.AcceleratorIndex, stream); AcceleratorStreamPool.Return(gpu.AcceleratorIndex, stream);
PerfectNumbers.Core/Gpu/GpuKernelPool.cs
+10 -3
View File
@@ -125,13 +125,20 @@ public class GpuKernelPool
{ {
var loaded = accelerator.LoadAutoGroupedStreamKernel< var loaded = accelerator.LoadAutoGroupedStreamKernel<
Index1D, ulong, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<int, Stride1D.Dense>, Index1D, ulong, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<int, Stride1D.Dense>,
ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense> #if DETAILED_LOG
ArrayView1D<ulong, Stride1D.Dense>,
#endif
ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>
>(BitContradictionKernels.BitContradictionKernelScanWithDeltaCache64); >(BitContradictionKernels.BitContradictionKernelScanWithDeltaCache64);
var kernel = KernelUtil.GetKernel(loaded); var kernel = KernelUtil.GetKernel(loaded);
return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<int, Stride1D.Dense>, return kernel.CreateLauncherDelegate<Action<AcceleratorStream, Index1D, ulong, ArrayView1D<ulong, Stride1D.Dense>, int,
ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>>>(); ArrayView1D<int, Stride1D.Dense>,
#if DETAILED_LOG
ArrayView1D<ulong, Stride1D.Dense>,
#endif
ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>>>();
}); });
} }
PerfectNumbers.Core/Gpu/KernelContainer.cs
+4 -1
View File
@@ -23,7 +23,10 @@ public sealed class KernelContainer
ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>? Pow2ModOrder; ResidueAutomatonArgs, ArrayView<int>, ArrayView1D<ulong, Stride1D.Dense>>? Pow2ModOrder;
public Action<AcceleratorStream, Index1D, ulong, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<int, Stride1D.Dense>, public Action<AcceleratorStream, Index1D, ulong, ArrayView1D<ulong, Stride1D.Dense>, int, ArrayView1D<int, Stride1D.Dense>,
ArrayView1D<ulong, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>>? BitContradiction; #if DETAILED_LOG
ArrayView1D<ulong, Stride1D.Dense>,
#endif
ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>>? BitContradiction;
public Kernel? SmallPrimeFactor; public Kernel? SmallPrimeFactor;
PerfectNumbers.Core/Gpu/Kernels/BitContradictionKernels.cs
+198 -239
View File
@@ -57,17 +57,21 @@ internal static class BitContradictionKernels
ArrayView1D<ulong, Stride1D.Dense> batchIndexWords, ArrayView1D<ulong, Stride1D.Dense> batchIndexWords,
int countQ, int countQ,
ArrayView1D<int, Stride1D.Dense> foundOut, ArrayView1D<int, Stride1D.Dense> foundOut,
#if DETAILED_LOG
ArrayView1D<ulong, Stride1D.Dense> debugOut, ArrayView1D<ulong, Stride1D.Dense> debugOut,
#endif
ArrayView1D<int, Stride1D.Dense> deltaCacheKeys, ArrayView1D<int, Stride1D.Dense> deltaCacheKeys,
ArrayView1D<int, Stride1D.Dense> deltaCache) ArrayView1D<int, Stride1D.Dense> deltaCache)
{ {
int slot = index; // firstQWordsEff is reused as slot here to limit registry pressure
foundOut[slot] = -1; int firstQWordsEff = index;
foundOut[firstQWordsEff] = -1;
int keyBase = slot * DeltaCacheSlots; int keyBase = firstQWordsEff * DeltaCacheSlots;
long deltaBase = (long)slot * DeltaCacheSlots * DeltaLength; long deltaBase = (long)firstQWordsEff * DeltaCacheSlots * DeltaLength;
// Initialize keys for this thread to avoid false hits across launches. // Initialize keys for this thread to avoid false hits across launches.
for (int i = 0; i < DeltaCacheSlots; i++) int i;
for (i = 0; i < DeltaCacheSlots; i++)
{ {
deltaCacheKeys[keyBase + i] = int.MinValue; deltaCacheKeys[keyBase + i] = int.MinValue;
} }
@@ -77,17 +81,17 @@ internal static class BitContradictionKernels
return; return;
} }
int localCountQ = countQ;
ulong twoP = exponent << 1; ulong twoP = exponent << 1;
int startOffset = slot * MaxQWordCount; // firstQWordsEff is reused as startOffset here to limit registry pressure
firstQWordsEff *= MaxQWordCount;
int localCountQ = countQ;
var baseKWords = LocalMemory.Allocate<ulong>(MaxQWordCount); var baseKWords = LocalMemory.Allocate<ulong>(MaxQWordCount);
var kWords = LocalMemory.Allocate<ulong>(MaxQWordCount); var kWords = LocalMemory.Allocate<ulong>(MaxQWordCount);
var qWords = LocalMemory.Allocate<ulong>(MaxQWordCount); var qWords = LocalMemory.Allocate<ulong>(MaxQWordCount);
var qOffsets = LocalMemory.Allocate<int>(MaxQOffsets); var qOffsets = LocalMemory.Allocate<int>(MaxQOffsets);
var qPrefix = LocalMemory.Allocate<ulong>(ForcedALowWords); var qPrefix = LocalMemory.Allocate<ulong>(ForcedALowWords);
var reversePrefix = LocalMemory.Allocate<ulong>(ForcedALowWords);
var prefixMask = LocalMemory.Allocate<ulong>(ForcedALowWords);
var invWords = LocalMemory.Allocate<ulong>(ForcedALowWords); var invWords = LocalMemory.Allocate<ulong>(ForcedALowWords);
var tmpWords = LocalMemory.Allocate<ulong>(ForcedALowWords); var tmpWords = LocalMemory.Allocate<ulong>(ForcedALowWords);
var tmpWords2 = LocalMemory.Allocate<ulong>(ForcedALowWords); var tmpWords2 = LocalMemory.Allocate<ulong>(ForcedALowWords);
@@ -105,9 +109,9 @@ internal static class BitContradictionKernels
var lo1 = LocalMemory.Allocate<long>(IntervalBufferLength); var lo1 = LocalMemory.Allocate<long>(IntervalBufferLength);
var hi1 = LocalMemory.Allocate<long>(IntervalBufferLength); var hi1 = LocalMemory.Allocate<long>(IntervalBufferLength);
for (int i = 0; i < MaxQWordCount; i++) for (i = 0; i < MaxQWordCount; i++)
{ {
baseKWords[i] = batchIndexWords[startOffset + i]; baseKWords[i] = batchIndexWords[firstQWordsEff + i];
} }
if (MultiplyWordsByUInt64(baseKWords, kWords, MaxQWordCount, (ulong)BatchCount) != 0UL) if (MultiplyWordsByUInt64(baseKWords, kWords, MaxQWordCount, (ulong)BatchCount) != 0UL)
@@ -163,7 +167,6 @@ internal static class BitContradictionKernels
int qWordCount = (qBitLen + 63) >> 6; int qWordCount = (qBitLen + 63) >> 6;
int offsetCount; int offsetCount;
int maxOffsetValue = 0;
if (!BuildQOneOffsetsWords(qWords, qWordCount, qOffsets, out offsetCount)) if (!BuildQOneOffsetsWords(qWords, qWordCount, qOffsets, out offsetCount))
{ {
continue; continue;
@@ -171,10 +174,10 @@ internal static class BitContradictionKernels
if (offsetCount <= 0) if (offsetCount <= 0)
{ {
maxOffsetValue = qOffsets[offsetCount - 1];
continue; continue;
} }
int maxOffsetValue = qOffsets[offsetCount - 1];
int windowSize = qBitLen < ForcedALowBits ? ForcedALowBits : qBitLen; int windowSize = qBitLen < ForcedALowBits ? ForcedALowBits : qBitLen;
windowSize = (windowSize + 63) & ~63; windowSize = (windowSize + 63) & ~63;
int aWordCount = (windowSize + 63) >> 6; int aWordCount = (windowSize + 63) >> 6;
@@ -193,106 +196,120 @@ internal static class BitContradictionKernels
ComputeInverseMod2k(qPrefix, invWords, tmpWords, tmpWords2, ForcedALowWords); ComputeInverseMod2k(qPrefix, invWords, tmpWords, tmpWords2, ForcedALowWords);
NegateMod2k(invWords, aLowWords, ForcedALowWords); NegateMod2k(invWords, aLowWords, ForcedALowWords);
for (int i = 0; i < qWordCount; i++) for (i = 0; i < qWordCount; i++)
{ {
qMaskWords[i] = qWords[i]; qMaskWords[i] = qWords[i];
} }
ulong lastWordMask = (qBitLen & 63) == 0 ? ulong.MaxValue : ((1UL << (qBitLen & 63)) - 1UL); i = qBitLen & 63;
ulong lastWordMask = i == 0 ? ulong.MaxValue : ((1UL << i) - 1UL);
if (qWordCount > 0) if (qWordCount > 0)
{ {
qMaskWords[qWordCount - 1] &= lastWordMask; qMaskWords[qWordCount - 1] &= lastWordMask;
} }
int firstRemaining;
ulong firstQLastEff, wordValue;
int maxFixed = maxAllowedA < (ForcedALowBits - 1) ? maxAllowedA : (ForcedALowBits - 1); int maxFixed = maxAllowedA < (ForcedALowBits - 1) ? maxAllowedA : (ForcedALowBits - 1);
if (maxFixed >= 0) if (maxFixed >= 0)
{ {
int fullWords = maxFixed >> 6; // firstQWordsEff is reused as fullWords here to limit registry pressure
int lastBit = maxFixed & 63; firstQWordsEff = maxFixed >> 6;
for (int w = 0; w < fullWords; w++) // firstRemaining is reused as lastBit here to limit registry pressure
firstRemaining = maxFixed & 63;
// i is reused as word to limit registry pressure
for (i = 0; i < firstQWordsEff; i++)
{ {
ulong wordValue = aLowWords[w]; wordValue = aLowWords[i];
knownOne0[w] = wordValue; knownOne0[i] = wordValue;
knownZero0[w] = ~wordValue; knownZero0[i] = ~wordValue;
} }
if (fullWords < aWordCount) if (firstQWordsEff < aWordCount)
{ {
ulong wordValue = aLowWords[fullWords]; wordValue = aLowWords[firstQWordsEff];
ulong mask = lastBit == 63 ? ulong.MaxValue : ((1UL << (lastBit + 1)) - 1UL); // firstQLastEff is reused as mask here to limit registry pressure
knownOne0[fullWords] = wordValue & mask; firstQLastEff = firstRemaining == 63 ? ulong.MaxValue : ((1UL << (firstRemaining + 1)) - 1UL);
knownZero0[fullWords] = (~wordValue) & mask; knownOne0[firstQWordsEff] = wordValue & firstQLastEff;
knownZero0[firstQWordsEff] = (~wordValue) & firstQLastEff;
} }
} }
int maxKnownA = maxFixed >= 0 ? maxFixed : -1; int maxKnownA = maxFixed >= 0 ? maxFixed : -1;
int firstForced = -1; int firstForced = -1;
int firstRemaining = -1; firstRemaining = -1;
int firstQWordsEff = 0; firstQWordsEff = 0;
ulong firstQLastEff = 0; firstQLastEff = 0;
long firstCarryMin = 0; long firstCarryMin = 0;
long firstCarryMax = 0; long firstCarryMax = 0;
int maxFixedPrefilter = 512 - 1; int maxFixedPreFilter = 512 - 1;
if (maxFixedPrefilter > maxKnownA) maxFixedPreFilter = maxFixedPreFilter > maxKnownA ? maxKnownA : maxFixedPreFilter;
{ maxFixedPreFilter = maxFixedPreFilter > maxAllowedA ? maxAllowedA : maxFixedPreFilter;
maxFixedPrefilter = maxKnownA;
}
if (maxFixedPrefilter > maxAllowedA) int column;
int word;
int aIndex;
int forced;
int bitShift;
if (maxFixedPreFilter >= 8)
{ {
maxFixedPrefilter = maxAllowedA; // tmpWords is reused for reversePrefix
} ReverseBits1024(qPrefix, tmpWords);
if (maxFixedPrefilter >= 8)
{
ReverseBits1024(qPrefix, reversePrefix);
bool prefixOk = true; bool prefixOk = true;
CarryRange carryLow = CarryRange.Zero; CarryRange carryLow = CarryRange.Zero;
for (int column = 0; column <= maxFixedPrefilter; column++) for (column = 0; column <= maxFixedPreFilter; column++)
{ {
int forced = 0; forced = 0;
if (column <= 1023) if (column <= 1023)
{ {
int shift = 1023 - column; // i is reused as shift here to limit registry pressure
int wordShift = shift >> 6; i = 1023 - column;
int bitShift = shift & 63; // word is reused as wordShift here to limit registry pressure
for (int word = 0; word < ForcedALowWords; word++) word = i >> 6;
bitShift = i & 63;
// i is reused as word to limit registry pressure
for (i = 0; i < ForcedALowWords; i++)
{ {
int src = word + wordShift; // aIndex is reused as src here to limit registry pressure
ulong value = 0UL; aIndex = i + word;
if (src < ForcedALowWords) wordValue = 0UL;
if (aIndex < ForcedALowWords)
{ {
value = reversePrefix[src] >> bitShift; wordValue = tmpWords[aIndex] >> bitShift;
if (bitShift != 0 && src + 1 < ForcedALowWords) if (bitShift != 0 && aIndex + 1 < ForcedALowWords)
{ {
value |= reversePrefix[src + 1] << (64 - bitShift); wordValue |= tmpWords[aIndex + 1] << (64 - bitShift);
} }
} }
prefixMask[word] = value; // tmpWords2 is reused for prefixMask
tmpWords2[i] = wordValue;
} }
for (int word = 0; word < ForcedALowWords; word++) // i is reused as word to limit registry pressure
for (i = 0; i < ForcedALowWords; i++)
{ {
forced += XMath.PopCount(knownOne0[word] & prefixMask[word]); forced += XMath.PopCount(knownOne0[i] & tmpWords2[i]);
} }
} }
else else
{ {
for (int i = 0; i < offsetCount; i++) for (i = 0; i < offsetCount; i++)
{ {
int t = qOffsets[i]; // bitShift is reused as t here to limit registry pressure
if (t > column) bitShift = qOffsets[i];
if (bitShift > column)
{ {
break; break;
} }
int aIndex = column - t; aIndex = column - bitShift;
int word = aIndex >> 6; word = aIndex >> 6;
ulong mask = 1UL << (aIndex & 63); // wordValue is reused as mask here to limit registry pressure
if ((knownOne0[word] & mask) != 0UL) wordValue = 1UL << (aIndex & 63);
if ((knownOne0[word] & wordValue) != 0UL)
{ {
forced++; forced++;
} }
@@ -312,45 +329,48 @@ internal static class BitContradictionKernels
} }
} }
int segmentBase = 0; // word is reused as segmentBase from here to limit registry pressure
int windowColumn = 0; word = 0;
// column is reused as windowColumn from here to limit registry pressure
column = 0;
int initialState = GetAKnownStateRowAware(0, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); // aIndex is reused as initialState from here to limit registry pressure
if (initialState == 1) aIndex = GetAKnownStateRowAware(0, word, windowSize, knownOne0, knownZero0, knownOne1, knownZero1);
{ aOneWin[0] |= aIndex == 1 ? 1UL : 0UL;
aOneWin[0] |= 1UL; aZeroWin[0] |= aIndex == 2 ? 1UL : 0UL;
}
else if (initialState == 2)
{
aZeroWin[0] |= 1UL;
}
if (!TryRunHighBitAndBorrowPrefiltersCombinedGpu(qOffsets, offsetCount, pLong, maxAllowedA, lo0, hi0, lo1, hi1)) if (!TryRunHighBitAndBorrowPrefiltersCombinedGpu(qOffsets, offsetCount, pLong, maxAllowedA, lo0, hi0, lo1, hi1))
{ {
continue; continue;
} }
int unknown = offsetCount; // forced is reused as unknown from here to limit registry pressure
int cacheSlot = unknown & (DeltaCacheSlots - 1); forced = offsetCount;
int keyIndex = keyBase + cacheSlot; // i is reused as cacheSlot from here to limit registry pressure
int cachedKey = deltaCacheKeys[keyIndex]; i = forced & (DeltaCacheSlots - 1);
int cacheSlotBase = (int)(deltaBase + (long)cacheSlot * DeltaLength); // bitShift is reused as keyIndex from here to limit registry pressure
var delta8 = deltaCache.SubView(cacheSlotBase, DeltaLength); bitShift = keyBase + i;
if (cachedKey != unknown) // i is reused as cacheSlotBase from here to limit registry pressure
i = (int)(deltaBase + (long)i * DeltaLength);
var delta8 = deltaCache.SubView(i, DeltaLength);
if (deltaCacheKeys[bitShift] != forced)
{ {
BuildStableUnknownDelta8(unknown, delta8); BuildStableUnknownDelta8(forced, delta8);
deltaCacheKeys[keyIndex] = unknown; deltaCacheKeys[bitShift] = forced;
} }
CarryRange carry = CarryRange.Zero; CarryRange carry = CarryRange.Zero;
ulong preKnownOne0 = knownOne0[0]; #if DETAILED_LOG
ulong preKnownZero0 = knownZero0[0]; int preWindowColumn = windowColumn;
ulong preAOneWin0 = aOneWin[0]; int preSegmentBase = segmentBase;
ulong preAZeroWin0 = aZeroWin[0]; ulong preKnownZero0 = knownZero0[0];
long preCarryMin = 0L; ulong preKnownOne0 = knownOne0[0];
long preCarryMax = 0L; ulong preAOneWin0 = aOneWin[0];
int preSegmentBase = segmentBase; ulong preAZeroWin0 = aZeroWin[0];
int preWindowColumn = windowColumn; long preCarryMin = 0L;
long preCarryMax = 0L;
#endif
bool ok = TryProcessBottomUpBlockRowAwareGpu( bool ok = TryProcessBottomUpBlockRowAwareGpu(
qOffsets, qOffsets,
offsetCount, offsetCount,
@@ -362,7 +382,7 @@ internal static class BitContradictionKernels
pLong, pLong,
ref carry, ref carry,
ref maxKnownA, ref maxKnownA,
ref segmentBase, ref word,
knownOne0, knownOne0,
knownOne1, knownOne1,
knownZero0, knownZero0,
@@ -371,7 +391,7 @@ internal static class BitContradictionKernels
aWordCount, aWordCount,
aOneWin, aOneWin,
aZeroWin, aZeroWin,
ref windowColumn, ref column,
delta8, delta8,
offsetCount, offsetCount,
ref firstForced, ref firstForced,
@@ -381,57 +401,59 @@ internal static class BitContradictionKernels
ref firstCarryMin, ref firstCarryMin,
ref firstCarryMax); ref firstCarryMax);
if (ok) #if DETAILED_LOG
{ if (ok)
int debugBase = slot * DebugWordCountPerSlot; {
for (int w = 0; w < ForcedALowWords; w++) int debugBase = slot * DebugWordCountPerSlot;
{ for (int w = 0; w < ForcedALowWords; w++)
debugOut[debugBase + w] = qWords[w]; {
} debugOut[debugBase + w] = qWords[w];
for (int w = 0; w < ForcedALowWords; w++) }
{ for (int w = 0; w < ForcedALowWords; w++)
debugOut[debugBase + ForcedALowWords + w] = invWords[w]; {
} debugOut[debugBase + ForcedALowWords + w] = invWords[w];
for (int w = 0; w < ForcedALowWords; w++) }
{ for (int w = 0; w < ForcedALowWords; w++)
debugOut[debugBase + (ForcedALowWords * 2) + w] = aLowWords[w]; {
} debugOut[debugBase + (ForcedALowWords * 2) + w] = aLowWords[w];
debugOut[debugBase + (ForcedALowWords * 3)] = (ulong)qBitLen; }
int extraBase = debugBase + (ForcedALowWords * 3) + 1; debugOut[debugBase + (ForcedALowWords * 3)] = (ulong)qBitLen;
debugOut[extraBase] = (ulong)qWordCount; int extraBase = debugBase + (ForcedALowWords * 3) + 1;
debugOut[extraBase + 1] = qMaskWords[0]; debugOut[extraBase] = (ulong)qWordCount;
debugOut[extraBase + 2] = qMaskWords[1]; debugOut[extraBase + 1] = qMaskWords[0];
debugOut[extraBase + 3] = knownOne0[0]; debugOut[extraBase + 2] = qMaskWords[1];
debugOut[extraBase + 4] = knownZero0[0]; debugOut[extraBase + 3] = knownOne0[0];
debugOut[extraBase + 5] = aOneWin[0]; debugOut[extraBase + 4] = knownZero0[0];
debugOut[extraBase + 6] = aZeroWin[0]; debugOut[extraBase + 5] = aOneWin[0];
debugOut[extraBase + 7] = (ulong)carry.Min; debugOut[extraBase + 6] = aZeroWin[0];
debugOut[extraBase + 8] = (ulong)carry.Max; debugOut[extraBase + 7] = (ulong)carry.Min;
debugOut[extraBase + 9] = (ulong)maxAllowedA; debugOut[extraBase + 8] = (ulong)carry.Max;
debugOut[extraBase + 10] = (ulong)maxFixed; debugOut[extraBase + 9] = (ulong)maxAllowedA;
debugOut[extraBase + 11] = (ulong)windowSize; debugOut[extraBase + 10] = (ulong)maxFixed;
debugOut[extraBase + 12] = (ulong)aWordCount; debugOut[extraBase + 11] = (ulong)windowSize;
debugOut[extraBase + 13] = (ulong)maxKnownA; debugOut[extraBase + 12] = (ulong)aWordCount;
debugOut[extraBase + 14] = (ulong)maxFixedPrefilter; debugOut[extraBase + 13] = (ulong)maxKnownA;
debugOut[extraBase + 15] = preKnownOne0; debugOut[extraBase + 14] = (ulong)maxFixedPrefilter;
debugOut[extraBase + 16] = preKnownZero0; debugOut[extraBase + 15] = preKnownOne0;
debugOut[extraBase + 17] = preAOneWin0; debugOut[extraBase + 16] = preKnownZero0;
debugOut[extraBase + 18] = preAZeroWin0; debugOut[extraBase + 17] = preAOneWin0;
debugOut[extraBase + 19] = (ulong)preCarryMin; debugOut[extraBase + 18] = preAZeroWin0;
debugOut[extraBase + 20] = (ulong)preCarryMax; debugOut[extraBase + 19] = (ulong)preCarryMin;
debugOut[extraBase + 21] = (ulong)preSegmentBase; debugOut[extraBase + 20] = (ulong)preCarryMax;
debugOut[extraBase + 22] = (ulong)preWindowColumn; debugOut[extraBase + 21] = (ulong)preSegmentBase;
debugOut[extraBase + 23] = (ulong)offsetCount; debugOut[extraBase + 22] = (ulong)preWindowColumn;
debugOut[extraBase + 24] = (ulong)maxOffsetValue; debugOut[extraBase + 23] = (ulong)offsetCount;
debugOut[extraBase + 25] = (ulong)firstForced; debugOut[extraBase + 24] = (ulong)maxOffsetValue;
debugOut[extraBase + 26] = (ulong)firstRemaining; debugOut[extraBase + 25] = (ulong)firstForced;
debugOut[extraBase + 27] = (ulong)firstQWordsEff; debugOut[extraBase + 26] = (ulong)firstRemaining;
debugOut[extraBase + 28] = firstQLastEff; debugOut[extraBase + 27] = (ulong)firstQWordsEff;
debugOut[extraBase + 29] = (ulong)firstCarryMin; debugOut[extraBase + 28] = firstQLastEff;
debugOut[extraBase + 30] = (ulong)firstCarryMax; debugOut[extraBase + 29] = (ulong)firstCarryMin;
foundOut[slot] = batch; debugOut[extraBase + 30] = (ulong)firstCarryMax;
return; foundOut[slot] = batch;
} return;
}
#endif
} }
} }
@@ -460,8 +482,8 @@ internal static class BitContradictionKernels
var qWords = LocalMemory.Allocate<ulong>(MaxQWordCount); var qWords = LocalMemory.Allocate<ulong>(MaxQWordCount);
var qOffsets = LocalMemory.Allocate<int>(MaxQOffsets); var qOffsets = LocalMemory.Allocate<int>(MaxQOffsets);
var qPrefix = LocalMemory.Allocate<ulong>(ForcedALowWords); var qPrefix = LocalMemory.Allocate<ulong>(ForcedALowWords);
var reversePrefix = LocalMemory.Allocate<ulong>(ForcedALowWords);
var prefixMask = LocalMemory.Allocate<ulong>(ForcedALowWords);
var invWords = LocalMemory.Allocate<ulong>(ForcedALowWords); var invWords = LocalMemory.Allocate<ulong>(ForcedALowWords);
var tmpWords = LocalMemory.Allocate<ulong>(ForcedALowWords); var tmpWords = LocalMemory.Allocate<ulong>(ForcedALowWords);
var tmpWords2 = LocalMemory.Allocate<ulong>(ForcedALowWords); var tmpWords2 = LocalMemory.Allocate<ulong>(ForcedALowWords);
@@ -610,19 +632,13 @@ internal static class BitContradictionKernels
long firstCarryMax = 0; long firstCarryMax = 0;
int maxFixedPrefilter = 512 - 1; int maxFixedPrefilter = 512 - 1;
if (maxFixedPrefilter > maxKnownA) maxFixedPrefilter = maxFixedPrefilter > maxKnownA ? maxKnownA : maxFixedPrefilter;
{ maxFixedPrefilter = maxFixedPrefilter > maxAllowedA ? maxAllowedA : maxFixedPrefilter;
maxFixedPrefilter = maxKnownA;
}
if (maxFixedPrefilter > maxAllowedA)
{
maxFixedPrefilter = maxAllowedA;
}
if (maxFixedPrefilter >= 8) if (maxFixedPrefilter >= 8)
{ {
ReverseBits1024(qPrefix, reversePrefix); // tmpWords is reused for reversePrefix
ReverseBits1024(qPrefix, tmpWords);
bool prefixOk = true; bool prefixOk = true;
CarryRange carryLow = CarryRange.Zero; CarryRange carryLow = CarryRange.Zero;
@@ -640,19 +656,20 @@ internal static class BitContradictionKernels
ulong value = 0UL; ulong value = 0UL;
if (src < ForcedALowWords) if (src < ForcedALowWords)
{ {
value = reversePrefix[src] >> bitShift; value = tmpWords[src] >> bitShift;
if (bitShift != 0 && src + 1 < ForcedALowWords) if (bitShift != 0 && src + 1 < ForcedALowWords)
{ {
value |= reversePrefix[src + 1] << (64 - bitShift); value |= tmpWords[src + 1] << (64 - bitShift);
} }
} }
prefixMask[word] = value; // tmpWords2 is reused for prefixMask
tmpWords2[word] = value;
} }
for (int word = 0; word < ForcedALowWords; word++) for (int word = 0; word < ForcedALowWords; word++)
{ {
forced += XMath.PopCount(knownOne0[word] & prefixMask[word]); forced += XMath.PopCount(knownOne0[word] & tmpWords2[word]);
} }
} }
else else
@@ -692,14 +709,8 @@ internal static class BitContradictionKernels
int windowColumn = 0; int windowColumn = 0;
int initialState = GetAKnownStateRowAware(0, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); int initialState = GetAKnownStateRowAware(0, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1);
if (initialState == 1) aOneWin[0] |= initialState == 1 ? 1UL : 0UL;
{ aZeroWin[0] |= initialState == 2 ? 1UL : 0UL;
aOneWin[0] |= 1UL;
}
else if (initialState == 2)
{
aZeroWin[0] |= 1UL;
}
if (!TryRunHighBitAndBorrowPrefiltersCombinedGpu(qOffsets, offsetCount, pLong, maxAllowedA, lo0, hi0, lo1, hi1)) if (!TryRunHighBitAndBorrowPrefiltersCombinedGpu(qOffsets, offsetCount, pLong, maxAllowedA, lo0, hi0, lo1, hi1))
{ {
@@ -1106,20 +1117,14 @@ internal static class BitContradictionKernels
int columnPlusOne = column + 1; int columnPlusOne = column + 1;
int highBitColumns = HighBitCarryPrefilterColumns; int highBitColumns = HighBitCarryPrefilterColumns;
if (highBitColumns > columnPlusOne) highBitColumns = highBitColumns > columnPlusOne ? columnPlusOne : highBitColumns;
{
highBitColumns = columnPlusOne;
}
int borrowColumns = 0; int borrowColumns = 0;
bool runBorrowPrefilter = qOffsetCount >= TopDownBorrowMinUnknown; bool runBorrowPrefilter = qOffsetCount >= TopDownBorrowMinUnknown;
if (runBorrowPrefilter) if (runBorrowPrefilter)
{ {
borrowColumns = TopDownBorrowColumns; borrowColumns = TopDownBorrowColumns;
if (borrowColumns > columnPlusOne) borrowColumns = borrowColumns > columnPlusOne ? columnPlusOne : borrowColumns;
{
borrowColumns = columnPlusOne;
}
} }
int sequentialColumns = highBitColumns > borrowColumns ? highBitColumns : borrowColumns; int sequentialColumns = highBitColumns > borrowColumns ? highBitColumns : borrowColumns;
@@ -1140,10 +1145,7 @@ internal static class BitContradictionKernels
int step = 0; int step = 0;
int upperT; int upperT;
int lowerT = column - maxAllowedA; int lowerT = column - maxAllowedA;
if (lowerT < 0) lowerT = lowerT < 0 ? 0 : lowerT;
{
lowerT = 0;
}
FindBounds(qOffsets, qOffsetCount, lowerT, column, out lowerT, out upperT); FindBounds(qOffsets, qOffsetCount, lowerT, column, out lowerT, out upperT);
int n = upperT - lowerT; int n = upperT - lowerT;
@@ -1172,7 +1174,7 @@ internal static class BitContradictionKernels
long rMax = outHi[i]; long rMax = outHi[i];
if (rMax < 0) continue; if (rMax < 0) continue;
if (rMin < 0) rMin = 0; rMin = rMin < 0 ? 0 : rMin;
if (rMax < rMin) continue; if (rMax < rMin) continue;
const long TwicePlusOneOverflowLimit = (long.MaxValue - 1) >> 1; const long TwicePlusOneOverflowLimit = (long.MaxValue - 1) >> 1;
@@ -1250,10 +1252,7 @@ internal static class BitContradictionKernels
} }
lowerT = column - maxAllowedA; lowerT = column - maxAllowedA;
if (lowerT < 0) lowerT = lowerT < 0 ? 0 : lowerT;
{
lowerT = 0;
}
FindBounds(qOffsets, qOffsetCount, lowerT, column, out lowerT, out upperT); FindBounds(qOffsets, qOffsetCount, lowerT, column, out lowerT, out upperT);
n = upperT - lowerT; n = upperT - lowerT;
@@ -1273,7 +1272,7 @@ internal static class BitContradictionKernels
return; return;
} }
if (lo < 0) lo = 0; lo = lo < 0 ? 0 : lo;
if (hi < 0) return; if (hi < 0) return;
if (count == 0) if (count == 0)
@@ -1287,11 +1286,7 @@ internal static class BitContradictionKernels
long lastHi = hiArr[count - 1]; long lastHi = hiArr[count - 1];
if (lo <= lastHi + 1) if (lo <= lastHi + 1)
{ {
if (hi > lastHi) hiArr[count - 1] = hi > lastHi ? hi : lastHi;
{
hiArr[count - 1] = hi;
}
return; return;
} }
@@ -1387,18 +1382,9 @@ internal static class BitContradictionKernels
{ {
ulong minuend = i == 0 ? 2UL : 0UL; ulong minuend = i == 0 ? 2UL : 0UL;
ulong sub = value[i]; ulong sub = value[i];
ulong res; bool hasBorrow = borrow != 0UL;
if (borrow == 0UL) ulong res = hasBorrow ? (minuend - sub - 1UL) : (minuend - sub);
{ borrow = hasBorrow ? (minuend <= sub ? 1UL : 0UL) : (minuend < sub ? 1UL : 0UL);
res = minuend - sub;
borrow = minuend < sub ? 1UL : 0UL;
}
else
{
res = minuend - sub - 1UL;
borrow = minuend <= sub ? 1UL : 0UL;
}
result[i] = res; result[i] = res;
} }
} }
@@ -1610,16 +1596,8 @@ internal static class BitContradictionKernels
rel = aIndex - segmentBase; rel = aIndex - segmentBase;
word = rel >> 6; word = rel >> 6;
mask = 1UL << (rel & 63); mask = 1UL << (rel & 63);
if (isOne) knownOne0[word] = isOne ? (knownOne0[word] | mask) : (knownOne0[word] & ~mask);
{ knownZero0[word] = isOne ? (knownZero0[word] & ~mask) : (knownZero0[word] | mask);
knownOne0[word] |= mask;
knownZero0[word] &= ~mask;
}
else
{
knownZero0[word] |= mask;
knownOne0[word] &= ~mask;
}
return; return;
} }
@@ -1630,16 +1608,8 @@ internal static class BitContradictionKernels
rel = aIndex - segment1Start; rel = aIndex - segment1Start;
word = rel >> 6; word = rel >> 6;
mask = 1UL << (rel & 63); mask = 1UL << (rel & 63);
if (isOne) knownOne1[word] = isOne ? (knownOne1[word] | mask) : (knownOne1[word] & ~mask);
{ knownZero1[word] = isOne ? (knownZero1[word] & ~mask) : (knownZero1[word] | mask);
knownOne1[word] |= mask;
knownZero1[word] &= ~mask;
}
else
{
knownZero1[word] |= mask;
knownOne1[word] &= ~mask;
}
} }
} }
@@ -1818,14 +1788,9 @@ internal static class BitContradictionKernels
? 2 ? 2
: GetAKnownStateRowAware(aIndex, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); : GetAKnownStateRowAware(aIndex, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1);
if (state == 1) ulong stepMask = 1UL << (step & 63);
{ aOneWin[step >> 6] |= state == 1 ? stepMask : 0UL;
aOneWin[step >> 6] |= 1UL << (step & 63); aZeroWin[step >> 6] |= state == 2 ? stepMask : 0UL;
}
else if (state == 2)
{
aZeroWin[step >> 6] |= 1UL << (step & 63);
}
} }
if (qWordCount > 0) if (qWordCount > 0)
@@ -1954,14 +1919,8 @@ internal static class BitContradictionKernels
? 2 ? 2
: GetAKnownStateRowAware(nextA, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); : GetAKnownStateRowAware(nextA, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1);
if (nextState == 1) aOneWin[0] |= nextState == 1 ? 1UL : 0UL;
{ aZeroWin[0] |= nextState == 2 ? 1UL : 0UL;
aOneWin[0] |= 1UL;
}
else if (nextState == 2)
{
aZeroWin[0] |= 1UL;
}
startColumn++; startColumn++;
windowColumn = startColumn; windowColumn = startColumn;