From b31aab4d8994d2c8aa0a92b805d512c366bad58f Mon Sep 17 00:00:00 2001 From: mlemanczyk Date: Mon, 26 Jan 2026 15:02:59 +0100 Subject: [PATCH] BitContradiction GPU performance improvements. Local variables partially optimized to limit registry pressure. --- ...erDivisorByDivisorCpuTesterWithTemplate.cs | 98 ++-- PerfectNumbers.Core/Gpu/GpuKernelPool.cs | 13 +- PerfectNumbers.Core/Gpu/KernelContainer.cs | 5 +- .../Gpu/Kernels/BitContradictionKernels.cs | 437 ++++++++---------- 4 files changed, 265 insertions(+), 288 deletions(-) diff --git a/PerfectNumbers.Core/Cpu/MersenneNumberDivisorByDivisorCpuTesterWithTemplate.cs b/PerfectNumbers.Core/Cpu/MersenneNumberDivisorByDivisorCpuTesterWithTemplate.cs index 6555130..aadda42 100644 --- a/PerfectNumbers.Core/Cpu/MersenneNumberDivisorByDivisorCpuTesterWithTemplate.cs +++ b/PerfectNumbers.Core/Cpu/MersenneNumberDivisorByDivisorCpuTesterWithTemplate.cs @@ -109,11 +109,13 @@ public struct MersenneNumberDivisorByDivisorCpuTesterWithTemplate() : IMersenneN int[] found = new int[maxThreadCount]; ulong[] batchIndexWords = new ulong[inputCapacity]; - int debugWordsPerSlot = BitContradictionKernels.DebugWordCountPerSlot; - using MemoryBuffer1D debugBuffer = gpu.Accelerator.Allocate1D(maxThreadCount * debugWordsPerSlot); using MemoryBuffer1D delta8Keys = gpu.Accelerator.Allocate1D(maxThreadCount * BitContradictionKernels.DeltaCacheSlots); using MemoryBuffer1D delta8Cache = gpu.Accelerator.Allocate1D(maxThreadCount * BitContradictionKernels.DeltaCacheSlots * 4194304); - ArrayView1D debugView = debugBuffer.View; + #if DETAILED_LOG + int debugWordsPerSlot = BitContradictionKernels.DebugWordCountPerSlot; + using MemoryBuffer1D debugBuffer = gpu.Accelerator.Allocate1D(maxThreadCount * debugWordsPerSlot); + ArrayView1D debugView = debugBuffer.View; + #endif BigInteger currentBatch = (currentK - BigInteger.One) / batchSizeBig; currentK = (currentBatch * batchSizeBig) + BigInteger.One; @@ -161,7 +163,11 @@ public struct MersenneNumberDivisorByDivisorCpuTesterWithTemplate() : IMersenneN } gpu.InputView.CopyFromCPU(stream, batchIndexWords.AsSpan(0, threadCount * wordCount)); - kernel(stream, threadCount, prime, gpu.InputView, countQ, gpu.OutputIntView, debugView, delta8Keys.View, delta8Cache.View); + kernel(stream, threadCount, prime, gpu.InputView, countQ, gpu.OutputIntView, + #if DETAILED_LOG + debugView, + #endif + delta8Keys.View, delta8Cache.View); stream.Synchronize(); gpu.OutputIntView.CopyToCPU(stream, found.AsSpan(0, threadCount)); @@ -211,47 +217,49 @@ public struct MersenneNumberDivisorByDivisorCpuTesterWithTemplate() : IMersenneN // if (!dividesExact) // { // throw new InvalidOperationException($"BitContradiction CPU/GPU false positive for p={prime}, q={candidateQ}, k={candidateK}, batch={batchIndex}"); - if (prime == 138000001UL && candidateQ == new BigInteger(276000003)) - { - ulong[] debugWords = new ulong[debugWordsPerSlot]; - debugView.SubView(i * debugWordsPerSlot, debugWordsPerSlot).CopyToCPU(stream, debugWords); - BigInteger gpuQ = ReadBigIntegerFromWords(debugWords.AsSpan(0, 16)); - BigInteger gpuInv = ReadBigIntegerFromWords(debugWords.AsSpan(16, 16)); - BigInteger gpuALow = ReadBigIntegerFromWords(debugWords.AsSpan(32, 16)); - ulong gpuQBits = debugWords[48]; - ulong gpuQWordCount = debugWords[49]; - ulong gpuQMask0 = debugWords[50]; - ulong gpuQMask1 = debugWords[51]; - ulong gpuKnownOne0 = debugWords[52]; - ulong gpuKnownZero0 = debugWords[53]; - ulong gpuAOneWin0 = debugWords[54]; - ulong gpuAZeroWin0 = debugWords[55]; - long gpuCarryMin = unchecked((long)debugWords[56]); - long gpuCarryMax = unchecked((long)debugWords[57]); - long gpuMaxAllowedA = unchecked((long)debugWords[58]); - long gpuMaxFixed = unchecked((long)debugWords[59]); - long gpuWindowSize = unchecked((long)debugWords[60]); - long gpuAWordCount = unchecked((long)debugWords[61]); - long gpuMaxKnownA = unchecked((long)debugWords[62]); - long gpuMaxFixedPrefilter = unchecked((long)debugWords[63]); - ulong preKnownOne0 = debugWords[64]; - ulong preKnownZero0 = debugWords[65]; - ulong preAOneWin0 = debugWords[66]; - ulong preAZeroWin0 = debugWords[67]; - long preCarryMin = unchecked((long)debugWords[68]); - long preCarryMax = unchecked((long)debugWords[69]); - long preSegmentBase = unchecked((long)debugWords[70]); - long preWindowColumn = unchecked((long)debugWords[71]); - long gpuOffsetCount = unchecked((long)debugWords[72]); - long gpuMaxOffset = unchecked((long)debugWords[73]); - long gpuFirstForced = unchecked((long)debugWords[74]); - long gpuFirstRemaining = unchecked((long)debugWords[75]); - long gpuFirstQWordsEff = unchecked((long)debugWords[76]); - ulong gpuFirstQLastEff = debugWords[77]; - long gpuFirstCarryMin = unchecked((long)debugWords[78]); - long gpuFirstCarryMax = unchecked((long)debugWords[79]); - throw new InvalidOperationException($"BitContradiction GPU debug for p={prime}, q={candidateQ}, k={candidateK}, qBits={gpuQBits}, qWordCount={gpuQWordCount}, qMask0=0x{gpuQMask0:X16}, qMask1=0x{gpuQMask1:X16}, knownOne0=0x{gpuKnownOne0:X16}, knownZero0=0x{gpuKnownZero0:X16}, aOneWin0=0x{gpuAOneWin0:X16}, aZeroWin0=0x{gpuAZeroWin0:X16}, carryMin={gpuCarryMin}, carryMax={gpuCarryMax}, maxAllowedA={gpuMaxAllowedA}, maxFixed={gpuMaxFixed}, windowSize={gpuWindowSize}, aWordCount={gpuAWordCount}, maxKnownA={gpuMaxKnownA}, maxFixedPrefilter={gpuMaxFixedPrefilter}, preKnownOne0=0x{preKnownOne0:X16}, preKnownZero0=0x{preKnownZero0:X16}, preAOneWin0=0x{preAOneWin0:X16}, preAZeroWin0=0x{preAZeroWin0:X16}, preCarryMin={preCarryMin}, preCarryMax={preCarryMax}, preSegmentBase={preSegmentBase}, preWindowColumn={preWindowColumn}, offsetCount={gpuOffsetCount}, maxOffset={gpuMaxOffset}, firstForced={gpuFirstForced}, firstRemaining={gpuFirstRemaining}, firstQWordsEff={gpuFirstQWordsEff}, firstQLastEff=0x{gpuFirstQLastEff:X16}, firstCarryMin={gpuFirstCarryMin}, firstCarryMax={gpuFirstCarryMax}, gpuQ={gpuQ}, gpuInv={gpuInv}, gpuALow={gpuALow}"); - } + #if DETAILED_LOG + if (prime == 138000001UL) + { + ulong[] debugWords = new ulong[debugWordsPerSlot]; + debugView.SubView(i * debugWordsPerSlot, debugWordsPerSlot).CopyToCPU(stream, debugWords); + BigInteger gpuQ = ReadBigIntegerFromWords(debugWords.AsSpan(0, 16)); + BigInteger gpuInv = ReadBigIntegerFromWords(debugWords.AsSpan(16, 16)); + BigInteger gpuALow = ReadBigIntegerFromWords(debugWords.AsSpan(32, 16)); + ulong gpuQBits = debugWords[48]; + ulong gpuQWordCount = debugWords[49]; + ulong gpuQMask0 = debugWords[50]; + ulong gpuQMask1 = debugWords[51]; + ulong gpuKnownOne0 = debugWords[52]; + ulong gpuKnownZero0 = debugWords[53]; + ulong gpuAOneWin0 = debugWords[54]; + ulong gpuAZeroWin0 = debugWords[55]; + long gpuCarryMin = unchecked((long)debugWords[56]); + long gpuCarryMax = unchecked((long)debugWords[57]); + long gpuMaxAllowedA = unchecked((long)debugWords[58]); + long gpuMaxFixed = unchecked((long)debugWords[59]); + long gpuWindowSize = unchecked((long)debugWords[60]); + long gpuAWordCount = unchecked((long)debugWords[61]); + long gpuMaxKnownA = unchecked((long)debugWords[62]); + long gpuMaxFixedPrefilter = unchecked((long)debugWords[63]); + ulong preKnownOne0 = debugWords[64]; + ulong preKnownZero0 = debugWords[65]; + ulong preAOneWin0 = debugWords[66]; + ulong preAZeroWin0 = debugWords[67]; + long preCarryMin = unchecked((long)debugWords[68]); + long preCarryMax = unchecked((long)debugWords[69]); + long preSegmentBase = unchecked((long)debugWords[70]); + long preWindowColumn = unchecked((long)debugWords[71]); + long gpuOffsetCount = unchecked((long)debugWords[72]); + long gpuMaxOffset = unchecked((long)debugWords[73]); + long gpuFirstForced = unchecked((long)debugWords[74]); + long gpuFirstRemaining = unchecked((long)debugWords[75]); + long gpuFirstQWordsEff = unchecked((long)debugWords[76]); + ulong gpuFirstQLastEff = debugWords[77]; + long gpuFirstCarryMin = unchecked((long)debugWords[78]); + long gpuFirstCarryMax = unchecked((long)debugWords[79]); + throw new InvalidOperationException($"BitContradiction GPU debug for p={prime}, q={candidateQ}, k={candidateK}, qBits={gpuQBits}, qWordCount={gpuQWordCount}, qMask0=0x{gpuQMask0:X16}, qMask1=0x{gpuQMask1:X16}, knownOne0=0x{gpuKnownOne0:X16}, knownZero0=0x{gpuKnownZero0:X16}, aOneWin0=0x{gpuAOneWin0:X16}, aZeroWin0=0x{gpuAZeroWin0:X16}, carryMin={gpuCarryMin}, carryMax={gpuCarryMax}, maxAllowedA={gpuMaxAllowedA}, maxFixed={gpuMaxFixed}, windowSize={gpuWindowSize}, aWordCount={gpuAWordCount}, maxKnownA={gpuMaxKnownA}, maxFixedPrefilter={gpuMaxFixedPrefilter}, preKnownOne0=0x{preKnownOne0:X16}, preKnownZero0=0x{preKnownZero0:X16}, preAOneWin0=0x{preAOneWin0:X16}, preAZeroWin0=0x{preAZeroWin0:X16}, preCarryMin={preCarryMin}, preCarryMax={preCarryMax}, preSegmentBase={preSegmentBase}, preWindowColumn={preWindowColumn}, offsetCount={gpuOffsetCount}, maxOffset={gpuMaxOffset}, firstForced={gpuFirstForced}, firstRemaining={gpuFirstRemaining}, firstQWordsEff={gpuFirstQWordsEff}, firstQLastEff=0x{gpuFirstQLastEff:X16}, firstCarryMin={gpuFirstCarryMin}, firstCarryMax={gpuFirstCarryMax}, gpuQ={gpuQ}, gpuInv={gpuInv}, gpuALow={gpuALow}"); + } + #endif foundDivisor = candidateQ; AcceleratorStreamPool.Return(gpu.AcceleratorIndex, stream); diff --git a/PerfectNumbers.Core/Gpu/GpuKernelPool.cs b/PerfectNumbers.Core/Gpu/GpuKernelPool.cs index 9eef28e..d3dfe41 100644 --- a/PerfectNumbers.Core/Gpu/GpuKernelPool.cs +++ b/PerfectNumbers.Core/Gpu/GpuKernelPool.cs @@ -125,13 +125,20 @@ public class GpuKernelPool { var loaded = accelerator.LoadAutoGroupedStreamKernel< Index1D, ulong, ArrayView1D, int, ArrayView1D, - ArrayView1D, ArrayView1D, ArrayView1D + #if DETAILED_LOG + ArrayView1D, + #endif + ArrayView1D, ArrayView1D >(BitContradictionKernels.BitContradictionKernelScanWithDeltaCache64); var kernel = KernelUtil.GetKernel(loaded); - return kernel.CreateLauncherDelegate, int, ArrayView1D, - ArrayView1D, ArrayView1D, ArrayView1D>>(); + return kernel.CreateLauncherDelegate, int, + ArrayView1D, + #if DETAILED_LOG + ArrayView1D, + #endif + ArrayView1D, ArrayView1D>>(); }); } diff --git a/PerfectNumbers.Core/Gpu/KernelContainer.cs b/PerfectNumbers.Core/Gpu/KernelContainer.cs index 4053aaa..b81086f 100644 --- a/PerfectNumbers.Core/Gpu/KernelContainer.cs +++ b/PerfectNumbers.Core/Gpu/KernelContainer.cs @@ -23,7 +23,10 @@ public sealed class KernelContainer ResidueAutomatonArgs, ArrayView, ArrayView1D>? Pow2ModOrder; public Action, int, ArrayView1D, - ArrayView1D, ArrayView1D, ArrayView1D>? BitContradiction; + #if DETAILED_LOG + ArrayView1D, + #endif + ArrayView1D, ArrayView1D>? BitContradiction; public Kernel? SmallPrimeFactor; diff --git a/PerfectNumbers.Core/Gpu/Kernels/BitContradictionKernels.cs b/PerfectNumbers.Core/Gpu/Kernels/BitContradictionKernels.cs index bf4da07..30d767a 100644 --- a/PerfectNumbers.Core/Gpu/Kernels/BitContradictionKernels.cs +++ b/PerfectNumbers.Core/Gpu/Kernels/BitContradictionKernels.cs @@ -57,17 +57,21 @@ internal static class BitContradictionKernels ArrayView1D batchIndexWords, int countQ, ArrayView1D foundOut, +#if DETAILED_LOG ArrayView1D debugOut, +#endif ArrayView1D deltaCacheKeys, ArrayView1D deltaCache) { - int slot = index; - foundOut[slot] = -1; + // firstQWordsEff is reused as slot here to limit registry pressure + int firstQWordsEff = index; + foundOut[firstQWordsEff] = -1; - int keyBase = slot * DeltaCacheSlots; - long deltaBase = (long)slot * DeltaCacheSlots * DeltaLength; + int keyBase = firstQWordsEff * DeltaCacheSlots; + long deltaBase = (long)firstQWordsEff * DeltaCacheSlots * DeltaLength; // Initialize keys for this thread to avoid false hits across launches. - for (int i = 0; i < DeltaCacheSlots; i++) + int i; + for (i = 0; i < DeltaCacheSlots; i++) { deltaCacheKeys[keyBase + i] = int.MinValue; } @@ -77,17 +81,17 @@ internal static class BitContradictionKernels return; } - int localCountQ = countQ; ulong twoP = exponent << 1; - int startOffset = slot * MaxQWordCount; + // firstQWordsEff is reused as startOffset here to limit registry pressure + firstQWordsEff *= MaxQWordCount; + int localCountQ = countQ; var baseKWords = LocalMemory.Allocate(MaxQWordCount); var kWords = LocalMemory.Allocate(MaxQWordCount); var qWords = LocalMemory.Allocate(MaxQWordCount); var qOffsets = LocalMemory.Allocate(MaxQOffsets); var qPrefix = LocalMemory.Allocate(ForcedALowWords); - var reversePrefix = LocalMemory.Allocate(ForcedALowWords); - var prefixMask = LocalMemory.Allocate(ForcedALowWords); + var invWords = LocalMemory.Allocate(ForcedALowWords); var tmpWords = LocalMemory.Allocate(ForcedALowWords); var tmpWords2 = LocalMemory.Allocate(ForcedALowWords); @@ -105,9 +109,9 @@ internal static class BitContradictionKernels var lo1 = LocalMemory.Allocate(IntervalBufferLength); var hi1 = LocalMemory.Allocate(IntervalBufferLength); - for (int i = 0; i < MaxQWordCount; i++) + for (i = 0; i < MaxQWordCount; i++) { - baseKWords[i] = batchIndexWords[startOffset + i]; + baseKWords[i] = batchIndexWords[firstQWordsEff + i]; } if (MultiplyWordsByUInt64(baseKWords, kWords, MaxQWordCount, (ulong)BatchCount) != 0UL) @@ -163,7 +167,6 @@ internal static class BitContradictionKernels int qWordCount = (qBitLen + 63) >> 6; int offsetCount; - int maxOffsetValue = 0; if (!BuildQOneOffsetsWords(qWords, qWordCount, qOffsets, out offsetCount)) { continue; @@ -171,10 +174,10 @@ internal static class BitContradictionKernels if (offsetCount <= 0) { - maxOffsetValue = qOffsets[offsetCount - 1]; continue; } + int maxOffsetValue = qOffsets[offsetCount - 1]; int windowSize = qBitLen < ForcedALowBits ? ForcedALowBits : qBitLen; windowSize = (windowSize + 63) & ~63; int aWordCount = (windowSize + 63) >> 6; @@ -193,106 +196,120 @@ internal static class BitContradictionKernels ComputeInverseMod2k(qPrefix, invWords, tmpWords, tmpWords2, ForcedALowWords); NegateMod2k(invWords, aLowWords, ForcedALowWords); - for (int i = 0; i < qWordCount; i++) + for (i = 0; i < qWordCount; i++) { qMaskWords[i] = qWords[i]; } - ulong lastWordMask = (qBitLen & 63) == 0 ? ulong.MaxValue : ((1UL << (qBitLen & 63)) - 1UL); + i = qBitLen & 63; + ulong lastWordMask = i == 0 ? ulong.MaxValue : ((1UL << i) - 1UL); if (qWordCount > 0) { qMaskWords[qWordCount - 1] &= lastWordMask; } + int firstRemaining; + ulong firstQLastEff, wordValue; int maxFixed = maxAllowedA < (ForcedALowBits - 1) ? maxAllowedA : (ForcedALowBits - 1); if (maxFixed >= 0) { - int fullWords = maxFixed >> 6; - int lastBit = maxFixed & 63; - for (int w = 0; w < fullWords; w++) + // firstQWordsEff is reused as fullWords here to limit registry pressure + firstQWordsEff = maxFixed >> 6; + // firstRemaining is reused as lastBit here to limit registry pressure + firstRemaining = maxFixed & 63; + // i is reused as word to limit registry pressure + for (i = 0; i < firstQWordsEff; i++) { - ulong wordValue = aLowWords[w]; - knownOne0[w] = wordValue; - knownZero0[w] = ~wordValue; + wordValue = aLowWords[i]; + knownOne0[i] = wordValue; + knownZero0[i] = ~wordValue; } - if (fullWords < aWordCount) + if (firstQWordsEff < aWordCount) { - ulong wordValue = aLowWords[fullWords]; - ulong mask = lastBit == 63 ? ulong.MaxValue : ((1UL << (lastBit + 1)) - 1UL); - knownOne0[fullWords] = wordValue & mask; - knownZero0[fullWords] = (~wordValue) & mask; + wordValue = aLowWords[firstQWordsEff]; + // firstQLastEff is reused as mask here to limit registry pressure + firstQLastEff = firstRemaining == 63 ? ulong.MaxValue : ((1UL << (firstRemaining + 1)) - 1UL); + knownOne0[firstQWordsEff] = wordValue & firstQLastEff; + knownZero0[firstQWordsEff] = (~wordValue) & firstQLastEff; } } int maxKnownA = maxFixed >= 0 ? maxFixed : -1; int firstForced = -1; - int firstRemaining = -1; - int firstQWordsEff = 0; - ulong firstQLastEff = 0; + firstRemaining = -1; + firstQWordsEff = 0; + firstQLastEff = 0; long firstCarryMin = 0; long firstCarryMax = 0; - int maxFixedPrefilter = 512 - 1; - if (maxFixedPrefilter > maxKnownA) - { - maxFixedPrefilter = maxKnownA; - } + int maxFixedPreFilter = 512 - 1; + maxFixedPreFilter = maxFixedPreFilter > maxKnownA ? maxKnownA : maxFixedPreFilter; + maxFixedPreFilter = maxFixedPreFilter > maxAllowedA ? maxAllowedA : maxFixedPreFilter; - if (maxFixedPrefilter > maxAllowedA) + int column; + int word; + int aIndex; + int forced; + int bitShift; + if (maxFixedPreFilter >= 8) { - maxFixedPrefilter = maxAllowedA; - } - - if (maxFixedPrefilter >= 8) - { - ReverseBits1024(qPrefix, reversePrefix); + // tmpWords is reused for reversePrefix + ReverseBits1024(qPrefix, tmpWords); bool prefixOk = true; CarryRange carryLow = CarryRange.Zero; - for (int column = 0; column <= maxFixedPrefilter; column++) + for (column = 0; column <= maxFixedPreFilter; column++) { - int forced = 0; + forced = 0; if (column <= 1023) { - int shift = 1023 - column; - int wordShift = shift >> 6; - int bitShift = shift & 63; - for (int word = 0; word < ForcedALowWords; word++) + // i is reused as shift here to limit registry pressure + i = 1023 - column; + // word is reused as wordShift here to limit registry pressure + word = i >> 6; + bitShift = i & 63; + // i is reused as word to limit registry pressure + for (i = 0; i < ForcedALowWords; i++) { - int src = word + wordShift; - ulong value = 0UL; - if (src < ForcedALowWords) + // aIndex is reused as src here to limit registry pressure + aIndex = i + word; + wordValue = 0UL; + if (aIndex < ForcedALowWords) { - value = reversePrefix[src] >> bitShift; - if (bitShift != 0 && src + 1 < ForcedALowWords) + wordValue = tmpWords[aIndex] >> bitShift; + if (bitShift != 0 && aIndex + 1 < ForcedALowWords) { - value |= reversePrefix[src + 1] << (64 - bitShift); + wordValue |= tmpWords[aIndex + 1] << (64 - bitShift); } } - prefixMask[word] = value; + // tmpWords2 is reused for prefixMask + tmpWords2[i] = wordValue; } - for (int word = 0; word < ForcedALowWords; word++) + // i is reused as word to limit registry pressure + for (i = 0; i < ForcedALowWords; i++) { - forced += XMath.PopCount(knownOne0[word] & prefixMask[word]); + forced += XMath.PopCount(knownOne0[i] & tmpWords2[i]); } } else { - for (int i = 0; i < offsetCount; i++) + for (i = 0; i < offsetCount; i++) { - int t = qOffsets[i]; - if (t > column) + // bitShift is reused as t here to limit registry pressure + bitShift = qOffsets[i]; + if (bitShift > column) { break; } - int aIndex = column - t; - int word = aIndex >> 6; - ulong mask = 1UL << (aIndex & 63); - if ((knownOne0[word] & mask) != 0UL) + aIndex = column - bitShift; + word = aIndex >> 6; + // wordValue is reused as mask here to limit registry pressure + wordValue = 1UL << (aIndex & 63); + if ((knownOne0[word] & wordValue) != 0UL) { forced++; } @@ -312,45 +329,48 @@ internal static class BitContradictionKernels } } - int segmentBase = 0; - int windowColumn = 0; + // word is reused as segmentBase from here to limit registry pressure + word = 0; + // column is reused as windowColumn from here to limit registry pressure + column = 0; - int initialState = GetAKnownStateRowAware(0, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); - if (initialState == 1) - { - aOneWin[0] |= 1UL; - } - else if (initialState == 2) - { - aZeroWin[0] |= 1UL; - } + // aIndex is reused as initialState from here to limit registry pressure + aIndex = GetAKnownStateRowAware(0, word, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); + aOneWin[0] |= aIndex == 1 ? 1UL : 0UL; + aZeroWin[0] |= aIndex == 2 ? 1UL : 0UL; if (!TryRunHighBitAndBorrowPrefiltersCombinedGpu(qOffsets, offsetCount, pLong, maxAllowedA, lo0, hi0, lo1, hi1)) { continue; } - int unknown = offsetCount; - int cacheSlot = unknown & (DeltaCacheSlots - 1); - int keyIndex = keyBase + cacheSlot; - int cachedKey = deltaCacheKeys[keyIndex]; - int cacheSlotBase = (int)(deltaBase + (long)cacheSlot * DeltaLength); - var delta8 = deltaCache.SubView(cacheSlotBase, DeltaLength); - if (cachedKey != unknown) + // forced is reused as unknown from here to limit registry pressure + forced = offsetCount; + // i is reused as cacheSlot from here to limit registry pressure + i = forced & (DeltaCacheSlots - 1); + // bitShift is reused as keyIndex from here to limit registry pressure + bitShift = keyBase + i; + // i is reused as cacheSlotBase from here to limit registry pressure + i = (int)(deltaBase + (long)i * DeltaLength); + var delta8 = deltaCache.SubView(i, DeltaLength); + if (deltaCacheKeys[bitShift] != forced) { - BuildStableUnknownDelta8(unknown, delta8); - deltaCacheKeys[keyIndex] = unknown; + BuildStableUnknownDelta8(forced, delta8); + deltaCacheKeys[bitShift] = forced; } CarryRange carry = CarryRange.Zero; - ulong preKnownOne0 = knownOne0[0]; - ulong preKnownZero0 = knownZero0[0]; - ulong preAOneWin0 = aOneWin[0]; - ulong preAZeroWin0 = aZeroWin[0]; - long preCarryMin = 0L; - long preCarryMax = 0L; - int preSegmentBase = segmentBase; - int preWindowColumn = windowColumn; + #if DETAILED_LOG + int preWindowColumn = windowColumn; + int preSegmentBase = segmentBase; + ulong preKnownZero0 = knownZero0[0]; + ulong preKnownOne0 = knownOne0[0]; + ulong preAOneWin0 = aOneWin[0]; + ulong preAZeroWin0 = aZeroWin[0]; + long preCarryMin = 0L; + long preCarryMax = 0L; + #endif + bool ok = TryProcessBottomUpBlockRowAwareGpu( qOffsets, offsetCount, @@ -362,7 +382,7 @@ internal static class BitContradictionKernels pLong, ref carry, ref maxKnownA, - ref segmentBase, + ref word, knownOne0, knownOne1, knownZero0, @@ -371,7 +391,7 @@ internal static class BitContradictionKernels aWordCount, aOneWin, aZeroWin, - ref windowColumn, + ref column, delta8, offsetCount, ref firstForced, @@ -381,57 +401,59 @@ internal static class BitContradictionKernels ref firstCarryMin, ref firstCarryMax); - if (ok) - { - int debugBase = slot * DebugWordCountPerSlot; - for (int w = 0; w < ForcedALowWords; w++) - { - debugOut[debugBase + w] = qWords[w]; - } - for (int w = 0; w < ForcedALowWords; w++) - { - debugOut[debugBase + ForcedALowWords + w] = invWords[w]; - } - for (int w = 0; w < ForcedALowWords; w++) - { - debugOut[debugBase + (ForcedALowWords * 2) + w] = aLowWords[w]; - } - debugOut[debugBase + (ForcedALowWords * 3)] = (ulong)qBitLen; - int extraBase = debugBase + (ForcedALowWords * 3) + 1; - debugOut[extraBase] = (ulong)qWordCount; - debugOut[extraBase + 1] = qMaskWords[0]; - debugOut[extraBase + 2] = qMaskWords[1]; - debugOut[extraBase + 3] = knownOne0[0]; - debugOut[extraBase + 4] = knownZero0[0]; - debugOut[extraBase + 5] = aOneWin[0]; - debugOut[extraBase + 6] = aZeroWin[0]; - debugOut[extraBase + 7] = (ulong)carry.Min; - debugOut[extraBase + 8] = (ulong)carry.Max; - debugOut[extraBase + 9] = (ulong)maxAllowedA; - debugOut[extraBase + 10] = (ulong)maxFixed; - debugOut[extraBase + 11] = (ulong)windowSize; - debugOut[extraBase + 12] = (ulong)aWordCount; - debugOut[extraBase + 13] = (ulong)maxKnownA; - debugOut[extraBase + 14] = (ulong)maxFixedPrefilter; - debugOut[extraBase + 15] = preKnownOne0; - debugOut[extraBase + 16] = preKnownZero0; - debugOut[extraBase + 17] = preAOneWin0; - debugOut[extraBase + 18] = preAZeroWin0; - debugOut[extraBase + 19] = (ulong)preCarryMin; - debugOut[extraBase + 20] = (ulong)preCarryMax; - debugOut[extraBase + 21] = (ulong)preSegmentBase; - debugOut[extraBase + 22] = (ulong)preWindowColumn; - debugOut[extraBase + 23] = (ulong)offsetCount; - debugOut[extraBase + 24] = (ulong)maxOffsetValue; - debugOut[extraBase + 25] = (ulong)firstForced; - debugOut[extraBase + 26] = (ulong)firstRemaining; - debugOut[extraBase + 27] = (ulong)firstQWordsEff; - debugOut[extraBase + 28] = firstQLastEff; - debugOut[extraBase + 29] = (ulong)firstCarryMin; - debugOut[extraBase + 30] = (ulong)firstCarryMax; - foundOut[slot] = batch; - return; - } + #if DETAILED_LOG + if (ok) + { + int debugBase = slot * DebugWordCountPerSlot; + for (int w = 0; w < ForcedALowWords; w++) + { + debugOut[debugBase + w] = qWords[w]; + } + for (int w = 0; w < ForcedALowWords; w++) + { + debugOut[debugBase + ForcedALowWords + w] = invWords[w]; + } + for (int w = 0; w < ForcedALowWords; w++) + { + debugOut[debugBase + (ForcedALowWords * 2) + w] = aLowWords[w]; + } + debugOut[debugBase + (ForcedALowWords * 3)] = (ulong)qBitLen; + int extraBase = debugBase + (ForcedALowWords * 3) + 1; + debugOut[extraBase] = (ulong)qWordCount; + debugOut[extraBase + 1] = qMaskWords[0]; + debugOut[extraBase + 2] = qMaskWords[1]; + debugOut[extraBase + 3] = knownOne0[0]; + debugOut[extraBase + 4] = knownZero0[0]; + debugOut[extraBase + 5] = aOneWin[0]; + debugOut[extraBase + 6] = aZeroWin[0]; + debugOut[extraBase + 7] = (ulong)carry.Min; + debugOut[extraBase + 8] = (ulong)carry.Max; + debugOut[extraBase + 9] = (ulong)maxAllowedA; + debugOut[extraBase + 10] = (ulong)maxFixed; + debugOut[extraBase + 11] = (ulong)windowSize; + debugOut[extraBase + 12] = (ulong)aWordCount; + debugOut[extraBase + 13] = (ulong)maxKnownA; + debugOut[extraBase + 14] = (ulong)maxFixedPrefilter; + debugOut[extraBase + 15] = preKnownOne0; + debugOut[extraBase + 16] = preKnownZero0; + debugOut[extraBase + 17] = preAOneWin0; + debugOut[extraBase + 18] = preAZeroWin0; + debugOut[extraBase + 19] = (ulong)preCarryMin; + debugOut[extraBase + 20] = (ulong)preCarryMax; + debugOut[extraBase + 21] = (ulong)preSegmentBase; + debugOut[extraBase + 22] = (ulong)preWindowColumn; + debugOut[extraBase + 23] = (ulong)offsetCount; + debugOut[extraBase + 24] = (ulong)maxOffsetValue; + debugOut[extraBase + 25] = (ulong)firstForced; + debugOut[extraBase + 26] = (ulong)firstRemaining; + debugOut[extraBase + 27] = (ulong)firstQWordsEff; + debugOut[extraBase + 28] = firstQLastEff; + debugOut[extraBase + 29] = (ulong)firstCarryMin; + debugOut[extraBase + 30] = (ulong)firstCarryMax; + foundOut[slot] = batch; + return; + } + #endif } } @@ -460,8 +482,8 @@ internal static class BitContradictionKernels var qWords = LocalMemory.Allocate(MaxQWordCount); var qOffsets = LocalMemory.Allocate(MaxQOffsets); var qPrefix = LocalMemory.Allocate(ForcedALowWords); - var reversePrefix = LocalMemory.Allocate(ForcedALowWords); - var prefixMask = LocalMemory.Allocate(ForcedALowWords); + + var invWords = LocalMemory.Allocate(ForcedALowWords); var tmpWords = LocalMemory.Allocate(ForcedALowWords); var tmpWords2 = LocalMemory.Allocate(ForcedALowWords); @@ -610,19 +632,13 @@ internal static class BitContradictionKernels long firstCarryMax = 0; int maxFixedPrefilter = 512 - 1; - if (maxFixedPrefilter > maxKnownA) - { - maxFixedPrefilter = maxKnownA; - } - - if (maxFixedPrefilter > maxAllowedA) - { - maxFixedPrefilter = maxAllowedA; - } + maxFixedPrefilter = maxFixedPrefilter > maxKnownA ? maxKnownA : maxFixedPrefilter; + maxFixedPrefilter = maxFixedPrefilter > maxAllowedA ? maxAllowedA : maxFixedPrefilter; if (maxFixedPrefilter >= 8) { - ReverseBits1024(qPrefix, reversePrefix); + // tmpWords is reused for reversePrefix + ReverseBits1024(qPrefix, tmpWords); bool prefixOk = true; CarryRange carryLow = CarryRange.Zero; @@ -640,19 +656,20 @@ internal static class BitContradictionKernels ulong value = 0UL; if (src < ForcedALowWords) { - value = reversePrefix[src] >> bitShift; + value = tmpWords[src] >> bitShift; if (bitShift != 0 && src + 1 < ForcedALowWords) { - value |= reversePrefix[src + 1] << (64 - bitShift); + value |= tmpWords[src + 1] << (64 - bitShift); } } - prefixMask[word] = value; + // tmpWords2 is reused for prefixMask + tmpWords2[word] = value; } for (int word = 0; word < ForcedALowWords; word++) { - forced += XMath.PopCount(knownOne0[word] & prefixMask[word]); + forced += XMath.PopCount(knownOne0[word] & tmpWords2[word]); } } else @@ -692,14 +709,8 @@ internal static class BitContradictionKernels int windowColumn = 0; int initialState = GetAKnownStateRowAware(0, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); - if (initialState == 1) - { - aOneWin[0] |= 1UL; - } - else if (initialState == 2) - { - aZeroWin[0] |= 1UL; - } + aOneWin[0] |= initialState == 1 ? 1UL : 0UL; + aZeroWin[0] |= initialState == 2 ? 1UL : 0UL; if (!TryRunHighBitAndBorrowPrefiltersCombinedGpu(qOffsets, offsetCount, pLong, maxAllowedA, lo0, hi0, lo1, hi1)) { @@ -1106,20 +1117,14 @@ internal static class BitContradictionKernels int columnPlusOne = column + 1; int highBitColumns = HighBitCarryPrefilterColumns; - if (highBitColumns > columnPlusOne) - { - highBitColumns = columnPlusOne; - } + highBitColumns = highBitColumns > columnPlusOne ? columnPlusOne : highBitColumns; int borrowColumns = 0; bool runBorrowPrefilter = qOffsetCount >= TopDownBorrowMinUnknown; if (runBorrowPrefilter) { borrowColumns = TopDownBorrowColumns; - if (borrowColumns > columnPlusOne) - { - borrowColumns = columnPlusOne; - } + borrowColumns = borrowColumns > columnPlusOne ? columnPlusOne : borrowColumns; } int sequentialColumns = highBitColumns > borrowColumns ? highBitColumns : borrowColumns; @@ -1140,10 +1145,7 @@ internal static class BitContradictionKernels int step = 0; int upperT; int lowerT = column - maxAllowedA; - if (lowerT < 0) - { - lowerT = 0; - } + lowerT = lowerT < 0 ? 0 : lowerT; FindBounds(qOffsets, qOffsetCount, lowerT, column, out lowerT, out upperT); int n = upperT - lowerT; @@ -1172,7 +1174,7 @@ internal static class BitContradictionKernels long rMax = outHi[i]; if (rMax < 0) continue; - if (rMin < 0) rMin = 0; + rMin = rMin < 0 ? 0 : rMin; if (rMax < rMin) continue; const long TwicePlusOneOverflowLimit = (long.MaxValue - 1) >> 1; @@ -1250,10 +1252,7 @@ internal static class BitContradictionKernels } lowerT = column - maxAllowedA; - if (lowerT < 0) - { - lowerT = 0; - } + lowerT = lowerT < 0 ? 0 : lowerT; FindBounds(qOffsets, qOffsetCount, lowerT, column, out lowerT, out upperT); n = upperT - lowerT; @@ -1273,7 +1272,7 @@ internal static class BitContradictionKernels return; } - if (lo < 0) lo = 0; + lo = lo < 0 ? 0 : lo; if (hi < 0) return; if (count == 0) @@ -1287,11 +1286,7 @@ internal static class BitContradictionKernels long lastHi = hiArr[count - 1]; if (lo <= lastHi + 1) { - if (hi > lastHi) - { - hiArr[count - 1] = hi; - } - + hiArr[count - 1] = hi > lastHi ? hi : lastHi; return; } @@ -1387,18 +1382,9 @@ internal static class BitContradictionKernels { ulong minuend = i == 0 ? 2UL : 0UL; ulong sub = value[i]; - ulong res; - if (borrow == 0UL) - { - res = minuend - sub; - borrow = minuend < sub ? 1UL : 0UL; - } - else - { - res = minuend - sub - 1UL; - borrow = minuend <= sub ? 1UL : 0UL; - } - + bool hasBorrow = borrow != 0UL; + ulong res = hasBorrow ? (minuend - sub - 1UL) : (minuend - sub); + borrow = hasBorrow ? (minuend <= sub ? 1UL : 0UL) : (minuend < sub ? 1UL : 0UL); result[i] = res; } } @@ -1610,16 +1596,8 @@ internal static class BitContradictionKernels rel = aIndex - segmentBase; word = rel >> 6; mask = 1UL << (rel & 63); - if (isOne) - { - knownOne0[word] |= mask; - knownZero0[word] &= ~mask; - } - else - { - knownZero0[word] |= mask; - knownOne0[word] &= ~mask; - } + knownOne0[word] = isOne ? (knownOne0[word] | mask) : (knownOne0[word] & ~mask); + knownZero0[word] = isOne ? (knownZero0[word] & ~mask) : (knownZero0[word] | mask); return; } @@ -1630,16 +1608,8 @@ internal static class BitContradictionKernels rel = aIndex - segment1Start; word = rel >> 6; mask = 1UL << (rel & 63); - if (isOne) - { - knownOne1[word] |= mask; - knownZero1[word] &= ~mask; - } - else - { - knownZero1[word] |= mask; - knownOne1[word] &= ~mask; - } + knownOne1[word] = isOne ? (knownOne1[word] | mask) : (knownOne1[word] & ~mask); + knownZero1[word] = isOne ? (knownZero1[word] & ~mask) : (knownZero1[word] | mask); } } @@ -1818,14 +1788,9 @@ internal static class BitContradictionKernels ? 2 : GetAKnownStateRowAware(aIndex, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); - if (state == 1) - { - aOneWin[step >> 6] |= 1UL << (step & 63); - } - else if (state == 2) - { - aZeroWin[step >> 6] |= 1UL << (step & 63); - } + ulong stepMask = 1UL << (step & 63); + aOneWin[step >> 6] |= state == 1 ? stepMask : 0UL; + aZeroWin[step >> 6] |= state == 2 ? stepMask : 0UL; } if (qWordCount > 0) @@ -1954,14 +1919,8 @@ internal static class BitContradictionKernels ? 2 : GetAKnownStateRowAware(nextA, segmentBase, windowSize, knownOne0, knownZero0, knownOne1, knownZero1); - if (nextState == 1) - { - aOneWin[0] |= 1UL; - } - else if (nextState == 2) - { - aZeroWin[0] |= 1UL; - } + aOneWin[0] |= nextState == 1 ? 1UL : 0UL; + aZeroWin[0] |= nextState == 2 ? 1UL : 0UL; startColumn++; windowColumn = startColumn;