DivisorSet.Percentile, Additive and TopDown modes implemented. State files ported to LMDB => FASTER to avoid millions of files in folders.

PerfectNumbers.Core/Persistence/ByDivisorClassModelRepository.cs

@@ -0,0 +1,181 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using FASTER.core;
+using LightningDB;
+using PerfectNumbers.Core.ByDivisor;
+
+namespace PerfectNumbers.Core.Persistence;
+
+/// <summary>
+/// Persists the by-divisor class model using Microsoft.FASTER.Core with optional migration from LightningDB.
+/// </summary>
+public sealed class ByDivisorClassModelRepository : IDisposable
+{
+    private const string ModelKey = "bydivisor_model";
+    private readonly FasterKV<string, string> _store;
+    private readonly IDevice _logDevice;
+    private readonly IDevice _objectLogDevice;
+    private static readonly Encoding Utf8 = Encoding.UTF8;
+
+    public string DatabasePath { get; }
+
+    public static ByDivisorClassModelRepository Open(
+        string directory,
+        string repositoryFileName = "bydivisor_model.faster",
+        string? lmdbPath = null,
+        string? lmdbDirectory = null,
+        bool deleteLmdbAfterMigration = true)
+    {
+        Directory.CreateDirectory(directory);
+        string dbPath = Path.Combine(directory, repositoryFileName);
+        var repo = new ByDivisorClassModelRepository(dbPath);
+        string lmdbSource = lmdbPath ?? Path.ChangeExtension(dbPath, ".lmdb");
+        if (!File.Exists(lmdbSource) && !Directory.Exists(lmdbSource) && !string.IsNullOrEmpty(lmdbDirectory))
+        {
+            lmdbSource = Path.Combine(lmdbDirectory, Path.GetFileName(Path.ChangeExtension(repositoryFileName, ".lmdb")));
+        }
+        repo.MigrateFromLmdb(lmdbSource, deleteLmdbAfterMigration);
+        return repo;
+    }
+
+    private ByDivisorClassModelRepository(string path)
+    {
+        DatabasePath = path;
+        _logDevice = Devices.CreateLogDevice(path + ".log");
+        _objectLogDevice = Devices.CreateLogDevice(path + ".obj.log");
+        _store = new FasterKV<string, string>(
+            1L << 16,
+            new LogSettings
+            {
+                LogDevice = _logDevice,
+                ObjectLogDevice = _objectLogDevice,
+            });
+    }
+
+    public void Save(ByDivisorClassModel model)
+    {
+        string json = JsonSerializer.Serialize(model, new JsonSerializerOptions { WriteIndented = true });
+        using var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+        session.Upsert(ModelKey, json);
+        session.CompletePending(true);
+    }
+
+    public bool TryLoad(out ByDivisorClassModel? model)
+    {
+        using var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+        if (session.Read(ModelKey, out string? json).Found && json is not null)
+        {
+            try
+            {
+                model = JsonSerializer.Deserialize<ByDivisorClassModel>(json);
+                return model is not null;
+            }
+            catch
+            {
+                // ignore deserialization errors
+            }
+        }
+
+        model = null;
+        return false;
+    }
+
+    public void Commit()
+    {
+        _store.Log.FlushAndEvict(wait: true);
+    }
+
+    public void Dispose()
+    {
+        _store.Dispose();
+        _logDevice.Dispose();
+        _objectLogDevice.Dispose();
+    }
+
+    private void MigrateFromLmdb(string lmdbPath, bool deleteAfter)
+    {
+        try
+        {
+            if (!LmdbExists(lmdbPath))
+            {
+                Console.WriteLine($"No LMDB by-divisor model found at '{lmdbPath}', skipping migration.");
+                return;
+            }
+
+            Console.WriteLine($"Migrating by-divisor model from LMDB at '{lmdbPath}' to FASTER...");
+            using var env = new LightningEnvironment(lmdbPath) { MaxDatabases = 2 };
+            env.Open();
+            using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+            var db = tx.OpenDatabase("bydivisor_model", new DatabaseConfiguration { Flags = DatabaseOpenFlags.None });
+            if (tx.TryGet(db, Utf8.GetBytes(ModelKey), out byte[]? jsonBytes) && jsonBytes is not null)
+            {
+                try
+                {
+                    string json = Utf8.GetString(jsonBytes);
+                    var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+                    session.Upsert(ModelKey, json);
+                    session.CompletePending(true);
+                    session.Dispose();
+                }
+                catch
+                {
+                    // ignore migration issues
+                }
+            }
+            _store.Log.FlushAndEvict(wait: true);
+            Console.WriteLine("Migration of by-divisor model to FASTER completed.");
+        }
+        catch
+        {
+            // ignore migration failures
+        }
+
+        if (deleteAfter)
+        {
+            Console.WriteLine($"Starting cleanup of LMDB by-divisor model at '{lmdbPath}'...");
+            try
+            {
+                if (Directory.Exists(lmdbPath))
+                {
+                    Directory.Delete(lmdbPath, recursive: true);
+                }
+                else if (File.Exists(lmdbPath))
+                {
+                    File.Delete(lmdbPath);
+                }
+            }
+            catch
+            {
+                // ignore cleanup failures
+            }
+        }
+    }
+
+    private static bool LmdbExists(string path)
+    {
+        if (Directory.Exists(path) || File.Exists(path))
+        {
+            return true;
+        }
+
+        string candidate = Path.Combine(path, "data.mdb");
+        if (File.Exists(candidate))
+        {
+            return true;
+        }
+
+        string? directory = Path.GetDirectoryName(path);
+        if (!string.IsNullOrEmpty(directory) && Directory.Exists(directory))
+        {
+            string sibling = Path.Combine(directory, "data.mdb");
+            if (File.Exists(sibling))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

PerfectNumbers.Core/Persistence/KStateRepository.cs

@@ -0,0 +1,186 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Numerics;
+using System.Text;
+using FASTER.core;
+using LightningDB;
+
+namespace PerfectNumbers.Core.Persistence;
+
+/// <summary>
+/// Stores last-checked k values (BigInteger) per prime using Microsoft.FASTER.Core.
+/// Supports one-time migration from an existing LightningDB (LMDB) store.
+/// </summary>
+public sealed class KStateRepository : IDisposable
+{
+    private readonly FasterKV<string, string> _store;
+    private readonly IDevice _logDevice;
+    private readonly IDevice _objectLogDevice;
+    private static readonly Encoding Utf8 = Encoding.UTF8;
+
+    public string DatabasePath { get; }
+
+    public static KStateRepository Open(
+        string directory,
+        string repositoryFileName = "kstate.faster",
+        string? lmdbPath = null,
+        string? lmdbDirectory = null,
+        string lmdbDatabaseName = "kstate",
+        bool deleteLmdbAfterMigration = true)
+    {
+        Directory.CreateDirectory(directory);
+        string dbPath = Path.Combine(directory, repositoryFileName);
+        var repo = new KStateRepository(dbPath);
+
+        string lmdbSource = lmdbPath ?? Path.ChangeExtension(dbPath, ".lmdb");
+        if (!File.Exists(lmdbSource) && !Directory.Exists(lmdbSource) && !string.IsNullOrEmpty(lmdbDirectory))
+        {
+            lmdbSource = Path.Combine(lmdbDirectory, Path.GetFileName(Path.ChangeExtension(repositoryFileName, ".lmdb")));
+        }
+
+        repo.MigrateFromLmdb(lmdbSource, lmdbDatabaseName, deleteLmdbAfterMigration);
+        return repo;
+    }
+
+    private KStateRepository(string path)
+    {
+        DatabasePath = path;
+        _logDevice = Devices.CreateLogDevice(path + ".log");
+        _objectLogDevice = Devices.CreateLogDevice(path + ".obj.log");
+        _store = new FasterKV<string, string>(
+            1L << 20,
+            new LogSettings
+            {
+                LogDevice = _logDevice,
+                ObjectLogDevice = _objectLogDevice,
+            });
+    }
+
+    public void Upsert(ulong prime, BigInteger k)
+    {
+        using var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+        string keyString = prime.ToString(CultureInfo.InvariantCulture);
+        string valueString = k.ToString(CultureInfo.InvariantCulture);
+        session.Upsert(keyString, valueString);
+        session.CompletePending(true);
+    }
+
+    public bool TryGet(ulong prime, out BigInteger value)
+    {
+        using var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+        string keyString = prime.ToString(CultureInfo.InvariantCulture);
+        if (session.Read(keyString, out string? stored).Found &&
+            stored is not null &&
+            BigInteger.TryParse(stored, NumberStyles.Integer, CultureInfo.InvariantCulture, out value))
+        {
+            return true;
+        }
+
+        value = BigInteger.Zero;
+        return false;
+    }
+
+    public void SeedFromStates(IEnumerable<(ulong Prime, BigInteger K)> states)
+    {
+        foreach (var (prime, k) in states)
+        {
+            Upsert(prime, k);
+        }
+    }
+
+    public void Commit()
+    {
+        _store.Log.FlushAndEvict(wait: true);
+    }
+
+    public void Dispose()
+    {
+        _store.Dispose();
+        _logDevice.Dispose();
+        _objectLogDevice.Dispose();
+    }
+
+    private void MigrateFromLmdb(string lmdbPath, string databaseName, bool deleteAfter)
+    {
+        try
+        {
+            if (!LmdbExists(lmdbPath))
+            {
+                Console.WriteLine($"No LMDB k-state found at '{lmdbPath}', skipping migration.");
+                return;
+            }
+
+            Console.WriteLine($"Migrating k-state from LMDB at '{lmdbPath}' to FASTER...");
+            using var env = new LightningEnvironment(lmdbPath) { MaxDatabases = 2 };
+            env.Open();
+            using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+            var db = tx.OpenDatabase(databaseName, new DatabaseConfiguration { Flags = DatabaseOpenFlags.None });
+            using var cursor = tx.CreateCursor(db);
+            foreach (var entry in cursor.AsEnumerable())
+            {
+                string keyString = Utf8.GetString(entry.Item1.CopyToNewArray());
+                string valueString = Utf8.GetString(entry.Item2.CopyToNewArray());
+                if (ulong.TryParse(keyString, NumberStyles.None, CultureInfo.InvariantCulture, out ulong prime) &&
+                    BigInteger.TryParse(valueString, NumberStyles.Integer, CultureInfo.InvariantCulture, out BigInteger k) &&
+                    k > BigInteger.Zero)
+                {
+                    Upsert(prime, k);
+                }
+            }
+            _store.Log.FlushAndEvict(wait: true);
+            Console.WriteLine("Migration of k-state to FASTER completed.");
+        }
+        catch
+        {
+            // Ignore migration failures; repo will simply start empty.
+        }
+
+        if (deleteAfter)
+        {
+            Console.WriteLine($"Starting cleanup of LMDB k-state at '{lmdbPath}'...");
+            try
+            {
+                if (Directory.Exists(lmdbPath))
+                {
+                    Directory.Delete(lmdbPath, recursive: true);
+                }
+                else if (File.Exists(lmdbPath))
+                {
+                    File.Delete(lmdbPath);
+                }
+            }
+            catch
+            {
+                // ignore cleanup failures
+            }
+        }
+    }
+
+    private static bool LmdbExists(string path)
+    {
+        if (Directory.Exists(path) || File.Exists(path))
+        {
+            return true;
+        }
+
+        string candidate = Path.Combine(path, "data.mdb");
+        if (File.Exists(candidate))
+        {
+            return true;
+        }
+
+        string? directory = Path.GetDirectoryName(path);
+        if (!string.IsNullOrEmpty(directory) && Directory.Exists(directory))
+        {
+            string sibling = Path.Combine(directory, "data.mdb");
+            if (File.Exists(sibling))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

PerfectNumbers.Core/Persistence/Pow2Minus1StateRepository.cs

@@ -0,0 +1,165 @@
+using System;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using FASTER.core;
+using LightningDB;
+
+namespace PerfectNumbers.Core.Persistence;
+
+/// <summary>
+/// Tracks primes for which the pow2-1 divisor check was completed, backed by Microsoft.FASTER.Core.
+/// Supports one-time migration from LightningDB.
+/// </summary>
+public sealed class Pow2Minus1StateRepository : IDisposable
+{
+    private readonly FasterKV<string, string> _store;
+    private readonly IDevice _logDevice;
+    private readonly IDevice _objectLogDevice;
+    private static readonly Encoding Utf8 = Encoding.UTF8;
+
+    public string DatabasePath { get; }
+
+    public static Pow2Minus1StateRepository Open(
+        string databasePath,
+        string lmdbPath = "pow2minus1.lmdb",
+        string? lmdbDirectory = null,
+        string lmdbDatabaseName = "pow2minus1",
+        bool deleteLmdbAfterMigration = true)
+    {
+        string directory = Path.GetDirectoryName(databasePath) ?? ".";
+        Directory.CreateDirectory(directory);
+
+        var repo = new Pow2Minus1StateRepository(databasePath);
+
+        if (!File.Exists(lmdbPath) && !Directory.Exists(lmdbPath) && !string.IsNullOrEmpty(lmdbDirectory))
+        {
+            lmdbPath = Path.Combine(lmdbDirectory, Path.GetFileName(lmdbPath));
+        }
+
+        repo.MigrateFromLmdb(lmdbPath, lmdbDatabaseName, deleteLmdbAfterMigration);
+        return repo;
+    }
+
+    private Pow2Minus1StateRepository(string path)
+    {
+        DatabasePath = path;
+        _logDevice = Devices.CreateLogDevice(path + ".log");
+        _objectLogDevice = Devices.CreateLogDevice(path + ".obj.log");
+        _store = new FasterKV<string, string>(
+            1L << 18,
+            new LogSettings
+            {
+                LogDevice = _logDevice,
+                ObjectLogDevice = _objectLogDevice,
+            });
+    }
+
+    public void MarkChecked(ulong prime)
+    {
+        using var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+        string key = prime.ToString(CultureInfo.InvariantCulture);
+        session.Upsert(key, "checked");
+        session.CompletePending(true);
+    }
+
+    public bool IsChecked(ulong prime)
+    {
+        using var session = _store.For(new SimpleFunctions<string, string>()).NewSession<SimpleFunctions<string, string>>();
+        string key = prime.ToString(CultureInfo.InvariantCulture);
+        var status = session.Read(key, out _);
+        session.CompletePending(true);
+        return status.Found;
+    }
+
+    public void Commit()
+    {
+        _store.Log.FlushAndEvict(wait: true);
+    }
+
+    public void Dispose()
+    {
+        _store.Dispose();
+        _logDevice.Dispose();
+        _objectLogDevice.Dispose();
+    }
+
+    private void MigrateFromLmdb(string lmdbPath, string databaseName, bool deleteAfter)
+    {
+        try
+        {
+            if (!LmdbExists(lmdbPath))
+            {
+                Console.WriteLine($"No LMDB pow2minus1 state found at '{lmdbPath}', skipping migration.");
+                return;
+            }
+
+            Console.WriteLine($"Migrating pow2minus1 state from LMDB at '{lmdbPath}' to FASTER...");
+            using var env = new LightningEnvironment(lmdbPath) { MaxDatabases = 2 };
+            env.Open();
+            using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+            var db = tx.OpenDatabase(databaseName, new DatabaseConfiguration { Flags = DatabaseOpenFlags.None });
+            using var cursor = tx.CreateCursor(db);
+            foreach (var entry in cursor.AsEnumerable())
+            {
+                string key = Utf8.GetString(entry.Item1.CopyToNewArray());
+                if (ulong.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out ulong prime))
+                {
+                    MarkChecked(prime);
+                }
+            }
+            _store.Log.FlushAndEvict(wait: true);
+            Console.WriteLine("Migration of pow2minus1 state to FASTER completed.");
+        }
+        catch
+        {
+            // ignore migration issues
+        }
+
+        if (deleteAfter)
+        {
+            Console.WriteLine($"Starting cleanup of LMDB pow2minus1 state at '{lmdbPath}'...");
+            try
+            {
+                if (Directory.Exists(lmdbPath))
+                {
+                    Directory.Delete(lmdbPath, recursive: true);
+                }
+                else if (File.Exists(lmdbPath))
+                {
+                    File.Delete(lmdbPath);
+                }
+            }
+            catch
+            {
+                // ignore cleanup failures
+            }
+        }
+    }
+
+    private static bool LmdbExists(string path)
+    {
+        if (Directory.Exists(path) || File.Exists(path))
+        {
+            return true;
+        }
+
+        string candidate = Path.Combine(path, "data.mdb");
+        if (File.Exists(candidate))
+        {
+            return true;
+        }
+
+        string? directory = Path.GetDirectoryName(path);
+        if (!string.IsNullOrEmpty(directory) && Directory.Exists(directory))
+        {
+            string sibling = Path.Combine(directory, "data.mdb");
+            if (File.Exists(sibling))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}