From 81c9fea6c7a7e2dff0d0d8ee1b3c6568ebdf0d91 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Mon, 26 Oct 2026 18:09:34 -0700 Subject: [PATCH 1/4] feat!: let the engine control SystemConfig defaults Stop pinning a copy of every engine default in this binding. All engine settings now default to None, and only explicitly set values are forwarded to either backend (capi starts from lbug_default_system_config(), pybind from SystemConfig()). Unset settings now follow engine-side default changes automatically. BREAKING CHANGE: Database() settings parameters changed from concrete defaults (e.g. enable_multi_writes=False) to None meaning "engine default"; explicitly passed values behave as before. --- src_cpp/include/py_database.h | 15 +++-- src_cpp/py_database.cpp | 70 +++++++++++++++------ src_py/_lbug_capi.py | 46 ++++++++------ src_py/database.py | 115 +++++++++++++++++++--------------- 4 files changed, 153 insertions(+), 93 deletions(-) diff --git a/src_cpp/include/py_database.h b/src_cpp/include/py_database.h index 6c97232..38d688f 100644 --- a/src_cpp/include/py_database.h +++ b/src_cpp/include/py_database.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "main/lbug.h" #include "main/storage_driver.h" @@ -19,10 +20,16 @@ class PyDatabase { static uint64_t getStorageVersion(); - explicit PyDatabase(const std::string& databasePath, uint64_t bufferPoolSize, - uint64_t maxNumThreads, bool compression, bool readOnly, uint64_t maxDBSize, - bool autoCheckpoint, int64_t checkpointThreshold, bool throwOnWalReplayFailure = true, - bool enableChecksums = true, bool enableMultiWrites = false); + explicit PyDatabase(const std::string& databasePath, + std::optional bufferPoolSize = std::nullopt, + std::optional maxNumThreads = std::nullopt, + std::optional compression = std::nullopt, std::optional readOnly = std::nullopt, + std::optional maxDBSize = std::nullopt, + std::optional autoCheckpoint = std::nullopt, + std::optional checkpointThreshold = std::nullopt, + std::optional throwOnWalReplayFailure = std::nullopt, + std::optional enableChecksums = std::nullopt, + std::optional enableMultiWrites = std::nullopt); ~PyDatabase(); diff --git a/src_cpp/py_database.cpp b/src_cpp/py_database.cpp index 8281610..200200f 100644 --- a/src_cpp/py_database.cpp +++ b/src_cpp/py_database.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "common/exception/runtime.h" #include "extension/extension.h" @@ -13,14 +14,17 @@ using namespace lbug::common; void PyDatabase::initialize(py::handle& m) { py::class_(m, "Database") - .def(py::init(), - py::arg("database_path"), py::arg("buffer_pool_size") = 0, - py::arg("max_num_threads") = 0, py::arg("compression") = true, - py::arg("read_only") = false, py::arg("max_db_size") = -1u, - py::arg("auto_checkpoint") = true, py::arg("checkpoint_threshold") = -1, - py::arg("throw_on_wal_replay_failure") = true, py::arg("enable_checksums") = true, - py::arg("enable_multi_writes") = false) + .def(py::init, std::optional, + std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, + std::optional, std::optional>(), + py::arg("database_path"), py::arg("buffer_pool_size") = py::none(), + py::arg("max_num_threads") = py::none(), py::arg("compression") = py::none(), + py::arg("read_only") = py::none(), py::arg("max_db_size") = py::none(), + py::arg("auto_checkpoint") = py::none(), py::arg("checkpoint_threshold") = py::none(), + py::arg("throw_on_wal_replay_failure") = py::none(), + py::arg("enable_checksums") = py::none(), + py::arg("enable_multi_writes") = py::none()) .def("scan_node_table_as_int64", &PyDatabase::scanNodeTable, py::arg("table_name"), py::arg("prop_name"), py::arg("indices"), py::arg("np_array"), py::arg("num_threads")) @@ -49,18 +53,46 @@ uint64_t PyDatabase::getStorageVersion() { return Version::getStorageVersion(); } -PyDatabase::PyDatabase(const std::string& databasePath, uint64_t bufferPoolSize, - uint64_t maxNumThreads, bool compression, bool readOnly, uint64_t maxDBSize, - bool autoCheckpoint, int64_t checkpointThreshold, bool throwOnWalReplayFailure, - bool enableChecksums, bool enableMultiWrites) { - auto systemConfig = SystemConfig(bufferPoolSize, maxNumThreads, compression, readOnly, - maxDBSize, autoCheckpoint); - if (checkpointThreshold >= 0) { - systemConfig.checkpointThreshold = static_cast(checkpointThreshold); +PyDatabase::PyDatabase(const std::string& databasePath, std::optional bufferPoolSize, + std::optional maxNumThreads, std::optional compression, + std::optional readOnly, std::optional maxDBSize, + std::optional autoCheckpoint, std::optional checkpointThreshold, + std::optional throwOnWalReplayFailure, std::optional enableChecksums, + std::optional enableMultiWrites) { + // Start from the engine defaults and only apply explicitly set settings, + // so engine-side default changes propagate automatically. + auto systemConfig = SystemConfig(); + if (bufferPoolSize.has_value()) { + systemConfig.bufferPoolSize = bufferPoolSize.value(); + } + if (maxNumThreads.has_value()) { + systemConfig.maxNumThreads = maxNumThreads.value(); + } + if (compression.has_value()) { + systemConfig.enableCompression = compression.value(); + } + if (readOnly.has_value()) { + systemConfig.readOnly = readOnly.value(); + } + if (maxDBSize.has_value()) { + systemConfig.maxDBSize = maxDBSize.value(); + } + if (autoCheckpoint.has_value()) { + systemConfig.autoCheckpoint = autoCheckpoint.value(); + } + if (checkpointThreshold.has_value() && checkpointThreshold.value() >= 0) { + systemConfig.checkpointThreshold = + static_cast(checkpointThreshold.value()); + } + if (throwOnWalReplayFailure.has_value()) { + systemConfig.throwOnWalReplayFailure = throwOnWalReplayFailure.value(); + } + if (enableChecksums.has_value()) { + systemConfig.enableChecksums = enableChecksums.value(); + } + if (enableMultiWrites.has_value()) { + systemConfig.enableMultiWrites = enableMultiWrites.value(); } - systemConfig.throwOnWalReplayFailure = throwOnWalReplayFailure; - systemConfig.enableChecksums = enableChecksums; - systemConfig.enableMultiWrites = enableMultiWrites; state = std::make_shared(); state->database = std::make_unique(databasePath, systemConfig); lbug::extension::ExtensionUtils::addTableFunc(*state->database); diff --git a/src_py/_lbug_capi.py b/src_py/_lbug_capi.py index 95806a6..d5f31f0 100644 --- a/src_py/_lbug_capi.py +++ b/src_py/_lbug_capi.py @@ -1197,33 +1197,43 @@ class Database: def __init__( self, database_path: str, - buffer_pool_size: int = 0, - max_num_threads: int = 0, - compression: bool = True, - read_only: bool = False, + buffer_pool_size: int | None = None, + max_num_threads: int | None = None, + compression: bool | None = None, + read_only: bool | None = None, max_db_size: int | None = None, - auto_checkpoint: bool = True, - checkpoint_threshold: int = -1, - throw_on_wal_replay_failure: bool = True, - enable_checksums: bool = True, - enable_multi_writes: bool = False, + auto_checkpoint: bool | None = None, + checkpoint_threshold: int | None = None, + throw_on_wal_replay_failure: bool | None = None, + enable_checksums: bool | None = None, + enable_multi_writes: bool | None = None, ): self._database = _LbugDatabase() + # Start from the engine defaults and only apply explicitly set + # settings, so engine-side default changes propagate automatically. config = _LIB.lbug_default_system_config() - config.buffer_pool_size = buffer_pool_size - config.max_num_threads = max_num_threads - config.enable_compression = compression - config.read_only = read_only + if buffer_pool_size is not None: + config.buffer_pool_size = buffer_pool_size + if max_num_threads is not None: + config.max_num_threads = max_num_threads + if compression is not None: + config.enable_compression = compression + if read_only is not None: + config.read_only = read_only if max_db_size is not None: config.max_db_size = max_db_size - config.auto_checkpoint = auto_checkpoint - if checkpoint_threshold >= 0: + if auto_checkpoint is not None: + config.auto_checkpoint = auto_checkpoint + if checkpoint_threshold is not None and checkpoint_threshold >= 0: config.checkpoint_threshold = checkpoint_threshold - config.throw_on_wal_replay_failure = throw_on_wal_replay_failure - config.enable_checksums = enable_checksums - config.enable_multi_writes = enable_multi_writes + if throw_on_wal_replay_failure is not None: + config.throw_on_wal_replay_failure = throw_on_wal_replay_failure + if enable_checksums is not None: + config.enable_checksums = enable_checksums + if enable_multi_writes is not None: + config.enable_multi_writes = enable_multi_writes state = _LIB.lbug_database_init( database_path.encode("utf-8"), config, ctypes.byref(self._database) diff --git a/src_py/database.py b/src_py/database.py index f0e2025..3734446 100644 --- a/src_py/database.py +++ b/src_py/database.py @@ -34,17 +34,17 @@ def __init__( self, database_path: str | Path | None = None, *, - buffer_pool_size: int = 0, - max_num_threads: int = 0, - compression: bool = True, + buffer_pool_size: int | None = None, + max_num_threads: int | None = None, + compression: bool | None = None, lazy_init: bool = False, - read_only: bool = False, + read_only: bool | None = None, max_db_size: int | None = None, - auto_checkpoint: bool = True, - checkpoint_threshold: int = -1, - throw_on_wal_replay_failure: bool = True, - enable_checksums: bool = True, - enable_multi_writes: bool = False, + auto_checkpoint: bool | None = None, + checkpoint_threshold: int | None = None, + throw_on_wal_replay_failure: bool | None = None, + enable_checksums: bool | None = None, + enable_multi_writes: bool | None = None, backend: str = "auto", ): """ @@ -54,14 +54,17 @@ def __init__( The path to database files. If the path is not specified, or empty, or equal to `:memory:`, the database will be created in memory. - buffer_pool_size : int - The maximum size of buffer pool in bytes. Defaults to ~80% of system memory. + buffer_pool_size : int, optional + The maximum size of buffer pool in bytes. If None (default), the + engine default (~80% of system memory) is used. - max_num_threads : int - The maximum number of threads to use for executing queries. + max_num_threads : int, optional + The maximum number of threads to use for executing queries. If None + (default), the engine default is used. - compression : bool - Enable database compression. + compression : bool, optional + Enable database compression. If None (default), the engine default + is used. lazy_init : bool If True, the database will not be initialized until the first query. @@ -69,13 +72,13 @@ def __init__( when the main process is forked. Default to False. - read_only : bool + read_only : bool, optional If true, the database is opened read-only. No write transactions is allowed on the `Database` object. Multiple read-only `Database` objects can be created with the same database path. However, there cannot be multiple `Database` objects created with the same database path. - Default to False. + If None (default), the engine default is used. max_db_size : int, optional The maximum size of the database in bytes. Note that this is introduced @@ -83,25 +86,28 @@ def __init__( space limit some environment. This will be removed once we implement a better solution later. If not specified, the backend's default is used. - auto_checkpoint: bool + auto_checkpoint: bool, optional If true, the database will automatically checkpoint when the size of - the WAL file exceeds the checkpoint threshold. + the WAL file exceeds the checkpoint threshold. If None (default), + the engine default is used. - checkpoint_threshold: int + checkpoint_threshold: int, optional The threshold of the WAL file size in bytes. When the size of the WAL file exceeds this threshold, the database will checkpoint if autoCheckpoint is true. + If None (default), the engine default is used. - throw_on_wal_replay_failure: bool + throw_on_wal_replay_failure: bool, optional If true, any WAL replaying failure when loading the database will throw an error. Otherwise, Lbug will silently ignore the failure and replay up to where the error - occured. + occured. If None (default), the engine default is used. - enable_checksums: bool + enable_checksums: bool, optional If true, the database will use checksums to detect corruption in the - WAL file. + WAL file. If None (default), the engine default is used. - enable_multi_writes: bool - If true, multiple concurrent write transactions are allowed. Default to False. + enable_multi_writes: bool, optional + If true, multiple concurrent write transactions are allowed. If None + (default), the engine default is used. backend : {"auto", "capi", "pybind"} Backend to use for database/query execution. @@ -213,6 +219,35 @@ def __getstate__(self) -> dict[str, Any]: } return state + def _engine_kwargs(self) -> dict[str, Any]: + """Return only the explicitly set engine settings. + + Unset (None) settings are omitted so the engine default applies, + instead of this binding pinning its own copy of the default. + """ + kwargs: dict[str, Any] = {} + if self.buffer_pool_size is not None: + kwargs["buffer_pool_size"] = self.buffer_pool_size + if self.max_num_threads is not None: + kwargs["max_num_threads"] = self.max_num_threads + if self.compression is not None: + kwargs["compression"] = self.compression + if self.read_only is not None: + kwargs["read_only"] = self.read_only + if self.max_db_size is not None: + kwargs["max_db_size"] = self.max_db_size + if self.auto_checkpoint is not None: + kwargs["auto_checkpoint"] = self.auto_checkpoint + if self.checkpoint_threshold is not None: + kwargs["checkpoint_threshold"] = self.checkpoint_threshold + if self.throw_on_wal_replay_failure is not None: + kwargs["throw_on_wal_replay_failure"] = self.throw_on_wal_replay_failure + if self.enable_checksums is not None: + kwargs["enable_checksums"] = self.enable_checksums + if self.enable_multi_writes is not None: + kwargs["enable_multi_writes"] = self.enable_multi_writes + return kwargs + def init_database(self) -> None: """Initialize the database.""" self.check_for_database_close() @@ -221,17 +256,7 @@ def init_database(self) -> None: self._database = self.init_pybind_database() else: self._database = get_capi_module().Database( - self.database_path, - self.buffer_pool_size, - self.max_num_threads, - self.compression, - self.read_only, - self.max_db_size, - self.auto_checkpoint, - self.checkpoint_threshold, - self.throw_on_wal_replay_failure, - self.enable_checksums, - self.enable_multi_writes, + self.database_path, **self._engine_kwargs() ) def init_pybind_database(self) -> Any | None: @@ -241,21 +266,7 @@ def init_pybind_database(self) -> Any | None: if pybind_module is None: return None if self._pybind_database is None: - kwargs = { - "database_path": self.database_path, - "buffer_pool_size": self.buffer_pool_size, - "max_num_threads": self.max_num_threads, - "compression": self.compression, - "read_only": self.read_only, - "auto_checkpoint": self.auto_checkpoint, - "checkpoint_threshold": self.checkpoint_threshold, - "throw_on_wal_replay_failure": self.throw_on_wal_replay_failure, - "enable_checksums": self.enable_checksums, - "enable_multi_writes": self.enable_multi_writes, - } - - if self.max_db_size is not None: - kwargs["max_db_size"] = self.max_db_size + kwargs = {"database_path": self.database_path, **self._engine_kwargs()} self._pybind_database = pybind_module.Database(**kwargs) From 337dff7c284040db9f2f07fae2e8d0b9d8c96c80 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Mon, 26 Oct 2026 18:32:37 -0700 Subject: [PATCH 2/4] test: opt into throwing WAL replay failures in checksums test The binding no longer pins throw_on_wal_replay_failure=True; unset settings follow the engine default (false: silently ignore). This test verifies that a checksum mismatch raises, so it must now opt into throwing explicitly. --- test/test_database.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_database.py b/test/test_database.py index 1da2730..be5449d 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -177,9 +177,15 @@ def test_database_enable_checksums_config(tmp_path: Path) -> None: conn.execute("call force_checkpoint_on_close=false") conn.execute("create node table test(id int64 primary key)") - # running again with enable_checksums=False should give an error + # running again with enable_checksums=False should give an error, as long as + # WAL replay failures are configured to throw (the engine default is to + # silently ignore them, and the binding no longer overrides that default). with pytest.raises(RuntimeError) as check: - db = lb.Database(database_path=database_path, enable_checksums=False) + db = lb.Database( + database_path=database_path, + enable_checksums=False, + throw_on_wal_replay_failure=True, + ) def test_database_close_order() -> None: From f3c76f9e81df6408892ff285b848a8fa67e83843 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Thu, 29 Oct 2026 15:38:28 -0700 Subject: [PATCH 3/4] test(database): cover torn WAL tail on the engine default Bring over the torn-tail test and replay docstring from Refs #75, adapted to engine-controlled defaults: a strict open still throws on the torn tail while the default open keeps the first insert. The docstring now says recovery replays up to the last complete commit and truncates there, dropping the torn tail and anything after a damaged record. --- src_py/database.py | 10 +++++++--- test/test_database.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src_py/database.py b/src_py/database.py index 3734446..684e436 100644 --- a/src_py/database.py +++ b/src_py/database.py @@ -98,8 +98,11 @@ def __init__( throw_on_wal_replay_failure: bool, optional If true, any WAL replaying failure when loading the database will throw an error. - Otherwise, Lbug will silently ignore the failure and replay up to where the error - occured. If None (default), the engine default is used. + Otherwise (the default, as in the engine), Lbug replays the WAL up to the last + complete commit before the first record it cannot read and truncates the file + there. This drops the torn tail a crash leaves while a commit is being written, + and anything recorded after a damaged record. + If None (default), the engine default is used. enable_checksums: bool, optional If true, the database will use checksums to detect corruption in the @@ -220,7 +223,8 @@ def __getstate__(self) -> dict[str, Any]: return state def _engine_kwargs(self) -> dict[str, Any]: - """Return only the explicitly set engine settings. + """ + Return only the explicitly set engine settings. Unset (None) settings are omitted so the engine default applies, instead of this binding pinning its own copy of the default. diff --git a/test/test_database.py b/test/test_database.py index be5449d..960b351 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -164,6 +164,34 @@ def test_database_throw_on_wal_replay_failure_config(tmp_path: Path) -> None: assert result.get_next()[0] == 1 +def test_database_opens_past_a_torn_wal_tail_by_default(tmp_path: Path) -> None: + database_path = get_db_file_path(tmp_path) + with ( + lb.Database(database_path=database_path, auto_checkpoint=False) as db, + lb.Connection(db) as conn, + ): + conn.execute("CALL force_checkpoint_on_close=false") + conn.execute("CREATE NODE TABLE T(id INT64 PRIMARY KEY)") + conn.execute("CREATE (:T {id: 1})") + conn.execute("CREATE (:T {id: 2})") + # A crash while the last commit is written leaves its records cut short. + wal_file_path = Path(str(database_path) + ".wal") + with Path.open(wal_file_path, "r+b") as wal_file: + wal_file.truncate(wal_file_path.stat().st_size - 8) + + with pytest.raises( + RuntimeError, + match=r"Checksum verification failed|Reading past the end of the file", + ): + lb.Database(database_path=database_path, throw_on_wal_replay_failure=True) + with ( + lb.Database(database_path=database_path) as db, + lb.Connection(db) as conn, + conn.execute("MATCH (t:T) RETURN t.id") as result, + ): + assert result.get_all() == [[1]] + + def test_database_enable_checksums_config(tmp_path: Path) -> None: database_path = get_db_file_path(tmp_path) # first construct database file with enable_checksums=True From 237ecbb54a1217d2de6b19ef3f18f925048e0f33 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Thu, 29 Oct 2026 16:45:17 -0700 Subject: [PATCH 4/4] test(database): exercise the new lenient default in checksums test The engine build CI uses now carries the checksum-mismatch fix, so a checksum-setting mismatch throws even on the lenient default. Drop the temporary explicit throw_on_wal_replay_failure opt-in; the test now covers the new default. --- test/test_database.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test/test_database.py b/test/test_database.py index 960b351..1ac0685 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -205,15 +205,10 @@ def test_database_enable_checksums_config(tmp_path: Path) -> None: conn.execute("call force_checkpoint_on_close=false") conn.execute("create node table test(id int64 primary key)") - # running again with enable_checksums=False should give an error, as long as - # WAL replay failures are configured to throw (the engine default is to - # silently ignore them, and the binding no longer overrides that default). + # running again with enable_checksums=False should give an error + # (the engine now throws on a checksum-setting mismatch in both modes). with pytest.raises(RuntimeError) as check: - db = lb.Database( - database_path=database_path, - enable_checksums=False, - throw_on_wal_replay_failure=True, - ) + db = lb.Database(database_path=database_path, enable_checksums=False) def test_database_close_order() -> None: