Let greedy join reorder own join decisions with column statistics

.github/workflows/ci.yml

@@ -36,6 +36,19 @@ jobs:
 
       - name: Run cargo fmt
         run: make fmt
+  # 2.1
+  clippy:
+    name: Rust clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Run cargo clippy
+        run: make clippy
   # 3
   e2e:
     name: Rust e2e sqllogictest
@@ -47,6 +60,19 @@ jobs:
 
       - name: Run sqllogictest suite
         run: make test-slt
+  # 3.1
+  sqlite-slt:
+    name: SQLite sqllogictest corpus
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Run SQLite sqllogictest corpus
+        run: make test-slt-sqlite
   # 4
   coverage:
     name: Rust coverage
@@ -165,3 +191,22 @@ jobs:
         with:
           name: fuzz-artifacts
           path: fuzz/artifacts
+  # 10
+  miri:
+    name: Miri (unsafe code UB check)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2026-10-04
+          components: miri, rust-src
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: miri
+
+      - name: Set up Miri sysroot
+        run: cargo +nightly-2026-10-04 miri setup
+
+      - name: Run Miri tests
+        run: make miri

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/kitesql-testing"]
+	path = tests/kitesql-testing
+	url = https://github.com/KipData/kitesql-testing.git

AGENT.md

@@ -77,6 +77,18 @@ PRs that modify logic but leave obvious test gaps untouched may be rejected.
 
 ---
 
+### 2.4 Write Only Necessary Tests
+
+Coverage means each behavior is tested once, not that every case gets its own test.
+
+- Test each distinct branch or behavior once, at the layer that owns it
+- Do not repeat in a driver or caller what the rule or module already tests
+- Group related cases into one table-driven test instead of near-identical tests
+- Skip tests for trivial control flow (simple loops, early returns, plain delegation)
+- When touching a test module, delete tests made redundant by others
+
+---
+
 ## 3. Code Simplicity Rules
 
 ### 3.1 Prefer Simple Code
@@ -134,6 +146,13 @@ If code looks similar in multiple places, it probably wants to be reused.
 - Avoid exposing internal details
 - Keep modules small, focused, and cohesive
 
+### 4.3 Rust Imports
+
+- Declare `use` imports at module scope, not inside functions or methods.
+- Import functions and types before using them; do not write fully qualified paths
+  such as `crate::optimizer::rule::normalization::reorder_joins(...)` in function bodies.
+- Use import aliases when names conflict; keep test imports at the test module scope.
+
 ---
 
 ## 5. Dependency Policy

Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name          = "kite_sql"
-version       = "0.5.0"
+version       = "0.6.0"
 edition       = "2021"
 build         = "build.rs"
 authors       = ["Kould <kould2333@gmail.com>", "Xwg <loloxwg@gmail.com>"]
@@ -56,7 +56,7 @@ required-features = ["pprof"]
 bumpalo               = { version = "3", default-features = false, features = ["collections"] }
 ordered-float         = { version = "4" }
 paste                 = { version = "1" }
-kite_sql_serde_macros = { version = "0.2.4", path = "kite_sql_serde_macros" }
+kite_sql_serde_macros = { version = "0.2.5", path = "kite_sql_serde_macros" }
 
 # Optional dependencies for features
 comfy-table           = { version = "7", default-features = false, optional = true }

Makefile

@@ -3,6 +3,8 @@ CARGO ?= cargo
 GRCOV ?= grcov
 WASM_PACK ?= wasm-pack
 SQLLOGIC_PATH ?= tests/slt/**/*.slt
+SQLITE_LOGIC_PATH ?= tests/kitesql-testing/data/sqlite/**/*.slt
+SQLITE_LOGIC_ARGS ?=
 PYO3_PYTHON ?= /usr/bin/python3.12
 TPCC_MEASURE_TIME ?= 15
 TPCC_NUM_WARE ?= 1
@@ -33,9 +35,12 @@ FUZZ_PASS_EXIT_sql_exec = 70
 # sql_gen: bounded generated queries, so crashes, timeouts and OOMs all fail.
 FUZZ_ARGS_sql_gen = -max_len=4096
 FUZZ_PASS_EXIT_sql_gen = 0
+MIRI_TOOLCHAIN ?= nightly-2026-10-04
+MIRI_FEATURES ?= spill,decimal,lmdb
+MIRI_TESTS ?= types::tuple:: execution::test::
 COVERAGE_REPORT_ARGS ?= --llvm --ignore-not-existing --keep-only 'src/**' --ignore 'src/**/tests/**' --ignore 'tests/**' --ignore 'tpcc/**' --excl-start 'GRCOV_EXCL_START' --excl-stop 'GRCOV_EXCL_STOP'
 
-.PHONY: test test-python test-wasm test-slt test-all codecov codecov-html wasm-build check tpcc tpcc-kitesql-rocksdb tpcc-kitesql-lmdb tpcc-lmdb-flamegraph tpcc-lmdb-heaptrack tpcc-sqlite tpcc-dual cargo-check build wasm-examples native-examples fmt clippy fuzz
+.PHONY: test test-python test-wasm test-slt test-all codecov codecov-html wasm-build check tpcc tpcc-kitesql-rocksdb tpcc-kitesql-lmdb tpcc-lmdb-flamegraph tpcc-lmdb-heaptrack tpcc-sqlite tpcc-dual cargo-check build wasm-examples native-examples fmt clippy fuzz miri
 
 ## Run default Rust tests in the current environment (non-WASM).
 test:
@@ -65,6 +70,12 @@ test-wasm:
 test-slt:
 	$(CARGO) run -p sqllogictest-test -- --path '$(SQLLOGIC_PATH)'
 
+.PHONY: test-slt-sqlite
+
+## Run the external SQLite corpus (select1–5 excluded unless --include-select is passed).
+test-slt-sqlite:
+	$(CARGO) run --release -p sqllogictest-test --bin sqlite -- --path '$(SQLITE_LOGIC_PATH)' $(SQLITE_LOGIC_ARGS)
+
 ## Convenience target to run every suite in sequence.
 test-all: test test-wasm test-slt test-python
 
@@ -103,6 +114,9 @@ fuzz:
 		echo "fuzz: $(FUZZ_TARGET) failed (exit $$status), see fuzz/artifacts/$(FUZZ_TARGET)/"; exit $$status; \
 	fi
 
+miri:
+	$(CARGO) +$(MIRI_TOOLCHAIN) miri test --lib --features '$(MIRI_FEATURES)' -- $(MIRI_TESTS)
+
 ## Run formatting (check mode) across the workspace.
 fmt:
 	$(CARGO) fmt --all -- --check

README.md

@@ -19,7 +19,7 @@
 <p align="center">
     <a href="https://github.com/KipData/KiteSQL/actions/workflows/ci.yml"><img src="https://github.com/KipData/KiteSQL/actions/workflows/ci.yml/badge.svg" alt="CI"></img></a>
     <a href="https://codecov.io/github/KipData/KiteSQL"><img src="https://codecov.io/github/KipData/KiteSQL/branch/main/graph/badge.svg" alt="Codecov"></img></a>
-    <a href="https://deepwiki.com/KipData/KiteSQL"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki"></img></a>
+    <a href="https://deepwiki.com/KipData/KiteSQL"><img src="https://img.shields.io/badge/DeepWiki-Ask_AI-1f6feb" alt="Ask DeepWiki"></img></a>
     <a href="https://discord.gg/dU8eVGpJ8h"><img src="https://img.shields.io/badge/Discord-Join_us-5865F2?logo=discord&logoColor=white" alt="Discord"></a>
     <a href="https://crates.io/crates/kite_sql/"><img src="https://img.shields.io/crates/v/kite_sql.svg"></a>
     <a href="https://github.com/KipData/KiteSQL" target="_blank">
@@ -131,14 +131,13 @@ print(list(db.run("select * from demo")))
 ## TPC-C
 `make tpcc` runs the benchmark (`--backend rocksdb|lmdb`); `make tpcc-dual` cross-checks every statement against SQLite.
 
-720-second single-thread run on an i9-13900HX (32 GB, KIOXIA EXCERIA PLUS G3), every backend pinned to the same P-core. Latencies are p90 in µs, including commit.
+720-second single-thread run on an i9-13900HX (32 GB, KIOXIA EXCERIA PLUS G3), every backend pinned to the same P-core and run without fsync on commit. Latencies are p90 in µs, including commit.
 
 | Backend | TpmC | New-Order | Payment | Order-Status | Delivery | Stock-Level |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| KiteSQL LMDB | 159616 | 315 | 81 | 44 | 435 | 332 |
-| KiteSQL RocksDB | 48035 | 459 | 258 | 200 | 11055 | 704 |
-| SQLite balanced | 67102 | 283 | 67 | 48 | 338 | 473 |
-| SQLite practical | 67153 | 336 | 66 | 41 | 427 | 339 |
+| KiteSQL LMDB | 165956 | 303 | 75 | 43 | 436 | 318 |
+| KiteSQL RocksDB | 50739 | 392 | 188 | 198 | 11743 | 812 |
+| SQLite (WAL, `synchronous=OFF`) | 191484 | 274 | 41 | 33 | 343 | 268 |
 
 👉 [Details and how to reproduce](tpcc/README.md)
 

examples/transaction.rs

@@ -16,7 +16,6 @@
 mod app {
     use kite_sql::db::DataBaseBuilder;
     use kite_sql::errors::DatabaseError;
-    use kite_sql::types::tuple::Tuple;
     use kite_sql::types::value::DataValue;
     use std::fs;
     use std::io::ErrorKind;
@@ -43,7 +42,7 @@ mod app {
         // Optimistic transactions are currently backed by RocksDB.
         let mut database = DataBaseBuilder::path(EXAMPLE_DB_PATH).build_optimistic()?;
         database.ddl("create table if not exists t1 (c1 int primary key, c2 int)")?;
-        let mut transaction = database.new_transaction()?;
+        let mut transaction = database.new_transaction(false)?;
 
         transaction
             .run("insert into t1 values(0, 0), (1, 1)")?
@@ -57,29 +56,21 @@ mod app {
 
         let mut iter = database.run("select * from t1")?;
         iter.next_tuple(|_, tuple| {
-            assert_eq!(
-                tuple,
-                &Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)])
-            );
+            assert_eq!(*tuple, vec![DataValue::Int32(0), DataValue::Int32(0)]);
         })?
         .unwrap();
         iter.next_tuple(|_, tuple| {
-            assert_eq!(
-                tuple,
-                &Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(1)])
-            );
+            assert_eq!(*tuple, vec![DataValue::Int32(1), DataValue::Int32(1)]);
         })?
         .unwrap();
         assert!(iter.next_tuple(|_, _| ())?.is_none());
         iter.done()?;
 
-        let mut tx2 = database.new_transaction()?;
+        let mut tx2 = database.new_transaction(false)?;
         tx2.run("update t1 set c2 = 99 where c1 = 0")?.done()?;
         let mut c2_iter = database.run("select c2 from t1 where c1 = 0")?;
         assert_eq!(
-            c2_iter
-                .next_tuple(|_, tuple| tuple.values[0].i32())?
-                .unwrap(),
+            c2_iter.next_tuple(|_, tuple| tuple[0].i32())?.unwrap(),
             Some(0)
         );
         c2_iter.done()?;

fuzz/fuzz_targets/sql_exec.rs

@@ -72,7 +72,7 @@ fn reset(db: &mut Database<LmdbStorage>) {
         let mut names = Vec::new();
         if let Ok(mut iter) = db.run(show) {
             while let Ok(Some(())) = iter.next_tuple(|_, tuple| {
-                if let Some(DataValue::Utf8 { value, .. }) = tuple.values.first() {
+                if let Some(DataValue::Utf8 { value, .. }) = tuple.get(0) {
                     names.push(value.clone());
                 }
             }) {}
@@ -165,7 +165,7 @@ fn run_one(db: &mut Database<LmdbStorage>, sql: &str) {
             // Drain and render every value, like the sqllogictest harness does.
             loop {
                 match iter.next_tuple(|_, tuple| {
-                    for value in tuple.values.iter() {
+                    for value in tuple.iter() {
                         let _ = value.to_string();
                     }
                 }) {

fuzz/fuzz_targets/sql_gen.rs

@@ -34,6 +34,7 @@
 use kite_sql::db::{DataBaseBuilder, Database};
 use kite_sql::errors::DatabaseError;
 use kite_sql::storage::lmdb::LmdbStorage;
+use kite_sql::types::tuple::TupleImpl;
 use kite_sql::types::value::DataValue;
 use libfuzzer_sys::arbitrary::{self, Unstructured};
 use libfuzzer_sys::fuzz_target;
@@ -735,7 +736,11 @@ fn open_db() -> Database<LmdbStorage> {
 fn check_tlp(db: &Database<LmdbStorage>, select: &Select, predicate: &str) {
     let base_sql = select.render(false, false);
     let mut base = Vec::new();
-    if for_each_row(db, &base_sql, |row| base.push(row.to_vec())).is_err() {
+    if for_each_row(db, &base_sql, |row| {
+        base.push(row.iter().cloned().collect())
+    })
+    .is_err()
+    {
         return;
     }
     let order_by = select.order_by.map(|(column, _)| column);
@@ -778,8 +783,11 @@ fn rows_match(
     }
     let mut matched = true;
     let mut i = offset;
+    let mut key = Vec::new();
     let rows = for_each_row(db, sql, |row| {
-        let in_base = match remaining.get_mut(row) {
+        key.clear();
+        key.extend(row.iter().cloned());
+        let in_base = match remaining.get_mut(key.as_slice()) {
             Some(count) if *count > 0 => {
                 *count -= 1;
                 true
@@ -807,15 +815,15 @@ fn run(db: &Database<LmdbStorage>, sql: &str) -> Result<usize, DatabaseError> {
 fn for_each_row(
     db: &Database<LmdbStorage>,
     sql: &str,
-    mut f: impl FnMut(&[DataValue]),
+    mut f: impl FnMut(&TupleImpl),
 ) -> Result<usize, DatabaseError> {
     // Printed before running so a crashing statement is still visible.
     if debug() {
         eprintln!("{sql};");
     }
     let mut iter = db.run(sql)?;
     let mut rows = 0;
-    while iter.next_tuple(|_, tuple| f(&tuple.values))?.is_some() {
+    while iter.next_tuple(|_, tuple| f(tuple))?.is_some() {
         rows += 1;
     }
     iter.done()?;

kite_sql_serde_macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name        = "kite_sql_serde_macros"
-version     = "0.2.4"
+version     = "0.2.5"
 edition     = "2021"
 description = "Derive macros for KiteSQL"
 documentation = "https://docs.rs/kite_sql_serde_macros/latest/kite_sql_serde_macros/"

kite_sql_serde_macros/src/orm.rs

@@ -387,7 +387,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result<TokenStream, Error> {
         {
             fn from_query_row(
                 schema: &::kite_sql::types::tuple::SchemaView<'_, '_>,
-                tuple: &mut ::kite_sql::types::tuple::Tuple,
+                tuple: &mut ::kite_sql::types::tuple::TupleImpl,
             ) -> ::std::result::Result<Self, ::kite_sql::errors::DatabaseError> {
                 let mut __kite_orm_found_fields = 0usize;
                 #(#field_index_declarations)*

kite_sql_serde_macros/src/projection.rs

@@ -107,7 +107,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result<TokenStream, Error> {
                 relation: &str,
             ) -> ::std::result::Result<::std::vec::Vec<::kite_sql::planner::ExprRef>, ::kite_sql::errors::DatabaseError>
             where
-                T: ::kite_sql::storage::Transaction,
+                T: ::kite_sql::storage::StorageTransaction,
                 A: AsRef<[(usize, ::kite_sql::types::LogicalType)]>,
             {
                 Ok(::std::vec![
@@ -121,7 +121,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result<TokenStream, Error> {
         {
             fn from_query_row(
                 schema: &::kite_sql::types::tuple::SchemaView<'_, '_>,
-                tuple: &mut ::kite_sql::types::tuple::Tuple,
+                tuple: &mut ::kite_sql::types::tuple::TupleImpl,
             ) -> ::std::result::Result<Self, ::kite_sql::errors::DatabaseError> {
                 let mut __kite_orm_found_fields = 0usize;
                 #(#field_index_declarations)*

kite_sql_serde_macros/src/reference_serialization.rs

@@ -148,7 +148,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result<TokenStream, Error> {
                         Ok(())
                     }
 
-                    fn decode<Tx: crate::storage::Transaction, R: std::io::Read, A: crate::planner::MetaArena + ?Sized>(
+                    fn decode<Tx: crate::storage::StorageTransaction, R: std::io::Read, A: crate::planner::MetaArena + ?Sized>(
                         reader: &mut R,
                         drive: Option<&crate::serdes::ReferenceDecodeContext<'_, Tx>>,
                         reference_tables: &crate::serdes::ReferenceTables,
@@ -229,7 +229,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result<TokenStream, Error> {
                         Ok(())
                     }
 
-                    fn decode<Tx: crate::storage::Transaction, R: std::io::Read, A: crate::planner::MetaArena + ?Sized>(
+                    fn decode<Tx: crate::storage::StorageTransaction, R: std::io::Read, A: crate::planner::MetaArena + ?Sized>(
                         reader: &mut R,
                         drive: Option<&crate::serdes::ReferenceDecodeContext<'_, Tx>>,
                         reference_tables: &crate::serdes::ReferenceTables,