diff --git a/changelog.d/749.performance.md b/changelog.d/749.performance.md new file mode 100644 index 000000000..777e41ecd --- /dev/null +++ b/changelog.d/749.performance.md @@ -0,0 +1,3 @@ +Hash joins keep their build side in fixed-size row blocks with a smaller, denser hash directory, and a join's +unused columns are joined in only after the filtering joins above it: TPC-H q21 at scale factor 1000 runs in +277 s instead of 474 s with no spill (laptop, cold run). diff --git a/crates/krishiv-common/src/config/registry.rs b/crates/krishiv-common/src/config/registry.rs index a7c3c9032..7ed5167d2 100644 --- a/crates/krishiv-common/src/config/registry.rs +++ b/crates/krishiv-common/src/config/registry.rs @@ -28,7 +28,8 @@ pub use lakehouse::{ CATALOG_MAX_COLUMNS, CATALOG_MAX_METADATA_BYTES, CATALOG_REQUEST_TIMEOUT, DELTA_LOG_THREADS, }; pub use optimizer::{ - EXPERIMENTAL_GROUP_KEY_CODE_FORCE, GROUP_KEY_CODE, SHARE_REPEATED_SUBTREES, SUBQUERY_AS_WINDOW, + EXPERIMENTAL_GROUP_KEY_CODE_FORCE, GROUP_KEY_CODE, LATE_PAYLOAD_JOIN, SHARE_REPEATED_SUBTREES, + SUBQUERY_AS_WINDOW, }; pub use result_cache::{RESULT_CACHE_ENABLED, RESULT_CACHE_MAX_BYTES, RESULT_CACHE_TTL}; pub use server::{ @@ -97,6 +98,7 @@ pub static REGISTRY: &[&ConfigEntry] = &[ &RESULT_CACHE_TTL, &RESULT_CACHE_MAX_BYTES, &SUBQUERY_AS_WINDOW, + &LATE_PAYLOAD_JOIN, &GROUP_KEY_CODE, &EXPERIMENTAL_GROUP_KEY_CODE_FORCE, &SHARE_REPEATED_SUBTREES, @@ -798,7 +800,8 @@ pub static DETERMINISTIC_ORDER: ConfigEntry = ConfigEntry { seed, so an operator that spills splits its rows the same way on every run. Costs \ parallelism where it falls back to one driver: meant for reproducing a result, \ comparing two builds, and tests. Memory pressure from other queries in the process \ - can still change when an operator spills, and so the order of what it returns. The \ + can still change when an operator spills, or whether a hash join build copies its \ + rows into blocks as they arrive, and so the order of what it returns. The \ branches of a `UNION ALL` under an operator that must see all of them (a hash join \ probing a union) take turns in branch order, so they lose their overlap too. Off by \ default.", diff --git a/crates/krishiv-common/src/config/registry/index.rs b/crates/krishiv-common/src/config/registry/index.rs index b9dcc2c17..74216e191 100644 --- a/crates/krishiv-common/src/config/registry/index.rs +++ b/crates/krishiv-common/src/config/registry/index.rs @@ -170,6 +170,7 @@ pub static INDEX: &[(&str, &ConfigEntry)] = &[ "krishiv.memory.reservationQuantum", &MEMORY_RESERVATION_QUANTUM, ), + ("krishiv.optimizer.latePayloadJoin", &LATE_PAYLOAD_JOIN), ( "krishiv.python.worker.executables", &PYTHON_WORKER_EXECUTABLES, diff --git a/crates/krishiv-common/src/config/registry/optimizer.rs b/crates/krishiv-common/src/config/registry/optimizer.rs index b2e4e39c0..46419d2d9 100644 --- a/crates/krishiv-common/src/config/registry/optimizer.rs +++ b/crates/krishiv-common/src/config/registry/optimizer.rs @@ -33,6 +33,23 @@ pub static SHARE_REPEATED_SUBTREES: ConfigEntry = ConfigEntry { spark_alias: None, }; +/// A dimension's payload joins above the semi and anti joins that never read it +/// (`krishiv.optimizer.latePayloadJoin`). +pub static LATE_PAYLOAD_JOIN: ConfigEntry = ConfigEntry { + key: "krishiv.optimizer.latePayloadJoin", + value_type: ValueType::Bool, + default: Some(ConfigValue::Bool(true)), + doc: "Join a dimension's payload columns (such as a name read only by the aggregate on top) \ + after the semi, anti and outer joins that never read them, and leave a semi join on the \ + dimension's keys where the join was, so those joins build and probe narrower rows. Applies \ + when statistics show the payload is wide and the dimension is the smaller side. Results \ + are the same either way.", + scope: Scope::Session, + settable_by: SettableBy::User, + version_added: "0.1.0", + spark_alias: None, +}; + /// Whether an aggregate's group keys from one dimension travel as one integer code (P2b D2). pub static GROUP_KEY_CODE: ConfigEntry = ConfigEntry { key: "krishiv.optimizer.groupKeyCode", diff --git a/crates/krishiv-common/src/config/tests.rs b/crates/krishiv-common/src/config/tests.rs index c677e202c..489339d08 100644 --- a/crates/krishiv-common/src/config/tests.rs +++ b/crates/krishiv-common/src/config/tests.rs @@ -1329,6 +1329,21 @@ fn fuse_probe_select_defaults_on_and_parses() { ); } +#[test] +fn late_payload_join_defaults_on_and_parses() { + let mut c = Config::new(); + assert_eq!( + c.get(&LATE_PAYLOAD_JOIN).and_then(ConfigValue::as_bool), + Some(true) + ); + c.set("krishiv.optimizer.latePayloadJoin", "false").unwrap(); + assert_eq!( + c.get(&LATE_PAYLOAD_JOIN).and_then(ConfigValue::as_bool), + Some(false) + ); + assert!(c.set("krishiv.optimizer.latePayloadJoin", "maybe").is_err()); +} + #[test] fn python_worker_entries_parse_and_default() { let mut c = Config::new(); diff --git a/crates/krishiv-exec/CLAUDE.md b/crates/krishiv-exec/CLAUDE.md index 35be37ff5..8bfaade40 100644 --- a/crates/krishiv-exec/CLAUDE.md +++ b/crates/krishiv-exec/CLAUDE.md @@ -36,7 +36,8 @@ idle after `krishiv.exec.join.idleSpillAfter`, so the arbitrator asks them befor 30. Each probe driver then holds the build's level-0 tables, `build.rs` `Level0`, whose memory includes the per-partition reservations moved at the seal; the last probe driver done after the flush barrier releases them, so a downstream sink's final phase gets that memory; a `Prebuilt` broadcast build is never released) · `grace.rs` spill partitions, -flush-phase grace join · `broadcast.rs` +flush-phase grace join · `block.rs` a +partition's chunks kept few: the probe side's tiered merge, the build side's cut into the arena's 2^14-row blocks · `broadcast.rs` (P2-T177 unit 3) `BroadcastTables`, a worker resource (`WorkerContext::with_broadcasts`): a broadcast join over a cut broadcast exchange builds its table once per job and worker (its own query over the resource's pool, no spill, its reservations never abort a query) and every task probes it through a `PrebuiltSink` that republishes its runtime diff --git a/crates/krishiv-exec/src/join/block.rs b/crates/krishiv-exec/src/join/block.rs new file mode 100644 index 000000000..e1e0b8275 --- /dev/null +++ b/crates/krishiv-exec/src/join/block.rs @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Keeping a partition's in-memory chunks few: the probe side's size-tiered +//! merge ([`Part::coalesce`]) and the build side's cut into the row arena's +//! blocks of [`ARENA_ROWS`] rows ([`Part::block`], [`block_chunks`]; X3' +//! step 3 option C). + +use arrow_schema::SchemaRef; +use krishiv_memory::Reservation; +use krishiv_types::batch::Batch; + +use super::grace::Part; +use super::table::ARENA_ROWS; +use crate::error::{Result, internal}; + +impl Part { + /// Merges runs of small in-memory chunks (each chunk carries fixed array + /// overhead), if each merged copy can be reserved first. + /// + /// Size-tiered, like an LSM tree: a chunk's tier is `log16` of its bytes, + /// and only a trailing run of at least `FANOUT` chunks no higher in tier + /// than the newest one is merged. Merging every chunk held whenever the + /// count reached the fan-out copied the whole partition again every 15 + /// pushes (quadratic in its size); tiered, a row is copied about once + /// per tier it climbs. + /// + /// Returns the bytes of the largest run merged (each merge reserves the + /// run again while it copies: the transient peak), 0 when none was. + pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result { + match self.memory.first_mut() { + Some(m) => merge_tiers(&mut self.chunks, m, schema, usize::MAX, 16), + None => Ok(0), + } + } + + /// [`block_chunks`] over the part's chunks, the copies reserved in its + /// first reservation (the one its chunks are reserved in). Runs of + /// [`BUILD_FANOUT`] chunks shorter than a block are first merged + /// size-tiered among themselves ([`merge_tiers`]): a build fed small + /// batches pushes chunks of a few rows (12 drivers cutting each batch + /// into its partitions: about 20 rows each on TPC-H q10 at SF10, 29,359 + /// chunks for 573,157 rows), and finalize's accounting and tail copy then + /// walked every one of them (finalize 9 -> 72-89 ms). Chunks of a block or + /// more are never merged again, and chunks large enough to fill a block + /// before [`BUILD_FANOUT`] of them are held are copied once, by the cut. + pub(crate) fn block(&mut self, schema: &SchemaRef) -> Result { + match self.memory.first_mut() { + Some(m) => { + // Cut first: a run that already fills a block is copied once, + // into the block, not merged and then cut again. + let cut = block_chunks(&mut self.chunks, m, schema)?; + let merged = merge_tiers(&mut self.chunks, m, schema, ARENA_ROWS, BUILD_FANOUT)?; + Ok(cut.max(merged)) + } + None => Ok(0), + } + } +} + +/// The build side's merge fan-out: 64 chunks of a block's 2^14 rows hold +/// 256 rows each on average, so only chunks smaller than that are merged +/// before the block cut copies them. +const BUILD_FANOUT: usize = 64; + +/// [`Part::coalesce`]'s size-tiered merge over the trailing chunks shorter +/// than `rows` rows, `fanout` of one tier at a time, each merged copy +/// reserved in `m` first. +fn merge_tiers( + chunks: &mut Vec, + m: &mut Reservation, + schema: &SchemaRef, + rows: usize, + fanout: usize, +) -> Result { + let size = |c: &Batch| c.data().get_array_memory_size(); + let tier = |c: &Batch| size(c).max(1).ilog(16); + let mut largest = 0; + while chunks.len() >= fanout { + let Some(top) = chunks.last().map(tier) else { + return Ok(largest); + }; + let run = chunks + .iter() + .rev() + .take_while(|c| c.num_rows() < rows && tier(c) <= top) + .count(); + if run < fanout { + return Ok(largest); + } + let from = chunks.len() - run; + let old: usize = chunks[from..].iter().map(size).sum(); + if m.size() < old || m.try_grow(old).is_err() { + return Ok(largest); + } + let merged = Batch::concat(schema, &chunks[from..]).map_err(|e| internal(e.to_string())); + let merged = match merged { + Ok(b) if size(&b) <= old => b, + other => { + m.shrink(old); + return other.map(|_| largest); + } + }; + largest = largest.max(old); + m.shrink(old + old - size(&merged)); + chunks.truncate(from); + chunks.push(merged); + } + Ok(largest) +} + +/// Cuts build rows into the arena's blocks as they come (X3' step 3 +/// option C): once the trailing chunks shorter than a block hold a +/// block's rows, they are copied into one block of exactly +/// [`ARENA_ROWS`] rows (the last chunk split, its tail left as a +/// zero-copy slice), if the copy can be reserved first. Chunks of a block +/// or more stay as they are: the arena slices them. Each row is copied at +/// most once here, and finalize copies only the tails left. +/// +/// Returns the bytes of the largest copy (each one reserves its chunks +/// again while it copies: the transient peak), 0 when none was made. +pub(crate) fn block_chunks( + chunks: &mut Vec, + m: &mut Reservation, + schema: &SchemaRef, +) -> Result { + let size = |c: &Batch| c.data().get_array_memory_size(); + let mut largest = 0; + loop { + let run = chunks + .iter() + .rev() + .take_while(|c| c.num_rows() < ARENA_ROWS) + .count(); + let from = chunks.len() - run; + // The chunks that make up the block: whole ones, then the head of + // the last. + let mut held = 0usize; + let Some(last) = chunks[from..].iter().position(|c| { + held += c.num_rows(); + held >= ARENA_ROWS + }) else { + return Ok(largest); + }; + let last = from + last; + let head = ARENA_ROWS - (held - chunks[last].num_rows()); + let whole = if head == chunks[last].num_rows() { + last + 1 + } else { + last + }; + let freed: usize = chunks[from..whole].iter().map(size).sum(); + let old = freed + + if whole == last { + size(&chunks[last]) + } else { + 0 + }; + if m.size() < old { + return Ok(largest); + } + if m.try_grow(old).is_err() { + return Ok(largest); + } + let pieces = chunks[from..last] + .iter() + .cloned() + .map(Ok) + .chain(std::iter::once(chunks[last].slice(0, head))) + .collect::, _>>() + .and_then(|p| Batch::concat(schema, &p)) + .map_err(|e| internal(e.to_string())); + let n = chunks[last].num_rows(); + let tail = (whole == last).then(|| { + chunks[last] + .slice(head, n - head) + .map_err(|e| internal(e.to_string())) + }); + let (block, tail) = match (pieces, tail.transpose()) { + (Ok(b), Ok(t)) => (b, t), + (Err(e), _) | (_, Err(e)) => { + m.shrink(old); + return Err(e); + } + }; + // A copy larger than its sources (buffer rounding) is kept if the + // excess can be reserved; dropping it would redo it on every push. + let new = size(&block); + let excess = new.saturating_sub(old); + if excess > 0 && m.try_grow(excess).is_err() { + m.shrink(old); + return Ok(largest); + } + largest = largest.max(old); + // Held now: the block, and the last chunk while its tail lives + // (`old + excess >= new`). + m.shrink(old + excess + freed - new); + // Chunks past `last` (held while an earlier cut could not be + // reserved) are not in the block: they stay, after its tail. + let rest: Vec = chunks.drain(last + 1..).collect(); + chunks.truncate(from); + chunks.push(block); + chunks.extend(tail); + chunks.extend(rest); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::cast::AsArray; + use arrow_array::{Int64Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use krishiv_memory::{MemoryConfig, MemoryPool, TreePool}; + + use super::*; + use crate::join::grace::reserve_chunks; + + /// Coalescing is size-tiered: a merged chunk is not copied again by the + /// next run of small chunks (merging all chunks every 16 pushes was + /// quadratic), the chunk count stays bounded, and no row is lost. + #[test] + fn coalescing_does_not_recopy_merged_chunks() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("coalesce", None); + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("s", DataType::Utf8, false), + ])); + let chunk = |i: i64| { + let k = Int64Array::from_iter_values(i * 300..(i + 1) * 300); + let s = StringArray::from_iter_values((0..300).map(|j| format!("row-{i}-{j}"))); + Batch::new( + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k), Arc::new(s)]).unwrap(), + ) + }; + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let push = |parts: &mut Vec, i: i64| { + let c = chunk(i); + reserve_chunks(parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].coalesce(&schema).unwrap() + }; + let sizes: usize = (0..16) + .map(|i| chunk(i).data().get_array_memory_size()) + .sum(); + let mut merged_run = 0; + for i in 0..16 { + merged_run = push(&mut parts, i); + } + assert_eq!(parts[0].chunks.len(), 1, "16 small chunks merge"); + // It reports the run it merged: the 16 chunks' bytes. + assert_eq!(merged_run, sizes); + let merged = Arc::clone(parts[0].chunks[0].data().column(0)); + for i in 16..31 { + assert_eq!(push(&mut parts, i), 0, "no merge below the fan-out"); + } + assert_eq!(parts[0].chunks.len(), 16); + let first = parts[0].chunks[0].data().column(0); + assert!( + Arc::ptr_eq(first, &merged), + "the merged chunk was copied again" + ); + let mut most = 0; + for i in 31..1024 { + push(&mut parts, i); + most = most.max(parts[0].chunks.len()); + } + assert!(most < 3 * 16, "{most} chunks held"); + assert_eq!(parts[0].mem_rows(), 1024 * 300); + let mut keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + keys.sort_unstable(); + assert!( + keys.into_iter().eq(0..1024 * 300), + "rows lost or duplicated" + ); + drop(parts); + assert_eq!(pool.reserved(), 0); + } + + /// Build chunks are cut into the arena's blocks as they come: small + /// chunks are copied once into blocks of exactly `ARENA_ROWS` rows (the + /// chunk that crosses a block's end split, its tail kept), chunks of a + /// block or more are left alone, no row is lost and the reservation + /// follows what is held. + #[test] + fn build_chunks_are_cut_into_blocks_once() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("block", None); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); + let chunk = |from: i64, n: i64| { + Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from_iter_values(from..from + n))], + ) + .unwrap(), + ) + }; + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + let mut push = |parts: &mut Vec, n: i64| { + let c = chunk(next, n); + next += n; + reserve_chunks(parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].block(&schema).unwrap() + }; + let block = ARENA_ROWS as i64; + // 1,000-row chunks: a block once 17 of them are held, the 17th split. + let mut copied = 0; + for _ in 0..16 { + copied = copied.max(push(&mut parts, 1000)); + } + assert_eq!(copied, 0); + assert!(push(&mut parts, 1000) > 0); + let rows: Vec = parts[0].chunks.iter().map(Batch::num_rows).collect(); + assert_eq!(rows, [ARENA_ROWS, 17 * 1000 - ARENA_ROWS]); + // A chunk of a block or more is not copied, and ends the run before it. + assert_eq!(push(&mut parts, 2 * block + 7), 0); + let blocked = Arc::clone(parts[0].chunks[0].data().column(0)); + for _ in 0..40 { + push(&mut parts, 1000); + } + assert!(Arc::ptr_eq(parts[0].chunks[0].data().column(0), &blocked)); + assert!( + parts[0] + .chunks + .iter() + .filter(|c| c.num_rows() < ARENA_ROWS) + .count() + <= 17 + ); + let held: usize = parts[0] + .chunks + .iter() + .map(|c| c.data().get_array_memory_size()) + .sum(); + assert!( + parts[0].bytes() >= held, + "{} reserved, {held} held", + parts[0].bytes() + ); + let mut keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + keys.sort_unstable(); + assert!(keys.into_iter().eq(0..next), "rows lost or duplicated"); + drop(parts); + assert_eq!(pool.reserved(), 0); + } + + /// A run of short chunks that fills a block is cut into the block, not + /// merged into one chunk past the block first: 63 chunks of 255 rows and + /// one of 400 give a block and an 81-row tail (review M6 of the S5 stack). + #[test] + fn a_run_that_fills_a_block_is_cut_not_merged() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("cut-first", None); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + for n in std::iter::repeat_n(255i64, 63).chain([400]) { + let c = Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from_iter_values(next..next + n))], + ) + .unwrap(), + ); + next += n; + reserve_chunks(&mut parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].block(&schema).unwrap(); + } + let rows: Vec = parts[0].chunks.iter().map(Batch::num_rows).collect(); + assert_eq!(rows, [ARENA_ROWS, 63 * 255 + 400 - ARENA_ROWS]); + drop(parts); + assert_eq!(pool.reserved(), 0); + } + + /// A build fed tiny chunks (TPC-H q10 at SF10: about 20 rows each) holds + /// few of them: runs of `BUILD_FANOUT` merge before the block cut, so + /// finalize does not walk thousands. Rows keep their order, none is + /// lost, and the reservation covers what is held. + #[test] + fn tiny_build_chunks_are_merged_before_the_block_cut() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("tiny", None); + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("s", DataType::Utf8, false), + ])); + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + let mut most = 0; + for _ in 0..5000 { + let k = Int64Array::from_iter_values(next..next + 20); + let s = StringArray::from_iter_values((next..next + 20).map(|j| format!("row-{j}"))); + next += 20; + let c = Batch::new( + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k), Arc::new(s)]).unwrap(), + ); + reserve_chunks(&mut parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].block(&schema).unwrap(); + most = most.max(parts[0].chunks.len()); + } + // Without the merge: up to a block's 819 chunks of 20 rows. + assert!(most < 2 * BUILD_FANOUT, "{most} chunks held"); + let keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + assert!( + keys.into_iter().eq(0..next), + "rows lost, duplicated or reordered" + ); + let held: usize = parts[0] + .chunks + .iter() + .map(|c| c.data().get_array_memory_size()) + .sum(); + assert!( + parts[0].bytes() >= held, + "{} reserved, {held} held", + parts[0].bytes() + ); + drop(parts); + assert_eq!(pool.reserved(), 0); + } + + /// Chunks held past the one that fills a block (a cut skipped while its + /// copy could not be reserved, then more chunks pushed; a grace reload + /// under pressure) stay: the cut drops only the chunks it copied. Three + /// chunks of 10,000 rows give a block, the second's tail and the third. + #[test] + fn chunks_past_the_block_are_kept() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("past-the-block", None); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + for _ in 0..3 { + let c = Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from_iter_values(next..next + 10_000))], + ) + .unwrap(), + ); + next += 10_000; + reserve_chunks(&mut parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + } + let part = &mut parts[0]; + let Some(m) = part.memory.first_mut() else { + panic!("no reservation"); + }; + block_chunks(&mut part.chunks, m, &schema).unwrap(); + let mut keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + keys.sort_unstable(); + assert!(keys.into_iter().eq(0..next), "rows lost or duplicated"); + assert_eq!(parts[0].chunks[0].num_rows(), ARENA_ROWS); + drop(parts); + assert_eq!(pool.reserved(), 0); + } +} diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 4aef8e098..03356767a 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -384,7 +384,9 @@ impl SpillUnread for IdleBuild { for (p, table, scratch) in jobs { let rows = table.arena().len() as u64; let mut w = scope.writer(Arc::clone(&schema), scratch)?; - write_sliced(&mut w, table.arena().batch())?; + for block in table.arena().blocks() { + write_sliced(&mut w, block)?; + } drop(table); files.push((p, w.finish()?, rows)); } @@ -723,11 +725,14 @@ impl BuildSink { } } -/// What indexing a partition's in-memory rows reserves: `(index, copy)`, -/// its [`JoinTable`] index (and match bitmap when `marks`) and its -/// transient concatenation copy (none for a single chunk); `None` if no -/// rows are in memory. -fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize)> { +/// What indexing a partition's in-memory rows reserves, as `(lasting, +/// transient, kept)`: lasting is its [`JoinTable`] index (and match bitmap when +/// `marks`) plus the copied chunk tails that stay beside chunks the arena +/// also slices (held through the probe); transient is the rest of the +/// tails' copy, which replaces chunks dropped once the arena is built +/// ([`RowArena::reblock_bytes`]); kept is the tails' part of lasting. +/// `None` if no rows are in memory. +fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize, usize)> { if part.chunks.is_empty() { return None; } @@ -737,12 +742,25 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz if marks { index = index.saturating_add(JoinTable::marks_bytes(rows)); } - let copy = if part.chunks.len() > 1 { - part.bytes() - } else { - 0 - }; - Some((index, copy)) + let (copy, kept) = RowArena::reblock_bytes(&part.chunks, 0); + Some((index.saturating_add(kept), copy.saturating_sub(kept), kept)) +} + +/// Returns what a built partition's chunks reserved beyond its arena: the +/// chunks were reserved at their allocated size (builder slack, and tails +/// charged at the chunk they were sliced from), the arena copied the tails +/// into exact blocks and the chunks are gone, so the reservation keeps +/// `bytes`, the arena's allocated bytes ([`RowArena::held_bytes`]), less +/// the `kept` tail bytes the index reservation still holds (so the two +/// together cover the arena; `kept` may be all of it, leaving the chunks' +/// reservation nothing). Never below zero, and only ever shrinks. +fn trim_to_arena(part: &mut Part, kept: usize, bytes: usize) { + let mut excess = part.bytes().saturating_sub(bytes.saturating_sub(kept)); + for m in part.memory.iter_mut() { + let cut = excess.min(m.size()); + m.shrink(cut); + excess -= cut; + } } /// A duration in whole nanoseconds, saturating. @@ -756,8 +774,13 @@ struct Job { /// Bytes of its rows in memory. bytes: usize, chunks: Vec, - /// What indexing it reserves (see [`build_need`]). + /// What indexing it reserves: lasting (index and kept tails) and the + /// transient copy (see [`build_need`]). need: (usize, usize), + /// Of the lasting part, the arena's copied tails that stay beside chunks + /// it also slices: reserved through the probe + /// ([`RowArena::reblock_bytes`]). + kept: usize, /// Its estimated distinct keys (row layout, when sketched). keys: Option, } @@ -986,7 +1009,7 @@ impl Sink for BuildSink { } for (p, c) in chunks { l.parts[p].chunks.push(c); - let merged = l.parts[p].coalesce(s.arena_schema(&g.hashing))?; + let merged = l.parts[p].block(s.arena_schema(&g.hashing))?; s.counters.max(Counter::MaxCoalesceCopyBytes, merged as u64); } if let Some((sketches, _)) = &mut l.sketches { @@ -1048,9 +1071,11 @@ impl Sink for BuildSink { let s = &self.0; let filter_memory = self.publish_filters(g, ctx); // Write what is left of spilled partitions; reserve the index of - // the rest, spilling the largest while it does not fit. A - // partition's table is built only while its concatenation copy - // is reserved, so the copies never need the build's size again. + // the rest, spilling the largest while it does not fit. The arenas + // read the blocks in place; finalize copies only the chunk tails. + // A spilling build reserves every index and kept tail and the + // largest transient copy, so one partition can always run and the + // copies never need the build's size again. let mut index = ctx.consumer().reservation(); let mut held_copy = 0; if s.spill.is_some() { @@ -1078,7 +1103,7 @@ impl Sink for BuildSink { let rows = g.hashing.rows(); let marks = s.join_type.builds_preserved(); let needs = c.parts.iter().filter_map(|p| build_need(p, rows, marks)); - needs.fold((0usize, 0usize), |(i, m), (index, copy)| { + needs.fold((0usize, 0usize), |(i, m), (index, copy, _)| { (i.saturating_add(index), usize::max(m, copy)) }) }; @@ -1102,11 +1127,12 @@ impl Sink for BuildSink { let marks = s.join_type.builds_preserved(); Some((i, build_need(p, g.hashing.rows(), marks)?, p, keys)) }) - .map(|(i, need, p, keys)| Job { + .map(|(i, (lasting, copy, kept), p, keys)| Job { part: i, bytes: p.bytes(), + kept, chunks: std::mem::take(&mut p.chunks), - need, + need: (lasting, copy), keys, }) .collect() @@ -1117,11 +1143,11 @@ impl Sink for BuildSink { // Nothing in memory is spillable any more. ctx.consumer().signal().set_spillable_bytes(0); // One blocking task per partition (the executor's blocking pool - // bounds how many run at once), each started only once its - // concatenation copy is reserved. A spilling build's reservation - // already funds the largest copy (`spare`), so one partition can - // always run; without a spill scope a partition's index and copy - // are asked for through arbitration when nothing else runs. + // bounds how many run at once), each started only once its tails' + // copy is reserved. A spilling build's reservation already funds + // the largest transient copy (`spare`), so one partition can always run; + // without a spill scope a partition's index and copy are asked + // for through arbitration when nothing else runs. let spill = s.spill.is_some(); let mut spare = held_copy; // The concatenation copies reserved at once (for the counter). @@ -1129,6 +1155,10 @@ impl Sink for BuildSink { let mut jobs = std::collections::VecDeque::from(jobs); let mut running = futures::stream::FuturesUnordered::new(); let mut tables = Vec::with_capacity(jobs.len()); + // Per built partition: the kept tails its index reservation + // holds and its arena's allocated bytes, to true up its chunks' + // reservation once they are gone. + let mut arena_held = Vec::with_capacity(jobs.len()); loop { if let Some(&Job { need: (index_bytes, copy), @@ -1144,7 +1174,7 @@ impl Sink for BuildSink { let funded = if spill && copy <= spare { spare -= copy; true - } else if index.try_grow(grow).is_ok() { + } else if grow == 0 || index.try_grow(grow).is_ok() { true } else if running.is_empty() { index.grow(grow).await.map_err(KrishivError::from)?; @@ -1157,6 +1187,7 @@ impl Sink for BuildSink { chunks, need: (index_bytes, copy), keys, + kept, .. }) = jobs.pop_front_if(|_| funded) { @@ -1171,7 +1202,7 @@ impl Sink for BuildSink { .run_blocking(move || { let t0 = std::time::Instant::now(); let schema = shared.arena_schema(&hashing); - let arena = RowArena::concat(schema, &chunks)?; + let arena = RowArena::over_chunks(schema, &chunks)?; drop(chunks); let t1 = std::time::Instant::now(); let layout = shared.keys.layout().filter(|_| hashing.rows()); @@ -1195,7 +1226,7 @@ impl Sink for BuildSink { table }) .await?; - Ok::<_, KrishivError>((p, table, copy, index_bytes)) + Ok::<_, KrishivError>((p, table, index_bytes, copy, kept)) }); continue; } @@ -1203,15 +1234,28 @@ impl Sink for BuildSink { let Some(done) = futures::StreamExt::next(&mut running).await else { break; }; - let (p, table, copy, index_bytes) = done?; - copying = copying.saturating_sub(copy); + let (p, table, lasting, transient, kept) = done?; + copying = copying.saturating_sub(transient); // The index was reserved for one key per row: keep what it - // holds. + // holds. The copied tails beside chunks the arena also slices + // (`kept`, part of `lasting`) stay reserved through the probe. index.shrink( - index_bytes + lasting + .saturating_sub(kept) .saturating_sub(table.index_memory()) .min(index.size()), ); + // The transient copy replaced chunks that are gone now. + if spill { + // Keep the largest transient copy funded; return the rest. + spare = spare.saturating_add(transient); + if spare > held_copy { + index.shrink((spare - held_copy).min(index.size())); + spare = held_copy; + } + } else { + index.shrink(transient.min(index.size())); + } let split = table.arena().memory_split(); let n = &self.0.counters; n.add( @@ -1225,19 +1269,13 @@ impl Sink for BuildSink { self.0.counters.or(Counter::IndexKinds, kind as u64); self.0.counters.add(Counter::HashEntries, keys as u64); self.0.counters.add(Counter::HashBuckets, buckets as u64); + // Of `kept`, what the index reservation still holds once the + // index is paid: an index past its estimate took the rest. + let covered = lasting.saturating_sub(table.index_memory()).min(kept); + arena_held.push((p, covered, table.arena().held_bytes())); tables.push((p, table)); - if spill { - // Keep the largest copy funded; return the rest. - spare = spare.saturating_add(copy); - if spare > held_copy { - index.shrink(spare - held_copy); - spare = held_copy; - } - } else { - index.shrink(copy); - } } - index.shrink(spare); + index.shrink(spare.min(index.size())); let mut c = lock(&g.collect); // Spilled partitions that hold rows: the probe defers only these // (an empty partition matches nothing, spilled or not). @@ -1268,6 +1306,11 @@ impl Sink for BuildSink { let mut memory = vec![index]; let mut grace = Grace::new(g.hashing.fanout0()); grace.keep_unprobed = s.join_type.keeps_null_build_keys(); + for (p, kept, bytes) in arena_held { + if let Some(part) = c.parts.get_mut(p) { + trim_to_arena(part, kept, bytes); + } + } for (p, part) in c.parts.iter_mut().enumerate() { if spilled & (1 << p) == 0 { self.0 diff --git a/crates/krishiv-exec/src/join/fallback.rs b/crates/krishiv-exec/src/join/fallback.rs index 15bff3719..1d1c2f9a0 100644 --- a/crates/krishiv-exec/src/join/fallback.rs +++ b/crates/krishiv-exec/src/join/fallback.rs @@ -323,7 +323,7 @@ fn build_rows( schema: &SchemaRef, ) -> Result { let data = chunk.data(); - let arena = RowArena::concat(shared.arena_schema(hashing), std::slice::from_ref(chunk))?; + let arena = RowArena::over_chunks(shared.arena_schema(hashing), std::slice::from_ref(chunk))?; let rows = u32::try_from(arena.len()).map_err(|_| internal("a chunk too large".to_owned()))?; let picks: Vec<(usize, u32)> = (0..rows).map(|r| (0, r)).collect(); let mut columns = shared.build_columns(hashing.rows(), &picks, |_| Some(&arena))?; diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index a1aea7d95..b118a0440 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -33,6 +33,7 @@ use krishiv_memory::{ }; use krishiv_types::batch::Batch; +use super::block::block_chunks; use super::build::BuildGlobal; use super::keys::{Hashing, MAX_DEPTH, PARTITIONS}; use super::probe::{Cursor, Tables}; @@ -78,61 +79,6 @@ impl Part { self.chunks.iter().map(Batch::num_rows).sum() } - /// Merges runs of small in-memory chunks (each chunk carries fixed array - /// overhead), if each merged copy can be reserved first. - /// - /// Size-tiered, like an LSM tree: a chunk's tier is `log16` of its bytes, - /// and only a trailing run of at least `FANOUT` chunks no higher in tier - /// than the newest one is merged. Merging every chunk held whenever the - /// count reached the fan-out copied the whole partition again every 15 - /// pushes (quadratic in its size); tiered, a row is copied about once - /// per tier it climbs. - /// - /// Returns the bytes of the largest run merged (each merge reserves the - /// run again while it copies: the transient peak), 0 when none was. - pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result { - const FANOUT: usize = 16; - let size = |c: &Batch| c.data().get_array_memory_size(); - let tier = |c: &Batch| size(c).max(1).ilog(FANOUT); - let mut largest = 0; - while self.chunks.len() >= FANOUT { - let Some(top) = self.chunks.last().map(tier) else { - return Ok(largest); - }; - let run = self - .chunks - .iter() - .rev() - .take_while(|c| tier(c) <= top) - .count(); - if run < FANOUT { - return Ok(largest); - } - let from = self.chunks.len() - run; - let old: usize = self.chunks[from..].iter().map(size).sum(); - let Some(m) = self.memory.first_mut().filter(|m| m.size() >= old) else { - return Ok(largest); - }; - if m.try_grow(old).is_err() { - return Ok(largest); - } - let merged = - Batch::concat(schema, &self.chunks[from..]).map_err(|e| internal(e.to_string())); - let merged = match merged { - Ok(b) if size(&b) <= old => b, - other => { - m.shrink(old); - return other.map(|_| largest); - } - }; - largest = largest.max(old); - m.shrink(old + old - size(&merged)); - self.chunks.truncate(from); - self.chunks.push(merged); - } - Ok(largest) - } - pub(crate) fn absorb(&mut self, other: Self) { self.chunks.extend(other.chunks); self.files.extend(other.files); @@ -466,6 +412,8 @@ struct Active { table: JoinTable, _memory: Reservation, chunks: Vec, + /// The build part's own reservations (its in-memory chunks, if any). + _build_memory: Vec, _chunk_memory: Vec, files: VecDeque, reader: Option, @@ -701,7 +649,7 @@ fn reverse_blocking( // The stream runs in one blocking call: a cancelled query stops it // at the next chunk. query.check_cancelled()?; - let arena = RowArena::concat(schema, std::slice::from_ref(&chunk))?; + let arena = RowArena::over_chunks(schema, std::slice::from_ref(&chunk))?; let rows = u32::try_from(arena.len()).map_err(|_| internal("build chunk too large".to_owned()))?; // (probe row, build row of this chunk), evaluated before it is dropped. @@ -1230,10 +1178,12 @@ impl Loading { ctx.consumer().consumer().release_credit(); self.memory.shrink_to_credit(headroom); let Task { build, probe, .. } = self.task; - drop(build); Active { table, _memory: self.memory, + // The arena reads its chunks in place: any build chunks the task + // still held stay reserved while the table may read them. + _build_memory: build.memory, chunks: probe.chunks, _chunk_memory: probe.memory, files: probe.files.into(), @@ -1254,6 +1204,16 @@ impl Loading { return Ok(Loaded::Need(bytes)); } self.chunks.push(b); + // Spill frames are slices of 64 KiB: joined into blocks as + // they come, one block's copy at a time, not all at the end. + let copied = block_chunks( + &mut self.chunks, + &mut self.memory, + shared.arena_schema(hashing), + )?; + shared + .counters + .max(Counter::MaxCoalesceCopyBytes, copied as u64); } if self.reader.is_none() { let Some(file) = self.task.build.files.get(self.file) else { @@ -1287,9 +1247,10 @@ impl Loading { Some(b) => self.pending = Some(b?), } } - // Every row read: the table's index, the concatenated copy of the - // rows and the probe rows' read-back headroom (exact, so a - // partition that loads can always be joined). + // Every row read: the table's index and the probe rows' read-back + // headroom (exact, so a partition that loads can always be joined). + // The arena reads the chunks in place, copying only blocks that span + // chunks. let chunks: Vec = self .task .build @@ -1299,28 +1260,26 @@ impl Loading { .cloned() .collect(); let rows = chunks.iter().map(Batch::num_rows).sum(); - let concat = if chunks.len() > 1 { - self.memory.size() + self.task.build.bytes() - } else { - 0 - }; let headroom = self.task.probe.read_headroom(); let marks = if shared.join_type.builds_preserved() { JoinTable::marks_bytes(rows) } else { 0 }; + // Blocks that span chunks are copied: reserved, and the part beside + // chunks that live on (the task's own build chunks) stays reserved. + let (copy, kept) = RowArena::reblock_bytes(&chunks, self.task.build.chunks.len()); let need = JoinTable::index_bytes(rows, key_bytes(&chunks), hashing.rows()) + marks - + concat - + headroom; + + headroom + + copy; if self.memory.try_grow(need).is_err() { return Ok(Loaded::Need(need)); } - let arena = RowArena::concat(shared.arena_schema(hashing), &chunks)?; + let arena = RowArena::over_chunks(shared.arena_schema(hashing), &chunks)?; drop(chunks); self.chunks.clear(); - self.memory.shrink(concat); + self.memory.shrink(copy - kept); let layout = shared.keys.layout().filter(|_| hashing.rows()); let mut table = JoinTable::build(arena, layout, hashing.g_seed())?; if shared.join_type.builds_preserved() { diff --git a/crates/krishiv-exec/src/join/mod.rs b/crates/krishiv-exec/src/join/mod.rs index 0def70854..b955a1348 100644 --- a/crates/krishiv-exec/src/join/mod.rs +++ b/crates/krishiv-exec/src/join/mod.rs @@ -154,6 +154,7 @@ //! the tables: two clock reads per batch, for //! [`QueryMetrics::hash_probe_share`](crate::QueryMetrics::hash_probe_share)). +mod block; mod broadcast; mod build; mod fallback; diff --git a/crates/krishiv-exec/src/join/probe.rs b/crates/krishiv-exec/src/join/probe.rs index 9483f956b..66cc00e13 100644 --- a/crates/krishiv-exec/src/join/probe.rs +++ b/crates/krishiv-exec/src/join/probe.rs @@ -248,7 +248,7 @@ impl Cursor { probe.push(u32::try_from(i).unwrap_or(u32::MAX)); build.push((p, self.chain)); } - self.chain = table.map_or(NONE, |t| t.next()[self.chain as usize]); + self.chain = table.map_or(NONE, |t| t.next_of(self.chain)); if self.chain == NONE { self.pos += 1; } diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index 711de093b..624656649 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -27,7 +27,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use arrow_array::cast::AsArray; use arrow_array::types::UInt64Type; -use arrow_array::{Array, LargeBinaryArray, UInt64Array}; +use arrow_array::{Array, LargeBinaryArray, RecordBatch, UInt64Array}; use arrow_buffer::Buffer; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use krishiv_types::batch::{Batch, Weights}; @@ -213,7 +213,7 @@ impl Cols { } } -/// An arena's bytes by part ([`RowArena::memory_split`]), as `get_array_memory_size` counts them. +/// An arena's bytes by part ([`RowArena::memory_split`]): each block's rows' share of its buffers. #[derive(Debug, Default, Clone, Copy)] pub(crate) struct ArenaBytes { pub(crate) hash: usize, @@ -222,9 +222,11 @@ pub(crate) struct ArenaBytes { pub(crate) other: usize, } -/// One partition's build rows, coalesced into a single batch. +/// One block of an arena: a batch of build rows and its columns, read in place. #[derive(Debug)] -pub(crate) struct RowArena { +struct ArenaPart { + /// `batch.num_rows()`, kept beside the columns `at` reads. + rows: u32, batch: Batch, hashes: UInt64Array, keys: Cols, @@ -234,14 +236,10 @@ pub(crate) struct RowArena { null_key: Option, } -impl RowArena { - /// Concatenates arena chunks (same schema). - pub(crate) fn concat(schema: &SchemaRef, chunks: &[Batch]) -> Result { - let batch = match chunks { - [one] => one.clone(), - _ => Batch::concat(schema, chunks).map_err(|e| internal(e.to_string()))?, - }; +impl ArenaPart { + fn of(batch: Batch) -> Result { let bad = || internal("malformed join row arena".to_owned()); + let rows = u32::try_from(batch.num_rows()).map_err(|_| bad())?; let hashes = batch.data().column_by_name("hash").ok_or_else(bad)?; let hashes = hashes .as_primitive_opt::() @@ -253,10 +251,8 @@ impl RowArena { Some(c) => Some(c.as_boolean_opt().ok_or_else(bad)?.clone()), None => None, }; - if u32::try_from(batch.num_rows()).is_err() { - return Err(internal("a join partition has over 2^32 rows".to_owned())); - } Ok(Self { + rows, batch, hashes, keys, @@ -264,46 +260,287 @@ impl RowArena { null_key, }) } +} + +/// Rows per arena block: `1 << ARENA_SHIFT` (X3' step 3 option C, study 10 +/// §4: 16,384 rows, about 1-6 MB of build rows). +pub(crate) const ARENA_SHIFT: u32 = 14; +/// [`ARENA_SHIFT`] as a row count. +pub(crate) const ARENA_ROWS: usize = 1 << ARENA_SHIFT; + +/// One partition's build rows, read in place (X3' step 3, design +/// `P2-X3-copy-free-join-finalize` option C): blocks of exactly +/// `1 << shift` rows, the last one shorter, so row `id` is row +/// `id & mask` of block `id >> shift`, found without a table or a search. +/// Rows are numbered across the blocks in order. +#[derive(Debug)] +pub(crate) struct RowArena { + parts: Vec, + shift: u32, + len: usize, +} + +/// How [`RowArena::over_chunks`] cuts chunks into blocks: each entry is a +/// block of pieces `(chunk, offset, rows)`; a block of one piece is a +/// zero-copy slice, one of several is concatenated. Every whole block a +/// chunk holds comes first, in order; then the tails (chunks shorter than a +/// block, and what is left of longer ones) are joined into blocks, the last +/// one shorter. A partition's chunks are its drivers' lists one after +/// another, each ending in a tail: in row order, the first tail would shift +/// every later block across two chunks and copy it. +fn block_plan(chunks: &[Batch], block: usize) -> Vec> { + let mut blocks = Vec::new(); + let mut tails = Vec::new(); + for (c, chunk) in chunks.iter().enumerate() { + let n = chunk.num_rows(); + let whole = n - n % block; + blocks.extend((0..whole).step_by(block).map(|at| vec![(c, at, block)])); + if whole < n { + tails.push((c, whole, n - whole)); + } + } + let mut pending: Vec<(usize, usize, usize)> = Vec::new(); + let mut held = 0usize; + for (c, mut at, mut n) in tails { + while n > 0 { + let take = (block - held).min(n); + pending.push((c, at, take)); + held += take; + at += take; + n -= take; + if held == block { + blocks.push(std::mem::take(&mut pending)); + held = 0; + } + } + } + if !pending.is_empty() { + blocks.push(pending); + } + blocks +} + +/// The bytes a slice of `len` rows at `offset` of `batch` takes once copied +/// (its rows' share of the buffers, weights included). +fn slice_bytes(batch: &Batch, offset: usize, len: usize) -> usize { + let data = batch.data(); + let columns: usize = data + .columns() + .iter() + .map(|c| { + let part = c.slice(offset, len).to_data(); + part.get_slice_memory_size() + .unwrap_or_else(|_| part.get_array_memory_size()) + }) + .fold(0, usize::saturating_add); + let weights = match batch.weights() { + Weights::Const(_) => 0, + Weights::Column(_) => len * size_of::(), + }; + columns.saturating_add(weights) +} + +impl RowArena { + /// The arena over `chunks` (same schema). Chunks are cut into blocks of + /// [`ARENA_ROWS`] rows: a block inside one chunk is a zero-copy slice, + /// the chunks' tails are copied into blocks ([`block_plan`] gives the + /// row order; [`reblock_bytes`](Self::reblock_bytes) bounds the copy). + /// One chunk keeps its row order. + /// An arena of no rows keeps one empty chunk of `schema`, so its key + /// format is still known. + pub(crate) fn over_chunks(schema: &SchemaRef, chunks: &[Batch]) -> Result { + Self::with_shift(schema, chunks, ARENA_SHIFT) + } + + fn with_shift(schema: &SchemaRef, chunks: &[Batch], shift: u32) -> Result { + let block = 1usize << shift; + let plan = block_plan(chunks, block); + let mut parts = Vec::with_capacity(plan.len().max(1)); + let mut len = 0usize; + for pieces in plan { + let batch = match pieces.as_slice() { + [(c, at, n)] => match chunks.get(*c) { + Some(chunk) if *at == 0 && *n == chunk.num_rows() => chunk.clone(), + Some(chunk) => chunk.slice(*at, *n).map_err(|e| internal(e.to_string()))?, + None => return Err(internal("join arena block out of range".to_owned())), + }, + _ => { + let slices = pieces + .iter() + .map(|&(c, at, n)| { + chunks + .get(c) + .ok_or_else(|| { + internal("join arena block out of range".to_owned()) + })? + .slice(at, n) + .map_err(|e| internal(e.to_string())) + }) + .collect::>>()?; + Batch::concat(schema, &slices).map_err(|e| internal(e.to_string()))? + } + }; + len = len.saturating_add(batch.num_rows()); + parts.push(ArenaPart::of(batch)?); + } + if parts.is_empty() { + let empty = match chunks.first() { + Some(c) => c.clone(), + None => Batch::new(RecordBatch::new_empty(Arc::clone(schema))), + }; + parts.push(ArenaPart::of(empty)?); + } + Self::row_id(len)?; + Ok(Self { parts, shift, len }) + } + + /// What [`over_chunks`](Self::over_chunks) copies to build the blocks + /// that span chunks: `(copy, kept)`, the bytes of the copy (reserved + /// before it, buffer rounding included) and of the part of it that stays + /// beside its source once the arena is built, because the source chunk + /// lives on: the first `live` chunks (held by their owner), and chunks + /// that also give whole blocks (sliced in place). The rest of the copy + /// replaces chunks that are dropped. + pub(crate) fn reblock_bytes(chunks: &[Batch], live: usize) -> (usize, usize) { + // Each new buffer rounds up to 64 bytes; about three per column. + let rounding = chunks + .first() + .map_or(0, |c| c.data().num_columns().saturating_add(1) * 3 * 64); + let mut copy = 0usize; + let mut kept = 0usize; + for pieces in block_plan(chunks, ARENA_ROWS) + .iter() + .filter(|p| p.len() > 1) + { + copy = copy.saturating_add(rounding); + for &(c, at, n) in pieces { + let Some(b) = chunks.get(c) else { continue }; + let bytes = slice_bytes(b, at, n); + copy = copy.saturating_add(bytes); + if c < live || b.num_rows() >= ARENA_ROWS { + kept = kept.saturating_add(bytes); + } + } + } + (copy, kept) + } + + fn row_id(n: usize) -> Result { + u32::try_from(n).map_err(|_| internal("a join partition has over 2^32 rows".to_owned())) + } + + /// The block holding row `row` and the row's index in it (`None` past + /// the end). + #[inline] + fn at(&self, row: u32) -> Option<(&ArenaPart, u32)> { + let part = self.parts.get((row >> self.shift) as usize)?; + let local = row & ((1u32 << self.shift) - 1); + (local < part.rows).then_some((part, local)) + } pub(crate) fn len(&self) -> usize { - self.batch.num_rows() + self.len } - /// The rows as one batch of the arena schema: what a spilled partition - /// writes and its reload concatenates. - pub(crate) fn batch(&self) -> &Batch { - &self.batch + /// The bytes the arena's blocks keep allocated: the capacity of every + /// buffer they reference, each counted once (blocks sliced from one + /// chunk share its buffers). A build's chunks were reserved at their + /// allocated size, builder slack included; once finalize copies the + /// tails into exact blocks and drops the chunks, this is what the + /// arena still holds (study 21 §4.4: the slack of the tails kept + /// reserved was ~27 MB per build at 12 threads). Once per partition at + /// finalize: it clones each column's `ArrayData`, so it stays off hot + /// paths. + pub(crate) fn held_bytes(&self) -> usize { + fn walk( + data: &arrow_data::ArrayData, + seen: &mut std::collections::HashSet, + sum: &mut usize, + ) { + let nulls = data.nulls().map(arrow_buffer::NullBuffer::buffer); + for b in data.buffers().iter().chain(nulls) { + if seen.insert(b.data_ptr().as_ptr() as usize) { + *sum = sum.saturating_add(b.capacity()); + } + } + for child in data.child_data() { + walk(child, seen, sum); + } + } + let columns = self + .parts + .first() + .map_or(0, |p| p.batch.data().num_columns()); + let mut seen = std::collections::HashSet::with_capacity(self.parts.len() * columns * 3); + let mut sum = 0usize; + for part in &self.parts { + for column in part.batch.data().columns() { + walk(&column.to_data(), &mut seen, &mut sum); + } + if let Weights::Column(w) = part.batch.weights() { + walk(&w.to_data(), &mut seen, &mut sum); + } + } + sum } - /// The arena's bytes by part (`get_array_memory_size`, the measure its reservation uses): the `hash` column, the key - /// columns (`key*`), and everything else (payload, NULL-key flags). + /// The arena's bytes by part (each block's share of its buffers): the `hash` column, the key columns (`key*`), and + /// everything else (payload, NULL-key flags). pub(crate) fn memory_split(&self) -> ArenaBytes { - let data = self.batch.data(); let mut out = ArenaBytes::default(); - for (field, column) in data.schema().fields().iter().zip(data.columns()) { - let bytes = column.get_array_memory_size(); - let name = field.name(); - if name == "hash" { - out.hash += bytes; - } else if name.starts_with("key") { - out.key += bytes; - } else { - out.other += bytes; + for part in &self.parts { + let data = part.batch.data(); + for (field, column) in data.schema().fields().iter().zip(data.columns()) { + // Blocks sliced from one chunk share its buffers: count each + // block's rows only. + let column = column.to_data(); + let bytes = column + .get_slice_memory_size() + .unwrap_or_else(|_| column.get_array_memory_size()); + let name = field.name(); + if name == "hash" { + out.hash += bytes; + } else if name.starts_with("key") { + out.key += bytes; + } else { + out.other += bytes; + } } } out } + /// The arena's blocks in row order, each a batch of the arena schema (what an idle build writes to disk). + pub(crate) fn blocks(&self) -> impl Iterator { + self.parts.iter().map(|part| &part.batch) + } + pub(crate) fn hash(&self, row: u32) -> u64 { - self.hashes.value(row as usize) + self.at(row).map_or(0, |(p, i)| p.hashes.value(i as usize)) } - /// The arrow-row key bytes, if the keys are in that format. - fn arrow_keys(&self) -> Option<&LargeBinaryArray> { - match &self.keys { - Cols::Arrow(k) => Some(k), - Cols::Rows(_) => None, - } + /// Whether the keys are arrow-row bytes (not row-layout rows). + fn has_arrow_keys(&self) -> bool { + self.parts + .first() + .is_some_and(|p| matches!(p.keys, Cols::Arrow(_))) + } + + /// The arrow-row key bytes of all rows, for sizing an index. + fn arrow_key_bytes(&self) -> usize { + self.parts + .iter() + .map(|p| match &p.keys { + // A block sliced from a chunk shares its values: count the + // block's own range. + Cols::Arrow(k) => { + let o = k.value_offsets(); + let span = o.last().copied().unwrap_or(0) - o.first().copied().unwrap_or(0); + usize::try_from(span).unwrap_or(0) + } + Cols::Rows(_) => 0, + }) + .sum() } /// The row's key bytes, comparable byte for byte with another row's of @@ -311,8 +548,9 @@ impl RowArena { /// row-layout row. `None` for a layout with string views, whose rows /// point into a heap. pub(crate) fn key_bytes(&self, row: u32) -> Option<&[u8]> { + let (part, row) = self.at(row)?; let row = row as usize; - match &self.keys { + match &part.keys { Cols::Arrow(k) => Some(k.value(row)), Cols::Rows(RowCols { fixed, @@ -326,8 +564,14 @@ impl RowArena { /// The row's arrow-row payload bytes (empty if the arena has none or /// holds payload rows). pub(crate) fn payload(&self, row: u32) -> &[u8] { - match &self.payload { - Some(Cols::Arrow(p)) if (row as usize) < p.len() => p.value(row as usize), + match self.at(row) { + Some(( + ArenaPart { + payload: Some(Cols::Arrow(p)), + .. + }, + i, + )) if (i as usize) < p.len() => p.value(i as usize), _ => &[], } } @@ -335,52 +579,78 @@ impl RowArena { /// The row's payload row and its heap (empty if the arena has none or /// holds arrow-row payload). pub(crate) fn payload_row(&self, row: u32) -> (&[u8], &[u8]) { - match &self.payload { - Some(Cols::Rows(p)) => p.row(row), + match self.at(row) { + Some(( + ArenaPart { + payload: Some(Cols::Rows(p)), + .. + }, + i, + )) => p.row(i), _ => (&[], &[]), } } /// The row's weight. pub(crate) fn weight(&self, row: u32) -> i64 { - match self.batch.weights() { - Weights::Const(w) => *w, - Weights::Column(c) => c.value(row as usize), + match self.at(row) { + Some((p, i)) => match p.batch.weights() { + Weights::Const(w) => *w, + Weights::Column(c) => c.value(i as usize), + }, + None => 0, } } /// Whether every weight is 1. pub(crate) fn is_unweighted(&self) -> bool { - self.batch.is_unweighted() + self.parts.iter().all(|p| p.batch.is_unweighted()) } } impl RowSource for RowArena { #[inline] fn prefetch(&self, id: u32) { - if let Cols::Rows(k) = &self.keys { - k.prefetch(id); + if let Some(( + ArenaPart { + keys: Cols::Rows(k), + .. + }, + i, + )) = self.at(id) + { + k.prefetch(i); } } #[inline] fn row(&self, id: u32) -> (&[u8], &[u8]) { - match &self.keys { - Cols::Rows(k) => k.row(id), - Cols::Arrow(_) => (&[], &[]), + match self.at(id) { + Some(( + ArenaPart { + keys: Cols::Rows(k), + .. + }, + i, + )) => k.row(i), + _ => (&[], &[]), } } #[inline] fn hash(&self, id: u32) -> u64 { - self.hashes.values().get(id as usize).copied().unwrap_or(0) + self.at(id) + .and_then(|(p, i)| p.hashes.values().get(i as usize).copied()) + .unwrap_or(0) } #[inline] fn indexed(&self, id: u32) -> bool { - self.null_key - .as_ref() - .is_none_or(|f| (id as usize) >= f.len() || !f.value(id as usize)) + self.at(id).is_none_or(|(p, i)| { + p.null_key + .as_ref() + .is_none_or(|f| (i as usize) >= f.len() || !f.value(i as usize)) + }) } } @@ -429,7 +699,7 @@ enum Index { /// [`KeyTable`] (hashbrown `HashTable`, E02 decision B), or a /// [`JoinIndex`] in the row layout, maps each distinct key to its newest /// row, and older rows with the same key are chained through -/// [`next`](Self::next). +/// [`next_of`](Self::next_of). #[derive(Debug)] pub(crate) struct JoinTable { arena: RowArena, @@ -438,6 +708,10 @@ pub(crate) struct JoinTable { /// One bit per row: a pair with the row matched (empty unless /// [`with_marks`](Self::with_marks)). marks: Vec, + /// No row has an older row with its key (every chain is one row long), + /// so [`next_of`](Self::next_of) never reads the chain (DuckDB's + /// `chains_longer_than_one`). + unique: bool, } impl JoinTable { @@ -561,7 +835,7 @@ impl JoinTable { let mut r = head; while r != NONE { self.mark(r); - r = self.next.get(r as usize).copied().unwrap_or(NONE); + r = self.next_of(r); } } @@ -597,15 +871,17 @@ impl JoinTable { let oom = |e: std::collections::TryReserveError| internal(e.to_string()); next.try_reserve_exact(n).map_err(oom)?; next.resize(n, NONE); - let index = match (arena.arrow_keys(), layout) { - (Some(arrow), _) => { + let index = match (arena.has_arrow_keys(), layout) { + (true, _) => { let mut keys = KeyTable::new(); - keys.reserve(n, arrow.values().len()); + keys.reserve(n, arena.arrow_key_bytes()); let mut heads: Vec = Vec::new(); heads.try_reserve_exact(n).map_err(oom)?; for row in (0..u32::try_from(n).unwrap_or(0)).filter(|&r| arena.indexed(r)) { - let (id, new) = - keys.insert_hashed(arrow.value(row as usize), arena.hash(row))?; + let Some(key) = arena.key_bytes(row) else { + continue; + }; + let (id, new) = keys.insert_hashed(key, arena.hash(row))?; if new { heads.push(row); } else if let Some(head) = heads.get_mut(id as usize) { @@ -616,7 +892,7 @@ impl JoinTable { } Index::Arrow { keys, heads } } - (None, Some(layout)) => Index::Rows { + (false, Some(layout)) => Index::Rows { index: JoinIndex::build_expecting( layout, &arena, @@ -628,13 +904,15 @@ impl JoinTable { )?, layout: Arc::clone(layout), }, - (None, None) => return Err(internal("join key rows without a layout".to_owned())), + (false, None) => return Err(internal("join key rows without a layout".to_owned())), }; + let unique = next.iter().all(|&n| n == NONE); Ok(Self { arena, index, next, marks: Vec::new(), + unique, }) } @@ -718,11 +996,30 @@ impl JoinTable { } } - /// `next()[r]`: the next build row with the same key as `r`, or [`NONE`]. + /// `next()[r]`: the next build row with the same key as `r`, or [`NONE`] + /// (tests; readers go through [`next_of`](Self::next_of)). + #[cfg(test)] pub(crate) fn next(&self) -> &[u32] { &self.next } + /// The row after `row` in its key's chain, or [`NONE`]; on a table whose + /// every key occurs once, `NONE` without reading the chain. + #[inline] + pub(crate) fn next_of(&self, row: u32) -> u32 { + if self.unique { + NONE + } else { + self.next.get(row as usize).copied().unwrap_or(NONE) + } + } + + /// Whether every key occurs once (no chain is longer than one row). + #[cfg(test)] + pub(crate) fn is_unique(&self) -> bool { + self.unique + } + /// The rows. pub(crate) fn arena(&self) -> &RowArena { &self.arena @@ -745,7 +1042,23 @@ mod tests { ], ) .unwrap_or_else(|e| panic!("{e}")); - RowArena::concat(&schema, &[Batch::new(data)]).unwrap_or_else(|e| panic!("{e}")) + RowArena::over_chunks(&schema, &[Batch::new(data)]).unwrap_or_else(|e| panic!("{e}")) + } + + #[test] + fn a_table_of_distinct_keys_is_unique_and_ends_every_chain() { + // Equal hashes, distinct keys: one-row chains only. + let t = JoinTable::build(arena(vec![7, 7, 9], vec![b"a", b"b", b"c"]), None, 0); + let t = t.unwrap_or_else(|e| panic!("{e}")); + assert!(t.is_unique()); + assert!((0..3).all(|r| t.next_of(r) == NONE)); + // Marking a head marks only that row. + let t = t.with_marks().unwrap_or_else(|e| panic!("{e}")); + t.mark_chain(1); + assert_eq!( + (0..3).map(|r| t.is_marked(r)).collect::>(), + [false, true, false] + ); } #[test] @@ -754,6 +1067,8 @@ mod tests { let t = JoinTable::build(arena(vec![7, 7, 7], vec![b"a", b"b", b"a"]), None, 0); let t = t.unwrap_or_else(|e| panic!("{e}")); assert_eq!(t.next(), &[NONE, NONE, 0]); + assert!(!t.is_unique()); + assert_eq!((t.next_of(2), t.next_of(0)), (0, NONE)); assert_eq!(t.arena().len(), 3); assert!(JoinTable::index_bytes(3, 3, false) > 0); assert!(JoinTable::index_bytes(3, 0, true) > 0); @@ -771,76 +1086,221 @@ mod tests { use arrow_array::{ArrayRef, Int64Array, StringArray}; let n = 3000usize; - let ints: ArrayRef = Arc::new(Int64Array::from_iter( - (0..n).map(|i| (i % 7 != 3).then_some((i % 500) as i64)), - )); - let strs: ArrayRef = Arc::new(StringArray::from_iter( - (0..n).map(|i| Some(format!("{}-a-long-enough-string", i % 3))), - )); - let cols = [ints, strs]; - let types = [DataType::Int64, DataType::Utf8]; - let codec = - KeyCodec::with_string_rows(&types, vec![false, true]).unwrap_or_else(|e| panic!("{e}")); - assert!( - KeyCodec::new(&types, vec![false, true]) - .unwrap_or_else(|e| panic!("{e}")) - .layout() - .is_none() - ); - let layout = codec.layout().cloned(); - let tables = |rows: bool, choice: IndexChoice| { - let hashing = Hashing::new(Some(3), rows); - let format = match layout.as_deref().filter(|_| rows) { - Some(keys) => ArenaFormat::Rows { - keys, - payload: None, - arrow_payload: false, - }, - None => ArenaFormat::Arrow { payload: false }, - }; - let schema = arena_schema(format).unwrap_or_else(|e| panic!("{e}")); - let keys = codec - .encode(&cols, &hashing) + // Repeated keys chain; distinct keys (every key once) make a unique + // table in every index kind. + for (m, unique) in [(500usize, false), (n, true)] { + let ints: ArrayRef = Arc::new(Int64Array::from_iter( + (0..n).map(|i| (i % 7 != 3).then_some((i % m) as i64)), + )); + let strs: ArrayRef = Arc::new(StringArray::from_iter( + (0..n).map(|i| Some(format!("{}-a-long-enough-string", i % 3))), + )); + let cols = [ints, strs]; + let types = [DataType::Int64, DataType::Utf8]; + let codec = KeyCodec::with_string_rows(&types, vec![false, true]) .unwrap_or_else(|e| panic!("{e}")); - let valid: Vec = (0..n).filter(|&i| keys.is_valid(i)).collect(); - let chunk = |idx: &[usize]| { - let mut c: Vec = vec![Arc::new( - UInt64Array::from_iter_values(idx.iter().map(|&i| keys.hashes()[i])), - )]; - let l = layout.as_deref().filter(|_| rows); - c.extend(keys.arena_columns(l, idx).unwrap_or_else(|e| panic!("{e}"))); - Batch::new( - RecordBatch::try_new(Arc::clone(&schema), c).unwrap_or_else(|e| panic!("{e}")), - ) + assert!( + KeyCodec::new(&types, vec![false, true]) + .unwrap_or_else(|e| panic!("{e}")) + .layout() + .is_none() + ); + let layout = codec.layout().cloned(); + let tables = |rows: bool, choice: IndexChoice| { + let hashing = Hashing::new(Some(3), rows); + let format = match layout.as_deref().filter(|_| rows) { + Some(keys) => ArenaFormat::Rows { + keys, + payload: None, + arrow_payload: false, + }, + None => ArenaFormat::Arrow { payload: false }, + }; + let schema = arena_schema(format).unwrap_or_else(|e| panic!("{e}")); + let keys = codec + .encode(&cols, &hashing) + .unwrap_or_else(|e| panic!("{e}")); + let valid: Vec = (0..n).filter(|&i| keys.is_valid(i)).collect(); + let chunk = |idx: &[usize]| { + let mut c: Vec = vec![Arc::new( + UInt64Array::from_iter_values(idx.iter().map(|&i| keys.hashes()[i])), + )]; + let l = layout.as_deref().filter(|_| rows); + c.extend(keys.arena_columns(l, idx).unwrap_or_else(|e| panic!("{e}"))); + Batch::new( + RecordBatch::try_new(Arc::clone(&schema), c) + .unwrap_or_else(|e| panic!("{e}")), + ) + }; + let (a, b) = valid.split_at(valid.len() / 3); + let arena = RowArena::over_chunks(&schema, &[chunk(a), chunk(b)]) + .unwrap_or_else(|e| panic!("{e}")); + let l = layout.as_ref().filter(|_| rows); + let t = JoinTable::build_with(arena, l, hashing.g_seed(), choice, None) + .unwrap_or_else(|e| panic!("{e}")); + let probe: Vec = (0..u32::try_from(n).unwrap_or(0)) + .filter(|&i| keys.is_valid(i as usize)) + .collect(); + let mut heads = Vec::new(); + t.find_heads(&keys, &probe, &mut heads); + // A probe in the other format finds nothing. + let other = codec + .encode(&cols, &Hashing::new(Some(3), !rows)) + .unwrap_or_else(|e| panic!("{e}")); + let mut none = Vec::new(); + t.find_heads(&other, &probe[..3], &mut none); + assert_eq!(none, [NONE; 3]); + assert_eq!(t.is_unique(), unique, "modulus {m}"); + (heads, t.next().to_vec(), t.flat_index()) }; - let (a, b) = valid.split_at(valid.len() / 3); - let arena = - RowArena::concat(&schema, &[chunk(a), chunk(b)]).unwrap_or_else(|e| panic!("{e}")); - let l = layout.as_ref().filter(|_| rows); - let t = JoinTable::build_with(arena, l, hashing.g_seed(), choice, None) - .unwrap_or_else(|e| panic!("{e}")); - let probe: Vec = (0..u32::try_from(n).unwrap_or(0)) - .filter(|&i| keys.is_valid(i as usize)) + let (arrow_heads, arrow_next, kind) = tables(false, IndexChoice::Adaptive); + assert_eq!(kind, None); + assert!(arrow_heads.iter().all(|&h| h != NONE)); + for choice in [IndexChoice::Flat, IndexChoice::Salted] { + let (heads, next, kind) = tables(true, choice); + assert_eq!(kind, Some(choice == IndexChoice::Flat)); + assert_eq!(heads, arrow_heads, "{choice:?}"); + assert_eq!(next, arrow_next, "{choice:?}"); + } + } + } + + /// An arena over several chunks (X3' step 3 option C) holds every row of the one batch of the same rows, in blocks + /// of `1 << shift` rows: whole blocks of a chunk first, in order (zero-copy), then the tails joined; with chunks + /// longer and shorter than a block, empty ones and many tiny ones. One chunk keeps its order; the index built over + /// the blocks finds the same keys. + #[test] + fn a_blocked_arena_holds_the_rows_of_one_batch() { + let schema = + arena_schema(ArenaFormat::Arrow { payload: false }).unwrap_or_else(|e| panic!("{e}")); + let n = 600usize; + let key = |i: usize| format!("k{}", i % 97).into_bytes(); + let batch = |rows: std::ops::Range| { + Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(UInt64Array::from_iter_values( + rows.clone().map(|i| i as u64), + )), + Arc::new(LargeBinaryArray::from_iter_values(rows.map(key))), + ], + ) + .unwrap_or_else(|e| panic!("{e}")), + ) + }; + let rows_of = |arena: &RowArena| { + let mut out: Vec<(u64, Vec)> = (0..u32::try_from(arena.len()).unwrap_or(0)) + .map(|r| { + ( + arena.hash(r), + arena.key_bytes(r).unwrap_or_default().to_vec(), + ) + }) .collect(); - let mut heads = Vec::new(); - t.find_heads(&keys, &probe, &mut heads); - // A probe in the other format finds nothing. - let other = codec - .encode(&cols, &Hashing::new(Some(3), !rows)) - .unwrap_or_else(|e| panic!("{e}")); - let mut none = Vec::new(); - t.find_heads(&other, &probe[..3], &mut none); - assert_eq!(none, [NONE; 3]); - (heads, t.next().to_vec(), t.flat_index()) + out.sort(); + out }; - let (arrow_heads, arrow_next, kind) = tables(false, IndexChoice::Adaptive); - assert_eq!(kind, None); - assert!(arrow_heads.iter().all(|&h| h != NONE)); - for choice in [IndexChoice::Flat, IndexChoice::Salted] { - let (heads, next, kind) = tables(true, choice); - assert_eq!(kind, Some(choice == IndexChoice::Flat)); - assert_eq!(heads, arrow_heads, "{choice:?}"); - assert_eq!(next, arrow_next, "{choice:?}"); + let one = RowArena::over_chunks(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")); + let all = rows_of(&one); + // One chunk, longer than a block: sliced in order, nothing copied. + let sliced = + RowArena::with_shift(&schema, &[batch(0..n)], 4).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(sliced.parts.len(), n.div_ceil(16)); + for row in 0..u32::try_from(n).unwrap_or(0) { + assert_eq!(sliced.hash(row), u64::from(row), "row {row}"); + } + assert_eq!(RowArena::reblock_bytes(&[batch(0..n)], 0), (0, 0)); + // Its key bytes are counted once, not once per block of the chunk. + assert_eq!(sliced.arrow_key_bytes(), one.arrow_key_bytes()); + let uneven: Vec = [ + 0..1, + 1..1, + 1..64, + 64..128, + 128..250, + 250..251, + 251..251, + 251..n, + ] + .into_iter() + .map(batch) + .collect(); + let many: Vec = (0..n / 2).map(|c| batch(2 * c..2 * c + 2)).collect(); + for chunks in [uneven, many] { + let arena = RowArena::with_shift(&schema, &chunks, 4).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(arena.len(), n); + assert_eq!(rows_of(&arena), all); + // Every block but the last is full. + let (last, full) = arena + .parts + .split_last() + .unwrap_or_else(|| panic!("no blocks")); + assert!(full.iter().all(|p| p.rows == 16)); + assert!((1..=16).contains(&last.rows)); + for row in 0..u32::try_from(n).unwrap_or(0) { + assert_eq!(arena.weight(row), 1, "row {row}"); + } + // Past the end: nothing, not a panic. + let end = u32::try_from(n).unwrap_or(0); + assert_eq!(arena.hash(end), 0); + assert_eq!(arena.key_bytes(end), None); + assert!(arena.memory_split().hash > 0); + let chained = JoinTable::build(arena, None, 0).unwrap_or_else(|e| panic!("{e}")); + let single = JoinTable::build( + RowArena::over_chunks(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")), + None, + 0, + ) + .unwrap_or_else(|e| panic!("{e}")); + assert_eq!(chained.index_size().0, single.index_size().0); + } + // Block-aligned chunks with tails between them (a partition's drivers' lists): the blocks are not copied, + // only the tails are. + let block = ARENA_ROWS; + let lists = [ + batch(0..block), + batch(block..block + 5), + batch(block + 5..3 * block + 8), + batch(3 * block + 8..3 * block + 12), + ]; + let arena = RowArena::over_chunks(&schema, &lists).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(arena.parts.len(), 4); + assert_eq!(arena.len(), 3 * block + 12); + // The two tails: one of a chunk that also gives blocks (kept beside it), one of a chunk the copy replaces. + let (copy, kept) = RowArena::reblock_bytes(&lists, 0); + assert!(kept > 0 && kept < copy && copy < 2048, "{copy} {kept}"); + assert_eq!( + RowArena::reblock_bytes(&lists, 4), + (copy, copy - 3 * 3 * 64) + ); + assert_eq!(arena.hash(0), 0); + assert_eq!( + arena.hash(u32::try_from(block).unwrap_or(0)), + (block + 5) as u64 + ); + assert_eq!( + arena.hash(u32::try_from(3 * block).unwrap_or(0)), + block as u64 + ); + } + + /// An arena of no rows (a grace partition whose build side is empty, its + /// chunks all empty or none at all) keeps its key format, builds a table and + /// finds nothing (Fable review of X3' step 3, blocker 1). + #[test] + fn an_empty_arena_builds_and_matches_nothing() { + let schema = + arena_schema(ArenaFormat::Arrow { payload: false }).unwrap_or_else(|e| panic!("{e}")); + let empty = Batch::new(RecordBatch::new_empty(Arc::clone(&schema))); + for chunks in [vec![], vec![empty.clone()], vec![empty.clone(), empty]] { + let arena = RowArena::over_chunks(&schema, &chunks).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(arena.len(), 0); + assert!(arena.has_arrow_keys()); + assert_eq!(arena.hash(0), 0); + assert_eq!(arena.key_bytes(0), None); + let table = JoinTable::build(arena, None, 0).unwrap_or_else(|e| panic!("{e}")); + assert!(table.next().is_empty()); } } @@ -858,7 +1318,7 @@ mod tests { ], ) .unwrap_or_else(|e| panic!("{e}")); - assert!(RowArena::concat(&schema, &[Batch::new(data)]).is_err()); + assert!(RowArena::over_chunks(&schema, &[Batch::new(data)]).is_err()); let rows = arena(vec![1], vec![b"a"]); assert_eq!(RowSource::row(&rows, 0), (&[][..], &[][..])); assert_eq!(RowSource::hash(&rows, 9), 0); diff --git a/crates/krishiv-exec/src/rows/flooding_tests.rs b/crates/krishiv-exec/src/rows/flooding_tests.rs index ee816ab8d..17a2171a4 100644 --- a/crates/krishiv-exec/src/rows/flooding_tests.rs +++ b/crates/krishiv-exec/src/rows/flooding_tests.rs @@ -157,10 +157,12 @@ fn crafted_partial_collisions_scatter() { /// `m` distinct keys forged to one full 64-bit `h`, among ordinary keys: /// every key is still found, absent keys with the same `h` are not, the -/// colliding keys' probe run grows linearly in `m` (at load 1/2 the -/// cluster takes in about as many neighbours: at most `2m` plus an -/// ordinary run); only the ordinary keys next to the cluster get longer -/// runs, so their total grows by `O(m²)`, whatever the table size. +/// colliding keys' probe run grows linearly in `m` (at this test's load, +/// about 1/2, the cluster takes in about as many neighbours: at most `2m` +/// plus an ordinary run; a directory near its 4/5 bound takes in more, still +/// linear in `m`, which this test does not measure); only the ordinary keys +/// next to the cluster get longer runs, so their total grows by `O(m²)`, +/// whatever the table size. #[test] fn full_collisions_stay_correct_and_bounded() { const BASE: usize = 3840; @@ -240,3 +242,28 @@ fn a_flat_index_has_no_probe_runs() { assert!(index.is_flat()); assert_eq!(index.probe_run(enc.hashes()[0]), None); } + +/// The directory is sized for its keys at load ≤ 4/5 and never grows: at +/// key counts where 5/4 of them lands exactly on a power of two and they are +/// not a multiple of 4 (1,639: 1,639 + 409 = 2,048), rounding 5/4 down left +/// the last key past the load bound, and the directory doubled with no +/// reservation behind it (review of S5's 5/4 sizing). +#[test] +fn the_directory_holds_its_keys_without_growing() { + let layout = layout(); + for n in [1_639_usize, 1_638, 1_640, 3_277, 6_554] { + let keys: Vec = (0..i64::try_from(n).unwrap_or(0)).collect(); + let (index, _) = salted(&layout, &keys, &[], 0, 7); + assert_eq!(index.len(), n); + assert!( + 5 * n <= 4 * index.buckets(), + "{n} keys in {}", + index.buckets() + ); + assert_eq!( + index.buckets(), + salted_bytes(n) / 8, + "{n} keys grew the directory" + ); + } +} diff --git a/crates/krishiv-exec/src/rows/index.rs b/crates/krishiv-exec/src/rows/index.rs index d2fef0496..9e145863d 100644 --- a/crates/krishiv-exec/src/rows/index.rs +++ b/crates/krishiv-exec/src/rows/index.rs @@ -7,7 +7,7 @@ //! enough to stay in a core's own cache, where its lower instruction count //! wins; //! - **salted**: a power-of-two directory of `u64` slots, each a 16-bit -//! salt plus `row id + 1`, probed linearly at load ≤ 1/2 (DuckDB's +//! salt plus `row id + 1`, probed linearly at load ≤ 4/5 (DuckDB's //! `ht_entry_t`, with row ids instead of pointers), above that size, //! where a probe costs one directory miss and reads a row only when its //! salt matches. Its lookups prefetch each chunk's slots, then its first @@ -123,9 +123,19 @@ pub fn flat_bytes(n: usize) -> usize { .saturating_add(16) } -/// Slots of a salted directory for `n` keys at load ≤ 1/2. +/// Load of a salted directory at most `SALTED_LOAD_NUM / SALTED_LOAD_DEN` +/// (4/5): the directory is the next power of two of 5/4 of its keys +/// (DuckDB's load factor; study 21 §5: `next_pow2(2 x keys)` landed at +/// 3.7 slots per key on q21's 145 M-key build at SF1000, a third of its +/// table's bytes), so a filled one sits between 2/5 and 4/5. The 16-bit +/// salt keeps a probe from reading a row for all but 1 in 65,536 of the +/// slots it passes. +const SALTED_LOAD_NUM: usize = 4; +const SALTED_LOAD_DEN: usize = 5; + +/// Slots of a salted directory for `n` keys at load ≤ 4/5. fn salted_slots(n: usize) -> usize { - n.saturating_mul(2) + n.saturating_add(n.div_ceil(SALTED_LOAD_NUM)) .checked_next_power_of_two() .unwrap_or(usize::MAX) .max(MIN_SLOTS) @@ -244,7 +254,7 @@ impl JoinIndex { slots.resize(len, 0); let mut mask = len - 1; for id in (0..n).filter(|&id| src.indexed(id)) { - if (keys + 1) * 2 > slots.len() { + if (keys + 1) * SALTED_LOAD_DEN > slots.len() * SALTED_LOAD_NUM { slots = grow_salted(&slots, g_of)?; mask = slots.len() - 1; } @@ -482,7 +492,8 @@ fn grow_salted(old: &[u64], g: impl Fn(u32) -> u64) -> Result> { /// its salt, so their cache misses overlap before the chunk is resolved in /// order. It reads only the first slot: on a probe that mostly misses the salt /// test is well predicted, where walking the run takes the `s == 0` branch a -/// second time, about even at load 1/2 (TPC-DS q72). An empty slot of salt 0 +/// second time, about even at load 1/2 (TPC-DS q72; measured before the +/// directory moved to load up to 4/5). An empty slot of salt 0 /// prefetches row 0, a harmless hint. fn prefetch_chunk(slots: &[u64], gs: &[u64], src: &impl RowSource) { let mask = slots.len() - 1; diff --git a/crates/krishiv-exec/tests/hash_join/main.rs b/crates/krishiv-exec/tests/hash_join/main.rs index e3951c750..599f2a810 100644 --- a/crates/krishiv-exec/tests/hash_join/main.rs +++ b/crates/krishiv-exec/tests/hash_join/main.rs @@ -981,3 +981,53 @@ async fn a_build_that_fits_with_one_partition_copy_completes() { ); assert_eq!(spill_files(&scope), 0); } + +/// Partitions whose chunks hold whole arena blocks (2^14 rows) keep the +/// copied tails beside them through the probe. A spilling build reserves +/// those up front with the indexes, so the copy it holds for one partition +/// at a time is never used up by kept tails: under every limit from the +/// in-memory peak down, the build either fits or spills, and never fails. +#[tokio::test(start_paused = true)] +async fn a_spilling_build_with_kept_tails_fits_or_spills_under_any_limit() { + // Two input batches over distinct keys: each of the 8 level-0 + // partitions gets two chunks of about 24,400 rows, a whole block and a + // tail of about 8,000 rows each. The two tails make one copied block, + // kept beside the chunks: a third of the build is kept tails. + let build = skewed(2 * 8 * 24_384, 1 << 30); + let chunk = build.len() / 2; + let (expect, q) = run_self_join(&build, chunk, 512 * MIB, None).await; + let expect = expect.unwrap(); + let single = peak(&q); + for step in 0..12 { + let limit = single - single * step / 32; + let (_manager, scope) = spill(); + let (got, _) = run_self_join(&build, chunk, limit, Some(scope)).await; + match got { + Ok(got) => assert!(got == expect, "the join result differs at {limit}"), + Err(e) => panic!("the build failed at limit {limit} ({single} in memory): {e}"), + } + } +} + +/// A build of many small batches leaves every partition's rows in chunk +/// tails shorter than a block, each reserved at its allocated size +/// (builder slack). Finalize copies the tails into exact blocks and drops +/// the chunks, so what the table keeps reserved through the probe is +/// trued up to the arena's allocated bytes, not the chunks' (study 21 +/// §4.4: ~27 MB of such slack per build at 12 threads). +#[tokio::test(start_paused = true)] +async fn a_built_table_keeps_reserved_what_its_arena_holds() { + let build = skewed(40_000, 40_000); + // Batches of 256 rows: about 32 rows per level-0 partition and push. + let (got, q) = run_self_join(&build, 256, 512 * MIB, None).await; + got.unwrap(); + let steady = metric(&q, "HashJoinBuild", "steady_reserved_bytes"); + let arena = metric(&q, "HashJoinBuild", "arena_bytes"); + assert!(arena > 0); + // Exact blocks round each buffer up to 64 bytes, nothing more; and + // the true-up never leaves the arena under-reserved. + assert!( + steady >= arena - arena / 20 && steady <= arena + arena / 20, + "steady {steady} B against an arena of {arena} B" + ); +} diff --git a/crates/krishiv-exec/tests/planner/late_payload.rs b/crates/krishiv-exec/tests/planner/late_payload.rs new file mode 100644 index 000000000..c0272a834 --- /dev/null +++ b/crates/krishiv-exec/tests/planner/late_payload.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The late payload join rule (P2b S5 lever (2), optimizer `late_payload`): +//! where it moves a dimension's payload above the semi and anti joins, the +//! executed plan returns the rows of the same plan with the rule off. +//! +//! The data set is the committed TPC-H SF0.01 Parquet fixture (the design note +//! asked for SF0.1, which is not committed). + +use std::path::Path; + +use super::partial_aggregate::{physical, runtime, sorted}; +use super::run; +use super::tpch::{parquet_tables, sql}; + +/// TPC-H q21 and q18, and queries whose dimension key repeats (five nations a +/// region), with the rule on and off over the same statistics. +#[test] +fn moved_payload_joins_return_the_rows_of_the_rule_off() { + let rt = runtime(); + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tpch-sf0.01-parquet"); + let t = parquet_tables(&dir, &rt); + let queries = [ + sql(21), + sql(18), + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name" + .to_owned(), + "SELECT n_name, count(*), sum(s_acctbal) FROM supplier, nation \ + WHERE s_nationkey = n_regionkey \ + AND NOT EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey \ + AND l_comment LIKE '%xyzzy%') GROUP BY n_name" + .to_owned(), + ]; + // A large query's semi and anti joins are not broadcast; at SF0.01 every + // one would be, and then none builds the path. + let on = [("spark.sql.autoBroadcastJoinThreshold", "-1")]; + let off = [ + ("spark.sql.autoBroadcastJoinThreshold", "-1"), + ("krishiv.optimizer.latePayloadJoin", "false"), + ]; + for q in &queries { + let (with, steps) = t.optimized_in(q, &on).unwrap(); + let (without, _) = t.optimized_in(q, &off).unwrap(); + let notes: Vec<&String> = steps + .iter() + .filter(|s| s.batch == "LatePayloadJoin") + .map(|s| &s.note) + .collect(); + // The rule fired: it is the only difference between the two plans. + assert!( + notes.iter().any(|n| n.starts_with("moved")) && !with.structurally_eq(&without), + "{q}\n{notes:?}\n{}", + with.root() + ); + let got = sorted(&run(&physical(&t, &with).unwrap()).unwrap()); + let want = sorted(&run(&physical(&t, &without).unwrap()).unwrap()); + assert!(!want.is_empty(), "{q}"); + assert_eq!(got, want, "{q}\n{}", with.root()); + } +} diff --git a/crates/krishiv-exec/tests/planner/main.rs b/crates/krishiv-exec/tests/planner/main.rs index 203ab5ece..2b33a7cb9 100644 --- a/crates/krishiv-exec/tests/planner/main.rs +++ b/crates/krishiv-exec/tests/planner/main.rs @@ -25,6 +25,7 @@ mod group_code; mod group_limit; mod hints; mod late_fetch; +mod late_payload; mod listing; mod orientation; mod partial_aggregate; diff --git a/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap b/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap index da701cfa2..6417177c5 100644 --- a/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap +++ b/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap @@ -5,27 +5,32 @@ expression: tpch.plan(q).to_string() TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] +- Exchange SinglePartition +- LocalTopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] - +- Project [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21)#4611686018427387946 AS sum(l_quantity)#50] + +- Project [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21)#4611686018427387952 AS sum(l_quantity)#50] +- HashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] +- Exchange hashpartitioning(c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11) +- PartialHashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] - +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey - :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey, probe filter: isnotnull(l_orderkey#17) - : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] - : +- BroadcastExchange - : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey, probe filter: (isnotnull(o_custkey#9) and isnotnull(o_orderkey#8)) - : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12], PushedFilters: [(isnotnull(o_custkey#9) and isnotnull(o_orderkey#8))] - : +- BroadcastExchange - : +- Filter isnotnull(c_custkey#0) - : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1], PushedFilters: [isnotnull(c_custkey#0)] - +- BroadcastExchange - +- Project [l_orderkey#33] - +- Filter (sum(l_quantity)#49 > 300.00) - +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387943 AS sum(l_quantity)#49] - +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - +- Exchange hashpartitioning(l_orderkey#33) - +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - +- Filter isnotnull(l_orderkey#17) (ids +16) - +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] + +- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] + +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #4: o_custkey#9 -> spark_catalog.default.orders.o_custkey + :- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey + : :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, o_custkey#9] + : : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey, probe filter: isnotnull(l_orderkey#17) + : : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] + : : +- BroadcastExchange + : : +- HashJoin LeftSemi, [o_custkey#9] = [c_custkey#51], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey, probe filter: (isnotnull(o_custkey#9) and isnotnull(o_orderkey#8)) + : : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12], PushedFilters: [(isnotnull(o_custkey#9) and isnotnull(o_orderkey#8))] + : : +- BroadcastExchange + : : +- Project [c_custkey#0] (ids +51) + : : +- Filter isnotnull(c_custkey#0) + : : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1], PushedFilters: [isnotnull(c_custkey#0)] + : +- BroadcastExchange + : +- Project [l_orderkey#33] + : +- Filter (sum(l_quantity)#49 > 300.00) + : +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387943 AS sum(l_quantity)#49] + : +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : +- Exchange hashpartitioning(l_orderkey#33) + : +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : +- Filter isnotnull(l_orderkey#17) (ids +16) + : +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] + +- BroadcastExchange + +- Filter isnotnull(c_custkey#0) + +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1], PushedFilters: [isnotnull(c_custkey#0)] diff --git a/crates/krishiv-exec/tests/planner/tpch.rs b/crates/krishiv-exec/tests/planner/tpch.rs index f8d418ca1..d475caedd 100644 --- a/crates/krishiv-exec/tests/planner/tpch.rs +++ b/crates/krishiv-exec/tests/planner/tpch.rs @@ -221,6 +221,30 @@ impl Tpch { }) } + /// `text` parsed, analyzed and optimized by the tables' statistics under + /// the session `settings`, with the optimizer's trace. + pub(crate) fn optimized_in( + &self, + text: &str, + settings: &[(&str, &str)], + ) -> Result<(Plan, Vec), KrishivError> { + let plan = + krishiv_sql::parse_and_lower(text, &krishiv_sql::ParserOptions::default(), SourceId(0)) + .map_err(KrishivError::from)?; + let plan: Plan = block_on(DefaultAnalyzer.analyze(&plan, &self.ctx))?.plan; + let stats = ProviderStatistics(Arc::clone(&self.tables) as _); + let mut config = Config::new(); + for (k, v) in settings { + config.set(k, v)?; + } + krishiv_optimizer::optimize_traced_with_udfs( + &plan, + &stats, + &krishiv_udf_sdk::UdfLibrary::new(), + &config, + ) + } + /// As [`Self::plan_sql_with`], its operators spilling into `spill`. pub(crate) fn plan_sql_in( &self, diff --git a/crates/krishiv-optimizer/CLAUDE.md b/crates/krishiv-optimizer/CLAUDE.md index 07536089f..c09b25bb7 100644 --- a/crates/krishiv-optimizer/CLAUDE.md +++ b/crates/krishiv-optimizer/CLAUDE.md @@ -19,6 +19,7 @@ Paths are relative to this crate. - `src/win_magic.rs` `SubqueryAsWindow` (P3 W1 shape (a), form M): a decorrelated `min`/`max`/decimal `avg` subquery over the outer block's own relations becomes a `Window` over the outer rows (`krishiv.optimizer.subqueryAsWindow`, on; q17, q02; refusals in `tests/it/win_magic.rs`; guarded by estimated window bytes, priced on the join-ordered outer block, <= second-scan bytes; no firing without statistics) - `src/eager.rs` `EagerAggregate` (P2-T167): an aggregate's calls run below a join and merge above it, gated on statistics; `krishiv.optimizer.eagerAggregation`, `Options::eager_aggregation` - `src/group_code.rs` `GroupKeyCode` (design P2b D2): an aggregate's dimension group keys travel as one integer code (`row_number` over the dimension's distinct keys, read twice as `Shift`s of one body; `D_c` is a `Shift` of offset 0, the fence join ordering does not look through), decoded by a join above the aggregate; statistics-gated (rows(I) read through a null-aware `NOT IN` anti join only, `gate_rows`), one dimension per aggregate, `D`'s filters may not raise; a decode join it adds is ordered by `order_and_restrict` at once; `krishiv.optimizer.groupKeyCode`, `Options::group_key_code`; tests `tests/it/group_code.rs`, executor differential `krishiv-exec` `tests/planner/group_code.rs` +- `src/late_payload.rs` `LatePayloadJoin` (P2b S5 lever (2), design `design/P2b-S5-dimension-payload-late.md`): a dimension's payload columns join above the semi/anti/outer joins that never read them, a key-only semi join left in place; runs after pruning, statistics only; `krishiv.optimizer.latePayloadJoin`, `Options::late_payload_join` - `src/join_order.rs` inner-join ordering (estimates the whole plan first, so a `CteRef` in a tree reads its `WITH` definition; enumerator choice `krishiv.optimizer.joinEnumerator` / `joinDpLimit`, a `TreeStats` note per tree in `EXPLAIN VERBOSE`); `src/join_order/dp.rs` the exact enumerator DPsub over a hypergraph of input masks (P2-T113 kept it over DPhyp: `exp/t113-dphyp`); `src/join_order/reads.rs` which columns are read above a tree, for `krishiv.optimizer.joinCost=bytes` (E5b: a join costs its result rows plus its build's bytes, the fewer-rows input stays the build within 2x of the other's bytes; default `rows`, `tests/it/join_cost.rs`); `src/stats.rs` `Statistics` it reads and the estimate types (`RelStats`, `AttrStats`, `StatSource`) - `src/estimate.rs` the `Estimator`: what each operator leaves known of its columns (a column a filter or join does not read keeps the distinct values the kept rows are expected to hold, `kept_distinct`, Cardenas); `src/narrow.rs` predicates over estimates (selectivity and narrowed ranges); `src/plan_stats.rs` `PlanStats`, one pass over an optimized plan (physical planner, `EXPLAIN COST`) - `tests/it/plan_stats.rs` one test per propagation rule; `tests/it/qerror.rs` the q-error report (`cargo xtask qerror`) @@ -40,6 +41,7 @@ Paths are relative to this crate. decimals are never rewritten. Conditions and the one residual assumption (fewer than 2^46 rows per aggregate) are in the module docs; the executor differential test is `krishiv-exec` `tests/planner/sum_rewrite.rs`. - Eager aggregation moves only calls whose result cannot depend on row order and whose arguments cannot raise (`eager.rs` module docs); it reads statistics only, runs after join ordering, and states each decision in `EXPLAIN VERBOSE`. The executor differential is `krishiv-exec` `tests/planner/eager_aggregate.rs`. +- Late payload joins move an inner equi-join only above filters, projections and joins that do not read the dimension's columns and are deterministic, with at least one non-inner join on the path (join ordering never moves an inner join back across one: fixed point); exact for any key multiplicity (join commutation, no uniqueness assumed); the dimension must be `restrict::repeatable` (computed twice). Tests: `tests/it/late_payload.rs`. - Join ordering re-plans each inner-join tree (through deterministic filters and column selections) from `Statistics`; it never adds a cross product, places each conjunct at the lowest join that has its inputs (subquery conjuncts as filters), and restores column order only for positional consumers. Inputs are diff --git a/crates/krishiv-optimizer/src/late_payload.rs b/crates/krishiv-optimizer/src/late_payload.rs new file mode 100644 index 000000000..db0332c80 --- /dev/null +++ b/crates/krishiv-optimizer/src/late_payload.rs @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `LatePayloadJoin`: a dimension's payload columns join above the semi and +//! anti joins that never read them (P2b S5 lever (2); design +//! `design/P2b-S5-dimension-payload-late.md` in krishiv-internal). +//! +//! ```text +//! Consumer(Path(X ⋈[x = d] D)) +//! => Consumer(Project[out(Path)](Path'(X ⋉[x = d'] Shift(π_d D)) ⋈[x = d] D)) +//! ``` +//! +//! `D` is one side of an inner equi-join whose other columns (the payload, +//! TPC-H q21's `s_name`) ride through every build of the path above it and are +//! first read by the consumer. Each operator on the path commutes with an +//! inner join on columns it does not read: `(X ⋈ D) ⋉ L = (X ⋉ L) ⋈ D` when +//! `L`'s condition reads only `X`, the same for an anti join, an existence +//! join and the preserved side of an outer join, and filters and projections +//! trivially. So the join moves to the top of the path with the same rows, +//! each `X` row joined to every matching `D` row whatever the multiplicity; +//! no key needs to be unique. The semi join left in its place drops only the +//! rows the top join drops (no match, NULL keys included), so the path keeps +//! the reduction the dimension gave it. `D` is computed twice, the semi +//! join's instance under a [`PlanNode::Shift`] as in `restrict`, so it must +//! be repeatable (`restrict::repeatable`). +//! +//! # When it applies +//! +//! - The join is inner, its condition only plain `=` of a column of each side, +//! of one type whose equality and grouping agree; +//! - the path from the join to its consumer holds only filters, projections +//! and joins that do not read the payload or `D`'s key columns, with the path +//! on the side whose rows they keep (inner either side; left semi, anti, +//! existence and outer on the left; right outer on the right), and every +//! expression on it is deterministic (a filter evaluated once per `X` row +//! instead of once per joined copy would sample differently); +//! - at least one join on the path is not inner: join ordering never moves an +//! inner join across one, so it cannot undo the rule, and once applied the +//! rule finds no such join between the moved join and its consumer: the +//! plan stays a fixed point. An all-inner path is join ordering's business; +//! - the join and the path's nodes are read by no other parent (rebuilding +//! them would copy a shared subtree); +//! - gates, from statistics only (both sides' estimates `from_statistics`, +//! never default row counts): the payload the path carries holds a +//! variable-width column or at least [`MIN_PAYLOAD_BYTES`] a row; `D` +//! estimates no more rows than `X`, and its scans read no more rows than +//! `X` holds (it is computed twice); and some join on the path builds the +//! path's side as the physical planner orients it (`builds_path`: an inner +//! join's right input, or the left input of a semi, anti or left outer join +//! whose right input costs more than twice as much and is not broadcast). A +//! payload on a probe side only streams, and moving it would only add work. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use krishiv_common::error::KrishivError; +use krishiv_plan::{ + AttrId, BoundColumn, Builtin, Expr, JoinCondition, JoinKind, LogicalPlan, PlanNode, +}; +use krishiv_types::DataType; + +use crate::dag; +use crate::estimate::Estimator; +use crate::expr::{ + Attrs, builtin, column, conjuncts, is_deterministic, output_of, recall, references, +}; +use crate::restrict::{MAX_ID, key_type, max_id, repeatable, shift}; +use crate::stats::Statistics; + +/// The least fixed width, in bytes a row, of a payload worth moving. +pub(crate) const MIN_PAYLOAD_BYTES: usize = 16; + +/// The most joins moved below one consumer. +const MAX_MOVES: usize = 16; + +/// The most plan nodes one walk down from a consumer visits. +const MAX_WALK: usize = 256; + +/// Moves dimension payload joins above the non-inner joins of their path, +/// entering subquery plans. Also returns one line per move, and per refused +/// candidate the gate that refused it (`EXPLAIN VERBOSE`). +/// +/// `broadcast_max` is the physical planner's broadcast threshold +/// (`spark.sql.autoBroadcastJoinThreshold`, `None` when off): a join that +/// broadcasts its other side never builds the path. +pub(crate) fn rewrite( + root: Arc, + stats: &dyn Statistics, + propagate: bool, + broadcast_max: Option, +) -> Result<(Arc, Vec), KrishivError> { + let mut ctx = Context { + next: max_id(&root, &mut crate::expr::AddrMap::default()).saturating_add(1), + notes: Vec::new(), + shared: shared_nodes(&root), + broadcast_max, + }; + let mut est = Estimator::with_propagation(stats, propagate); + let out = dag::rewrite_up(root, &mut |p| { + // Several dimensions may carry payloads to one consumer: move each, + // so one pass reaches the fixed point. + let mut at = p; + for _ in 0..MAX_MOVES { + match at_consumer(&at, &mut est, &mut ctx) { + Some(moved) => at = moved, + None => break, + } + } + Ok(at) + })?; + Ok((out, ctx.notes)) +} + +/// What one pass carries between consumers. +struct Context { + /// The next free attribute id. + next: u64, + /// `EXPLAIN VERBOSE` lines, each once. + notes: Vec, + /// Nodes read by more than one parent in the input plan. + shared: HashSet<*const LogicalPlan>, + broadcast_max: Option, +} + +/// Appends `line`; a refusal only once (the moves below one consumer walk +/// the same refused candidates again), every move. +fn note(notes: &mut Vec, line: String) { + if line.starts_with("moved") || !notes.contains(&line) { + notes.push(line); + } +} + +/// The nodes of `root` that more than one parent reads (subquery plans not +/// entered: the rule rewrites inside them as their own roots). +fn shared_nodes(root: &Arc) -> HashSet<*const LogicalPlan> { + let mut parents: HashMap<*const LogicalPlan, usize> = HashMap::new(); + let mut seen = HashSet::new(); + let mut stack = vec![Arc::clone(root)]; + while let Some(p) = stack.pop() { + if !seen.insert(Arc::as_ptr(&p)) { + continue; + } + p.for_each_child(&mut |c| { + *parents.entry(Arc::as_ptr(c)).or_default() += 1; + stack.push(Arc::clone(c)); + }); + } + parents + .into_iter() + .filter(|(_, n)| *n > 1) + .map(|(p, _)| p) + .collect() +} + +/// Whether the rule can put a new child under `p` (`with_child`). +fn replaceable(p: &LogicalPlan) -> bool { + matches!( + p.node, + PlanNode::Join { .. } + | PlanNode::Project { .. } + | PlanNode::Filter { .. } + | PlanNode::Aggregate { .. } + | PlanNode::Sort { .. } + | PlanNode::Limit { .. } + | PlanNode::Offset { .. } + | PlanNode::SubqueryAlias { .. } + | PlanNode::WithCte { .. } + | PlanNode::Shift { .. } + | PlanNode::JoinHint { .. } + | PlanNode::Repartition { .. } + | PlanNode::Window { .. } + | PlanNode::Generate { .. } + | PlanNode::ReplicateRows { .. } + ) +} + +/// How much larger the other side's table must be before the physical +/// planner builds a semi, anti or left outer join's left input +/// (`krishiv-exec` `planner/orient.rs` `STATS_FACTOR`, both sides from +/// statistics; `DEFAULT_FACTOR` otherwise). +const STATS_FACTOR: f64 = 2.0; +const DEFAULT_FACTOR: f64 = 8.0; + +/// Whether the physical planner builds the hash table of `s`'s join from the +/// path's input (`krishiv-exec` `planner/join.rs` and `planner/orient.rs`): +/// an inner join's right input, or the left input of a semi, anti (not +/// null-aware) or left outer join whose right input costs more than the +/// factor times as much and is not broadcast. Existence, null-aware anti and +/// right outer joins never build their left input. +fn builds_path(s: &Step, est: &mut Estimator<'_>, broadcast_max: Option) -> bool { + let PlanNode::Join { + kind, + condition: JoinCondition::On(on), + .. + } = &s.node.node + else { + return false; + }; + let kids = children(&s.node); + let (Some(path), Some(other)) = (kids.get(s.child), kids.get(usize::from(s.child == 0))) else { + return false; + }; + match kind { + JoinKind::Inner => s.child == 1, + JoinKind::LeftSemi | JoinKind::LeftOuter => { + s.child == 0 && flips(path, other, est, broadcast_max) + } + JoinKind::LeftAnti => { + s.child == 0 && !null_aware(on) && flips(path, other, est, broadcast_max) + } + _ => false, + } +} + +/// `orient.rs`'s decision to build the left input over `right`. +fn flips( + left: &Arc, + right: &Arc, + est: &mut Estimator<'_>, + broadcast_max: Option, +) -> bool { + let (l, r) = (est.plan(left), est.plan(right)); + let (lb, rb) = (bytes(left, l.rows), bytes(right, r.rows)); + if broadcast_max.is_some_and(|max| rb <= max) { + return false; + } + let factor = if l.from_statistics && r.from_statistics { + STATS_FACTOR + } else { + DEFAULT_FACTOR + }; + rb > factor * lb +} + +/// Spark's size estimate of `rows` rows of `p`'s output. +fn bytes(p: &LogicalPlan, rows: f64) -> f64 { + let width = p + .resolved_output() + .unwrap_or_default() + .iter() + .map(|c| crate::plan_stats::default_size(&c.data_type)) + .fold(8, u64::saturating_add); + #[expect(clippy::cast_precision_loss, reason = "an estimate")] + let width = width as f64; + rows.max(0.0) * width +} + +/// Whether `on` is Spark's null-aware anti-join key `(a = b) OR isnull(a = b)`. +fn null_aware(on: &Arc) -> bool { + let conj = conjuncts(on); + let [only] = conj.as_slice() else { + return false; + }; + matches!(builtin(only), Some((Builtin::Or, [_, is_null], _)) + if matches!(builtin(is_null), Some((Builtin::IsNull, _, _)))) +} + +/// The rows the scans of `p` read, as estimated. +fn scanned_rows(p: &Arc, est: &mut Estimator<'_>) -> f64 { + let mut total = 0.0; + let mut seen = HashSet::new(); + let mut stack = vec![Arc::clone(p)]; + while let Some(x) = stack.pop() { + if !seen.insert(Arc::as_ptr(&x)) { + continue; + } + if matches!(x.node, PlanNode::Scan(_)) { + total += est.plan(&x).rows; + } + x.for_each_child(&mut |c| stack.push(Arc::clone(c))); + } + total +} + +/// One step of a path: a node and the index of the child the path came from. +#[derive(Clone)] +struct Step { + node: Arc, + child: usize, +} + +/// A join that may move: the path from just above it to the consumer's child +/// (bottom first), the join, and which side is the dimension. +struct Candidate { + path: Vec, + join: Arc, + dim_left: bool, +} + +/// `consumer` with one payload join below it moved up, if any qualifies. +fn at_consumer( + consumer: &Arc, + est: &mut Estimator<'_>, + ctx: &mut Context, +) -> Option> { + let kids = children(consumer); + if !replaceable(consumer) { + return None; + } + let (next, notes) = (&mut ctx.next, &mut ctx.notes); + for (i, kid) in kids.iter().enumerate() { + let mut found = Vec::new(); + let mut budget = MAX_WALK; + walk(kid, &mut Vec::new(), false, &mut found, &mut budget); + for cand in found { + if cand + .path + .iter() + .any(|s| ctx.shared.contains(&Arc::as_ptr(&s.node))) + || ctx.shared.contains(&Arc::as_ptr(&cand.join)) + { + // Rebuilding a node other parents read would copy it. + continue; + } + let Some(shape) = Shape::of(&cand, ctx.broadcast_max) else { + continue; + }; + // The consumer must be where the path ends for this payload: a + // node the payload could pass is not the consumer (its parent is). + if passes(consumer, i, &shape.blocked) { + continue; + } + // Nothing of the dimension above the join: nothing to move. + if shape.carried(&cand).is_empty() { + continue; + } + if let Err(why) = shape.gates(&cand, est) { + note( + notes, + format!("not applied to the join on [{}]: {why}", shape.keys_text()), + ); + continue; + } + let Some(moved) = shape.moved(&cand, kid, next) else { + note( + notes, + "not applied: the plan has no attribute ids left".to_owned(), + ); + continue; + }; + let out = with_child(consumer, i, moved); + note( + notes, + format!( + "moved the join on [{}] above {} non-inner join(s), payload [{}]", + shape.keys_text(), + cand.path + .iter() + .filter(|s| is_non_inner_join(&s.node)) + .count(), + shape.payload_text(), + ), + ); + return Some(out); + } + } + None +} + +/// Every inner join reachable from `at` through pass-through nodes with a +/// non-inner join on the path above it (nothing else can move), with that +/// path. `path` holds the nodes from the consumer's child down to `at`'s +/// parent (top first while walking); `non_inner` says it holds a non-inner +/// join, so a plan of inner joins only yields no candidate to read. +fn walk( + at: &Arc, + path: &mut Vec, + non_inner: bool, + found: &mut Vec, + budget: &mut usize, +) { + if *budget == 0 { + return; + } + *budget -= 1; + if let PlanNode::Join { + kind: JoinKind::Inner, + .. + } = &at.node + && non_inner + { + for dim_left in [true, false] { + found.push(Candidate { + path: path.iter().rev().cloned().collect(), + join: Arc::clone(at), + dim_left, + }); + } + } + // Continue down through the children a path may come from (the payload is + // not known yet, so `passes` is checked once it is). + let kids = children(at); + for (i, kid) in kids.iter().enumerate() { + if may_pass(at, i) { + path.push(Step { + node: Arc::clone(at), + child: i, + }); + walk(kid, path, non_inner || is_non_inner_join(at), found, budget); + path.pop(); + } + } +} + +/// Whether a path may come up through `node` from child `i`, whatever it +/// carries: the node kinds and join sides of the module docs. +fn may_pass(node: &LogicalPlan, i: usize) -> bool { + match &node.node { + PlanNode::Project { .. } | PlanNode::Filter { .. } => i == 0, + PlanNode::Join { + kind, condition, .. + } => { + let det = match condition { + JoinCondition::On(on) => is_deterministic(on), + _ => false, + }; + det && match kind { + JoinKind::Inner => true, + JoinKind::LeftSemi + | JoinKind::LeftAnti + | JoinKind::LeftOuter + | JoinKind::Existence(_) => i == 0, + JoinKind::RightOuter => i == 1, + _ => false, + } + } + _ => false, + } +} + +/// Whether a path carrying the columns `blocked` (the payload and the +/// dimension's keys) may come up through `node` from child `i`: the node does +/// not read them and its expressions are deterministic. A projection may pass +/// a blocked column through as a bare column. +fn passes(node: &LogicalPlan, i: usize, blocked: &Attrs) -> bool { + if !may_pass(node, i) { + return false; + } + match &node.node { + PlanNode::Project { exprs, .. } => exprs.iter().all(|e| { + is_deterministic(e) && (plain_column(e) || references(e).is_disjoint(blocked)) + }), + PlanNode::Filter { predicate, .. } => { + is_deterministic(predicate) && references(predicate).is_disjoint(blocked) + } + PlanNode::Join { + condition: JoinCondition::On(on), + .. + } => references(on).is_disjoint(blocked), + _ => false, + } +} + +fn is_non_inner_join(p: &LogicalPlan) -> bool { + matches!(&p.node, PlanNode::Join { kind, .. } if *kind != JoinKind::Inner) +} + +/// A candidate's join, read: the equalities, the dimension's columns. +struct Shape { + /// `(x, d)` per equality: the other side's column, the dimension's. + eqs: Vec<(BoundColumn, BoundColumn)>, + /// The conjuncts, as written. + conj: Vec>, + /// The dimension's output columns other than its keys. + payload: Vec, + /// The payload and the dimension's keys. + blocked: Attrs, + /// The broadcast threshold the physical planner uses, `None` when off. + broadcast_max: Option, +} + +impl Shape { + fn of(c: &Candidate, broadcast_max: Option) -> Option { + let PlanNode::Join { + left, + right, + condition: JoinCondition::On(on), + .. + } = &c.join.node + else { + return None; + }; + let (dim, other) = if c.dim_left { + (left, right) + } else { + (right, left) + }; + let dim_cols = dim.resolved_output()?; + let other_cols = other.resolved_output()?; + let in_dim = |id: AttrId| dim_cols.iter().any(|c| c.id == id); + let in_other = |id: AttrId| other_cols.iter().any(|c| c.id == id); + let conj = conjuncts(on); + let mut eqs = Vec::new(); + for e in &conj { + let (op, args, _) = builtin(e)?; + if op != Builtin::Eq || args.len() != 2 { + return None; + } + let (a, b) = (bare_column(&args[0])?, bare_column(&args[1])?); + if a.data_type != b.data_type || !key_type(&a.data_type) { + return None; + } + let pair = if in_other(a.id) && in_dim(b.id) { + (a.clone(), b.clone()) + } else if in_dim(a.id) && in_other(b.id) { + (b.clone(), a.clone()) + } else { + return None; + }; + eqs.push(pair); + } + if eqs.is_empty() || !repeatable(dim) { + return None; + } + let keys: Attrs = eqs.iter().map(|(_, d)| d.id).collect(); + let payload: Vec = dim_cols + .iter() + .filter(|c| !keys.contains(&c.id)) + .cloned() + .collect(); + let mut blocked = keys; + blocked.extend(payload.iter().map(|c| c.id)); + let shape = Self { + eqs, + conj, + payload, + blocked, + broadcast_max, + }; + // Every node on the path must pass what the shape blocks. + c.path + .iter() + .all(|s| passes(&s.node, s.child, &shape.blocked)) + .then_some(shape) + } + + /// The statistics gates, or why the move is not worth it. + fn gates(&self, c: &Candidate, est: &mut Estimator<'_>) -> Result<(), String> { + let carried = self.carried(c); + let width: Option = carried.iter().map(|c| fixed_width(&c.data_type)).sum(); + if width.is_some_and(|w| w < MIN_PAYLOAD_BYTES) { + return Err(format!( + "a payload of {} B a row is under {MIN_PAYLOAD_BYTES} B", + width.unwrap_or(0) + )); + } + let PlanNode::Join { left, right, .. } = &c.join.node else { + return Err("not a join".to_owned()); + }; + let (dim, other) = if c.dim_left { + (left, right) + } else { + (right, left) + }; + let (de, xe) = (est.plan(dim), est.plan(other)); + // Never blind: a default row count says nothing of which side builds. + if !de.from_statistics || !xe.from_statistics { + return Err("the estimates rest on default row counts".to_owned()); + } + let (d, x) = (de.rows, xe.rows); + if d > x { + return Err(format!( + "the dimension estimates more rows ({d:.0}) than the other side ({x:.0})" + )); + } + // Computing D a second time must cost less than the rows it narrows. + let scanned = scanned_rows(dim, est); + if scanned > x { + return Err(format!( + "the dimension scans more rows ({scanned:.0}) than the other side holds ({x:.0})" + )); + } + // A payload on a join's probe side only streams through it; one on + // its build side is held in the hash table. Some join on the path must + // build the path's side, as the physical planner orients it. + if !c + .path + .iter() + .any(|s| builds_path(s, est, self.broadcast_max)) + { + return Err("no join on the path builds it: the payload only streams".to_owned()); + } + Ok(()) + } + + /// The payload columns the path's top still outputs. + fn carried(&self, c: &Candidate) -> Vec { + let top = c.path.last().map_or(&c.join, |s| &s.node); + let out = top.resolved_output().unwrap_or_default(); + self.payload + .iter() + .filter(|p| out.iter().any(|o| o.id == p.id)) + .cloned() + .collect() + } + + /// The path rebuilt over the semi join, the moved join over it, and a + /// projection giving the path's original output; `None` when attribute + /// ids run out or a plan has no resolved output. + fn moved( + &self, + c: &Candidate, + top: &Arc, + next: &mut u64, + ) -> Option> { + let PlanNode::Join { + left, + right, + kind, + condition, + .. + } = &c.join.node + else { + return None; + }; + let (dim, other) = if c.dim_left { + (left, right) + } else { + (right, left) + }; + + // X ⋉ Shift(π_d D) on x = shifted d. + let offset = *next; + let high = max_id(dim, &mut crate::expr::AddrMap::default()); + *next = offset + .checked_add(high)? + .checked_add(1) + .filter(|n| *n <= MAX_ID)?; + let mut key_cols: Vec = Vec::new(); + for (_, d) in &self.eqs { + if !key_cols.iter().any(|k| k.id == d.id) { + key_cols.push(d.clone()); + } + } + let keys_only = Arc::new(LogicalPlan::new(PlanNode::Project { + input: Arc::clone(dim), + exprs: key_cols.iter().map(column).collect(), + })); + let shifted = Arc::new(LogicalPlan::new(PlanNode::Shift { + input: keys_only, + offset, + })); + let semi_cond = self + .conj + .iter() + .map(|e| { + let (op, args, f) = builtin(e)?; + let args = args + .iter() + .map(|a| { + if bare_column(a).is_some_and(|b| self.blocked.contains(&b.id)) { + shift(a, offset) + } else { + Arc::clone(a) + } + }) + .collect(); + Some(recall(f, op, args)) + }) + .collect::>>()?; + let semi_cond = crate::expr::and_all(semi_cond)?; + let mut below = Arc::new( + LogicalPlan::new(PlanNode::Join { + left: Arc::clone(other), + right: shifted, + kind: JoinKind::LeftSemi, + condition: JoinCondition::On(semi_cond), + }) + .with_origin(c.join.origin), + ); + + // The path, bottom up, without the dimension's columns and with the + // other side's key columns kept for the moved join. + let xs: Vec = self.eqs.iter().map(|(x, _)| x.clone()).collect(); + for s in &c.path { + below = match &s.node.node { + PlanNode::Project { exprs, .. } => { + let mut kept: Vec> = exprs + .iter() + .filter(|e| output_of(e).is_none_or(|o| !self.blocked.contains(&o.id))) + .cloned() + .collect(); + for x in &xs { + if !kept + .iter() + .any(|e| output_of(e).is_some_and(|o| o.id == x.id)) + { + kept.push(column(x)); + } + } + let mut p = LogicalPlan::clone(&s.node); + if let PlanNode::Project { input, exprs } = &mut p.node { + *input = below; + *exprs = kept; + } + Arc::new(p) + } + _ => with_child(&s.node, s.child, below), + }; + } + + // The moved join, then the path's original output. + let joined = Arc::new( + LogicalPlan::new(PlanNode::Join { + left: below, + right: Arc::clone(dim), + kind: *kind, + condition: condition.clone(), + }) + .with_origin(c.join.origin), + ); + let out = top.resolved_output()?; + Some(Arc::new( + LogicalPlan::new(PlanNode::Project { + input: joined, + exprs: out.iter().map(column).collect(), + }) + .with_origin(top.origin), + )) + } + + fn keys_text(&self) -> String { + let v: Vec = self + .eqs + .iter() + .map(|(x, d)| format!("{} = {}", x.name, d.name)) + .collect(); + v.join(", ") + } + + fn payload_text(&self) -> String { + let v: Vec<&str> = self.payload.iter().map(|c| c.name.as_str()).collect(); + v.join(", ") + } +} + +/// Whether `e` is a column, not renamed. +fn plain_column(e: &Expr) -> bool { + matches!( + &e.node, + krishiv_plan::ExprNode::Resolved(krishiv_plan::ResolvedExpr::Column(_)) + ) +} + +/// The column `e` is, under aliases only. +fn bare_column(e: &Arc) -> Option<&BoundColumn> { + match &crate::expr::strip_alias(e).node { + krishiv_plan::ExprNode::Resolved(krishiv_plan::ResolvedExpr::Column(c)) => Some(c), + _ => None, + } +} + +/// Bytes a row of a fixed-width type; `None` for a variable-width one. +fn fixed_width(t: &DataType) -> Option { + Some(match t { + DataType::Boolean | DataType::Byte => 1, + DataType::Short => 2, + DataType::Integer | DataType::Float | DataType::Date => 4, + DataType::Long | DataType::Double | DataType::Timestamp | DataType::TimestampNtz => 8, + DataType::Decimal(d) if d.precision() <= 18 => 8, + DataType::Decimal(_) => 16, + _ => return None, + }) +} + +fn children(p: &LogicalPlan) -> Vec> { + let mut v = Vec::new(); + p.for_each_child(&mut |c| v.push(Arc::clone(c))); + v +} + +/// `p` with its child `i` replaced (a relation with one input, or a join). +fn with_child(p: &Arc, i: usize, child: Arc) -> Arc { + match &p.node { + PlanNode::Join { .. } => { + let mut j = LogicalPlan::clone(p); + if let PlanNode::Join { left, right, .. } = &mut j.node { + if i == 0 { + *left = child; + } else { + *right = child; + } + } + Arc::new(j) + } + _ => dag::with_input(p, child), + } +} diff --git a/crates/krishiv-optimizer/src/lib.rs b/crates/krishiv-optimizer/src/lib.rs index 54378779d..c9d8529bf 100644 --- a/crates/krishiv-optimizer/src/lib.rs +++ b/crates/krishiv-optimizer/src/lib.rs @@ -38,6 +38,10 @@ //! 5. **Tidy**: the rewrite batch again over the pruned plan; prune and //! tidy repeat until neither changes the plan (folding can leave columns //! unread). +//! **Late payload join**, once, with statistics (`late_payload`): a +//! dimension's payload columns join above the semi, anti and outer joins +//! that never read them, a key-only semi join left in place; then join +//! order, eager aggregation, prune and tidy again when it moved a join. //! 6. **CSE** (`cse`, once): common-subexpression elimination within one //! operator; disabled by [`Options::cse`]. //! @@ -64,6 +68,7 @@ mod group_code; mod group_limit; mod inline_cte; mod join_order; +mod late_payload; mod limit; mod narrow; mod plan_stats; @@ -80,9 +85,9 @@ mod win_magic; use std::sync::Arc; use krishiv_common::config::{ - ANSI_ENABLED, Config, ConfigValue, EAGER_AGGREGATION, GROUP_KEY_CODE, JOIN_COST, JOIN_DP_LIMIT, - JOIN_ENUMERATOR, SESSION_TIME_ZONE, SHARE_CTE_INSTANCES, SHARE_REPEATED_SUBTREES, - SUBQUERY_AS_WINDOW, + ANSI_ENABLED, AUTO_BROADCAST_JOIN_THRESHOLD, Config, ConfigValue, EAGER_AGGREGATION, + GROUP_KEY_CODE, JOIN_COST, JOIN_DP_LIMIT, JOIN_ENUMERATOR, LATE_PAYLOAD_JOIN, + SESSION_TIME_ZONE, SHARE_CTE_INSTANCES, SHARE_REPEATED_SUBTREES, SUBQUERY_AS_WINDOW, }; use krishiv_common::error::{InternalError, KrishivError}; use krishiv_plan::{ @@ -162,6 +167,10 @@ pub struct Options { /// statistics say it pays. Needs statistics; a session configuration, /// when given, can switch it off. pub eager_aggregation: bool, + /// Late payload joins (`late_payload`): a dimension's payload columns + /// join above the semi and anti joins that never read them. Needs + /// statistics; a session configuration, when given, can switch it off. + pub late_payload_join: bool, /// Subquery as window (`win_magic`, P3 W1 shape (a)): a correlated /// aggregate subquery over the outer block's own join runs as a window. /// Runs only when `krishiv.optimizer.subqueryAsWindow` is also on. @@ -183,6 +192,7 @@ impl Default for Options { cse: true, stats_propagation: true, eager_aggregation: true, + late_payload_join: true, subquery_as_window: true, share_repeated_subtrees: true, group_key_code: true, @@ -489,16 +499,7 @@ fn run( }); let root = match stats { Some(stats) if eager => { - let (after, notes) = - eager::rewrite(Arc::clone(&root), stats, options.stats_propagation)?; - t.notes("EagerAggregate", &after, notes); - // The moved aggregate changes what its join's inputs estimate, - // and so their canonical order. - if Arc::ptr_eq(&after, &root) { - after - } else { - order_and_restrict(after, stats, options.stats_propagation, joins, &mut t)? - } + eager_pass(&root, stats, options.stats_propagation, joins, &mut t)? } _ => root, }; @@ -558,27 +559,44 @@ fn run( } else { root }; - let pruned = t.once("Prune", &root, |r| Ok(prune::prune(r)))?; - let mut root = batch(Arc::clone(&pruned), "Tidy", &mut t)?; - // Tidying can fold a filter to a constant `false` and leave columns - // unread (a `false` that reached a filter on `b` frees `b`; constants - // pass grouped aggregates), so after a tidy that changed the plan and - // left such a filter, prune and tidy again until nothing changes: - // then the plan is a fixed point. Other plans (no `Filter false`, or - // a tidy that changed nothing) take no extra pass. - let mut changed = !Arc::ptr_eq(&root, &pruned) && has_false_filter(&root); - let mut round = 2; - while changed && round <= MAX_ITERATIONS { - let pruned = prune::prune(&root); - t.step("Prune", round, "Prune", &root, &pruned); - if Arc::ptr_eq(&pruned, &root) { - break; + let root = prune_and_tidy(&root, &mut t)?; + // After pruning, so a projection on the path holds only what is read + // above it and the payload is what the path really carries; the moved + // join is then ordered with the dimension's own joins and its path + // pruned and tidied again (the rule finds nothing more to move there). + let late = options.late_payload_join + && udfs.is_none_or(|(_, config)| { + config + .get(&LATE_PAYLOAD_JOIN) + .and_then(ConfigValue::as_bool) + .unwrap_or(true) + }); + let root = match stats { + Some(stats) if late => { + let (after, notes) = late_payload::rewrite( + Arc::clone(&root), + stats, + options.stats_propagation, + broadcast_max(udfs.map(|(_, config)| config)), + )?; + t.notes("LatePayloadJoin", &after, notes); + if Arc::ptr_eq(&after, &root) { + after + } else { + let mut ordered = + order_and_restrict(after, stats, options.stats_propagation, joins, &mut t)?; + // The moved join may now sit directly under its aggregate, + // the shape eager aggregation reads (where its gates allow, + // the aggregate then runs by the dimension's key below it). + if eager { + ordered = + eager_pass(&ordered, stats, options.stats_propagation, joins, &mut t)?; + } + prune_and_tidy(&ordered, &mut t)? + } } - let tidied = batch(Arc::clone(&pruned), "Tidy", &mut t)?; - changed = !Arc::ptr_eq(&tidied, &pruned); - root = tidied; - round += 1; - } + _ => root, + }; // Common-subexpression elimination runs once after the final // rewrite batch: the conjunct order of filters above is then the // order that executes, so "first operand" is execution order. @@ -691,6 +709,68 @@ fn join_settings(config: Option<&Config>, trace: bool) -> join_order::Settings { s } +/// The physical planner's broadcast threshold (`krishiv-exec` +/// `planner/join.rs` `broadcast_max_bytes`): the session's +/// `spark.sql.autoBroadcastJoinThreshold`, 10 MiB without a session, `None` +/// when negative (off). +fn broadcast_max(config: Option<&Config>) -> Option { + let max = config + .and_then(|c| c.get(&AUTO_BROADCAST_JOIN_THRESHOLD)) + .and_then(ConfigValue::as_int) + .unwrap_or(10 << 20); + #[expect(clippy::cast_precision_loss, reason = "compared with an estimate")] + (max >= 0).then_some(max as f64) +} + +/// Column pruning, then the rewrite batch over the pruned plan, repeated +/// while tidying leaves a constant `false` filter and changes the plan. +fn prune_and_tidy( + root: &Arc, + t: &mut Trace, +) -> Result, KrishivError> { + let pruned = t.once("Prune", root, |r| Ok(prune::prune(r)))?; + let mut root = batch(Arc::clone(&pruned), "Tidy", t)?; + // Tidying can fold a filter to a constant `false` and leave columns + // unread (a `false` that reached a filter on `b` frees `b`; constants + // pass grouped aggregates), so after a tidy that changed the plan and + // left such a filter, prune and tidy again until nothing changes: + // then the plan is a fixed point. Other plans (no `Filter false`, or + // a tidy that changed nothing) take no extra pass. + let mut changed = !Arc::ptr_eq(&root, &pruned) && has_false_filter(&root); + let mut round = 2; + while changed && round <= MAX_ITERATIONS { + let pruned = prune::prune(&root); + t.step("Prune", round, "Prune", &root, &pruned); + if Arc::ptr_eq(&pruned, &root) { + break; + } + let tidied = batch(Arc::clone(&pruned), "Tidy", t)?; + changed = !Arc::ptr_eq(&tidied, &pruned); + root = tidied; + round += 1; + } + Ok(root) +} + +/// Eager aggregation, then join ordering again when it moved an aggregate. +fn eager_pass( + root: &Arc, + stats: &dyn Statistics, + propagate: bool, + joins: join_order::Settings, + t: &mut Trace, +) -> Result, KrishivError> { + let (after, notes) = eager::rewrite(Arc::clone(root), stats, propagate)?; + t.notes("EagerAggregate", &after, notes); + // The moved aggregate changes what its join's inputs estimate, and so + // their canonical order. + if Arc::ptr_eq(&after, root) { + Ok(after) + } else { + order_and_restrict(after, stats, propagate, joins, t) + } +} + /// Join ordering alternating with restriction and inferred join filters, /// until neither changes the plan. The trees of the last ordering, the /// plan's own, are described in the trace. diff --git a/crates/krishiv-optimizer/src/restrict.rs b/crates/krishiv-optimizer/src/restrict.rs index acf8776d7..2459e6144 100644 --- a/crates/krishiv-optimizer/src/restrict.rs +++ b/crates/krishiv-optimizer/src/restrict.rs @@ -639,7 +639,7 @@ fn cannot_raise(e: &Arc) -> bool { } /// Types whose grouping and `=` agree value for value. -fn key_type(t: &DataType) -> bool { +pub(crate) fn key_type(t: &DataType) -> bool { matches!( t, DataType::Boolean @@ -677,7 +677,7 @@ fn node_exprs_deterministic(p: &LogicalPlan) -> bool { /// function of their inputs (no limit or sample; no CTE reference, which a /// shift does not renumber), deterministic expressions (subquery plans /// included) and no outer references. -fn repeatable(p: &Arc) -> bool { +pub(crate) fn repeatable(p: &Arc) -> bool { let mut seen = HashSet::new(); let mut stack = vec![Arc::clone(p)]; while let Some(x) = stack.pop() { @@ -704,7 +704,7 @@ fn repeatable(p: &Arc) -> bool { /// `e` with every column id raised by `offset` (no subqueries inside: the /// restricting equalities never hold one). -fn shift(e: &Arc, offset: u64) -> Arc { +pub(crate) fn shift(e: &Arc, offset: u64) -> Arc { let out = Arc::clone(e).transform_up(&mut |x| { Ok::<_, std::convert::Infallible>(match &x.node { ExprNode::Resolved(ResolvedExpr::Column(c)) => { diff --git a/crates/krishiv-optimizer/tests/it/late_payload.rs b/crates/krishiv-optimizer/tests/it/late_payload.rs new file mode 100644 index 000000000..c4843faff --- /dev/null +++ b/crates/krishiv-optimizer/tests/it/late_payload.rs @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `LatePayloadJoin` (P2b S5 lever (2)): where a dimension's payload joins +//! above the semi and anti joins that never read it, where it stays, and that +//! the rows never change (any key multiplicity, NULL keys) and the plan is a +//! fixed point. + +use std::collections::HashMap; + +use krishiv_optimizer::{ + NoStatistics, Options, Statistics, TableStats, optimize_traced_with_options, + optimize_with_options, +}; +use krishiv_plan::{ReadSource, Scan}; + +use crate::join_order::fixture; +use crate::support::{self, data, eval}; +use crate::tpcds; + +/// The optimized plan, its text and the rule's notes, with the rule on or off. +/// The optimizer's options with the group-key code (D2) off: without a +/// session configuration it would run, and code the dimension keys these +/// tests move as payload before `LatePayloadJoin` sees them. Sessions run +/// it only with `krishiv.optimizer.groupKeyCode` on (off by default). +fn options() -> Options { + let mut options = Options::default(); + options.group_key_code = false; + options +} + +fn run_with( + sql: &str, + stats: &dyn Statistics, + on: bool, +) -> (krishiv_plan::Plan, String, Vec) { + let plan = support::resolve(sql, &support::ctx()); + let mut options = options(); + options.late_payload_join = on; + let (optimized, steps) = optimize_traced_with_options(&plan, stats, &options).unwrap(); + let notes = steps + .into_iter() + .filter(|s| s.batch == "LatePayloadJoin") + .map(|s| s.note) + .collect(); + let text = optimized.root().to_string(); + (optimized, text, notes) +} + +fn run(sql: &str, on: bool) -> (krishiv_plan::Plan, String, Vec) { + run_with(sql, &fixture(), on) +} + +/// The rule's result returns the unoptimised plan's rows, keeps its output and +/// is a fixed point. +fn check(sql: &str) -> (String, Vec) { + let plan = support::resolve(sql, &support::ctx()); + let (optimized, text, notes) = run(sql, true); + assert_eq!( + optimized.root().resolved_output(), + plan.root().resolved_output(), + "{text}" + ); + let again = optimize_with_options(&optimized, &fixture(), &options()).unwrap(); + assert!( + again.structurally_eq(&optimized), + "not a fixed point:\n{}", + again.root() + ); + let db = data::tpch(7, 150); + assert_eq!( + eval::rows(&db, optimized.root()), + eval::rows(&db, plan.root()), + "{text}" + ); + (text, notes) +} + +fn moved(notes: &[String]) -> Vec<&String> { + notes.iter().filter(|n| n.starts_with("moved")).collect() +} + +/// TPC-H q21's `s_name` joins above the `EXISTS` and `NOT EXISTS` joins, and +/// `lineitem` keeps the Saudi suppliers' reduction through a key-only semi +/// join; q18's `c_name` leaves the `orders` build of the join with +/// `lineitem`. No other TPC-H query changes. +#[test] +fn q21_and_q18_payloads_join_above_their_semi_and_anti_joins() { + for q in 1..=22 { + let (text, notes) = check(&support::tpch_sql(q)); + match q { + 21 => { + assert_eq!( + moved(¬es), + [ + "moved the join on [l_suppkey = s_suppkey] above 2 non-inner join(s), payload [s_name]" + ], + "{notes:?}" + ); + let anti = text.find("Join LeftAnti").unwrap(); + let semi_on_supplier = text.find("Join LeftSemi, (s_suppkey#").unwrap(); + assert!(anti < semi_on_supplier, "{text}"); + // No build below the anti join carries the name. + assert!(!text[anti..].contains("s_name#1,"), "{text}"); + } + 18 => assert_eq!( + moved(¬es), + [ + "moved the join on [o_custkey = c_custkey] above 1 non-inner join(s), payload [c_name]" + ], + "{notes:?}" + ), + _ => assert!(moved(¬es).is_empty(), "q{q:02}: {notes:?}"), + } + } +} + +/// A dimension key that repeats: each supplier meets the five nations of its +/// region, and every one of them still counts it. NULL keys meet nothing, +/// before and after. +#[test] +fn repeating_and_null_dimension_keys_keep_every_joined_row() { + for sql in [ + // EXISTS over lineitem: a semi join building the suppliers. + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name", + // NOT EXISTS: an anti join. + "SELECT n_name, count(*), sum(s_acctbal) FROM supplier, nation \ + WHERE s_nationkey = n_regionkey \ + AND NOT EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey AND l_quantity > 49) \ + GROUP BY n_name", + // A payload read by a projection, not an aggregate. + "SELECT s_suppkey, upper(n_name) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey)", + // A key that is NULL for a third of the suppliers. + "SELECT n_name, count(*) FROM (SELECT s_suppkey, \ + CASE WHEN s_suppkey % 3 = 0 THEN NULL ELSE s_nationkey END AS nk FROM supplier) s, nation \ + WHERE nk = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name", + ] { + let (text, notes) = check(sql); + assert_eq!(moved(¬es).len(), 1, "{sql}\n{notes:?}\n{text}"); + } +} + +/// Two dimensions under one consumer both move in one pass. +#[test] +fn two_dimensions_under_one_consumer_both_move() { + let sql = "SELECT p_name, s_name, count(*) FROM partsupp, part, supplier \ + WHERE ps_partkey = p_partkey AND ps_suppkey = s_suppkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_partkey = ps_partkey \ + AND l_suppkey = ps_suppkey) GROUP BY p_name, s_name"; + let (text, notes) = check(sql); + assert_eq!(moved(¬es).len(), 2, "{notes:?}\n{text}"); +} + +/// A left outer join with the dimension's side preserved passes it too. +#[test] +fn an_outer_join_on_the_preserved_side_passes_the_payload() { + let sql = "SELECT n_name, count(l_orderkey) FROM (SELECT * FROM supplier, nation \ + WHERE s_nationkey = n_regionkey) sn \ + LEFT JOIN lineitem ON l_suppkey = s_suppkey AND l_quantity > 49 GROUP BY n_name"; + let (text, notes) = check(sql); + assert_eq!(moved(¬es).len(), 1, "{notes:?}\n{text}"); +} + +/// Where the move cannot help or is not exact, the plan stays as it was, with +/// the gate that said so where one did. +#[test] +fn the_rule_stays_out_where_it_does_not_apply() { + for (sql, why) in [ + // The semi join reads the payload. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey AND l_comment <> n_name) \ + GROUP BY n_name", + None, + ), + // The semi join reads the dimension's key. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey \ + AND l_linenumber <> n_regionkey) GROUP BY n_name", + None, + ), + // A dimension that is not the same rows twice. + ( + "SELECT n_name, count(*) FROM supplier, (SELECT * FROM nation WHERE rand() < 0.5) n \ + WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name", + None, + ), + // Eight bytes of payload. + ( + "SELECT n_nationkey, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_nationkey", + Some("a payload of 8 B a row is under 16 B"), + ), + // The dimension is the larger side. + ( + "SELECT s_name, count(*) FROM nation, supplier WHERE s_nationkey = n_nationkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = n_regionkey) GROUP BY s_name", + Some("the dimension estimates more rows"), + ), + // The semi join's other side is small: it is broadcast and the path + // only streams through it. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM region WHERE r_regionkey = s_nationkey) GROUP BY n_name", + Some("no join on the path builds it"), + ), + // An existence join (EXISTS under OR) never builds its left input. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND (EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) OR s_acctbal > 0) \ + GROUP BY n_name", + Some("no join on the path builds it"), + ), + // A right outer join with the path on its right builds its left input. + ( + "SELECT n_name, count(l_orderkey) FROM lineitem RIGHT JOIN (SELECT * FROM supplier, \ + nation WHERE s_nationkey = n_regionkey) sn ON l_suppkey = s_suppkey \ + AND l_quantity > 49 GROUP BY n_name", + Some("no join on the path builds it"), + ), + // Only inner joins: join ordering's business. + ( + "SELECT n_name, count(*) FROM supplier, nation, lineitem \ + WHERE s_nationkey = n_nationkey AND l_suppkey = s_suppkey GROUP BY n_name", + None, + ), + ] { + let (_, text, notes) = run(sql, true); + let (_, off, _) = run(sql, false); + assert_eq!(text, off, "{sql}\n{notes:?}"); + assert!(moved(¬es).is_empty(), "{sql}\n{notes:?}"); + if let Some(why) = why { + assert!(notes.iter().any(|n| n.contains(why)), "{sql}\n{notes:?}"); + } + if !sql.contains("rand()") { + check(sql); + } + } +} + +/// Default row counts never move a join. +#[test] +fn default_row_counts_move_nothing() { + let (_, _, notes) = run_with(&support::tpch_sql(21), &NoStatistics, true); + assert!(moved(¬es).is_empty(), "{notes:?}"); + assert!( + notes + .iter() + .any(|n| n.contains("rest on default row counts")), + "{notes:?}" + ); +} + +/// Switched off, by option or by the session key, nothing moves. +#[test] +fn the_rule_can_be_switched_off() { + let sql = support::tpch_sql(21); + let (_, on, _) = run(&sql, true); + let (_, off, notes) = run(&sql, false); + assert_ne!(on, off); + assert!(notes.is_empty(), "{notes:?}"); + + let plan = support::resolve(&sql, &support::ctx()); + let library = krishiv_udf_sdk::UdfLibrary::new(); + let mut config = krishiv_common::config::Config::new(); + let with = krishiv_optimizer::optimize_with_udfs(&plan, &fixture(), &library, &config).unwrap(); + assert_eq!(with.root().to_string(), on); + config + .set("krishiv.optimizer.latePayloadJoin", "false") + .unwrap(); + let without = + krishiv_optimizer::optimize_with_udfs(&plan, &fixture(), &library, &config).unwrap(); + assert_eq!(without.root().to_string(), off); +} + +/// TPC-DS SF1 row counts (no column statistics): enough for the rule's +/// `from_statistics` gate. +#[derive(Debug)] +struct TpcdsRows; + +const TPCDS_SF1_ROWS: [(&str, u64); 24] = [ + ("call_center", 6), + ("catalog_page", 11_718), + ("catalog_returns", 144_067), + ("catalog_sales", 1_441_548), + ("customer", 100_000), + ("customer_address", 50_000), + ("customer_demographics", 1_920_800), + ("date_dim", 73_049), + ("household_demographics", 7_200), + ("income_band", 20), + ("inventory", 11_745_000), + ("item", 18_000), + ("promotion", 300), + ("reason", 35), + ("ship_mode", 20), + ("store", 12), + ("store_returns", 287_514), + ("store_sales", 2_880_404), + ("time_dim", 86_400), + ("warehouse", 5), + ("web_page", 60), + ("web_returns", 71_763), + ("web_sales", 719_384), + ("web_site", 30), +]; + +impl Statistics for TpcdsRows { + fn table(&self, scan: &Scan) -> Option { + let ReadSource::Table { name, .. } = &scan.source else { + return None; + }; + let table = name.last()?; + let (_, rows) = TPCDS_SF1_ROWS.iter().find(|t| t.0 == table)?; + Some(TableStats { + rows: *rows, + columns: HashMap::new(), + }) + } +} + +/// Over TPC-DS with row counts, every query keeps its output and is a fixed +/// point (q35 moves two dimensions under one consumer); the queries the rule +/// changes are listed, so a change to the list is a reviewed decision. +#[test] +fn tpcds_with_row_counts_is_a_fixed_point() { + let ctx = support::ctx_over(tpcds::catalog()); + let mut changed = Vec::new(); + for q in 1..=99 { + let Some(plan) = support::try_resolve(&tpcds::sql(q), &ctx) else { + continue; + }; + let (optimized, steps) = + optimize_traced_with_options(&plan, &TpcdsRows, &Options::default()).unwrap(); + assert_eq!( + optimized.root().resolved_output(), + plan.root().resolved_output(), + "q{q:02}" + ); + let again = optimize_with_options(&optimized, &TpcdsRows, &Options::default()).unwrap(); + assert!( + again.structurally_eq(&optimized), + "q{q:02} is not a fixed point:\n{}", + again.root() + ); + let moves = steps + .iter() + .filter(|s| s.batch == "LatePayloadJoin" && s.note.starts_with("moved")) + .count(); + if moves > 0 { + changed.push((q, moves)); + } + } + insta::assert_debug_snapshot!(changed); +} diff --git a/crates/krishiv-optimizer/tests/it/main.rs b/crates/krishiv-optimizer/tests/it/main.rs index 0d35fe490..383b5444d 100644 --- a/crates/krishiv-optimizer/tests/it/main.rs +++ b/crates/krishiv-optimizer/tests/it/main.rs @@ -13,6 +13,7 @@ mod group_code; mod join_cost; mod join_enumerator; mod join_order; +mod late_payload; mod oracle; mod plan_stats; mod props; diff --git a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap index ec7b1eed8..f858daaf3 100644 --- a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap +++ b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap @@ -5,20 +5,25 @@ expression: optimized.root().to_string() Limit 100 +- Sort [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST], true +- Aggregate [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21) AS sum(l_quantity)#50] - +- Join LeftSemi, (o_orderkey#8 == l_orderkey#33) - :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- Join Inner, (o_orderkey#8 == l_orderkey#17) - : :- Filter isnotnull(l_orderkey#17) - : : +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (shared #1) - : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- Join Inner, (c_custkey#0 == o_custkey#9) - : :- Filter (isnotnull(o_custkey#9) and isnotnull(o_orderkey#8)) - : : +- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] - : +- Filter isnotnull(c_custkey#0) - : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] - +- Project [l_orderkey#33] - +- Filter (sum(l_quantity)#49 > 300.00) - +- Aggregate [l_orderkey#33], [l_orderkey#33, sum(l_quantity#37) AS sum(l_quantity)#49] - +- Shift +16 - +- Filter isnotnull(l_orderkey#17) - +- Shared #1, printed above + +- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] + +- Join Inner, (c_custkey#0 == o_custkey#9) + :- Join LeftSemi, (o_orderkey#8 == l_orderkey#33) + : :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, o_custkey#9] + : : +- Join Inner, (o_orderkey#8 == l_orderkey#17) + : : :- Filter isnotnull(l_orderkey#17) + : : : +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (shared #1) + : : +- Join LeftSemi, (c_custkey#51 == o_custkey#9) + : : :- Filter (isnotnull(o_custkey#9) and isnotnull(o_orderkey#8)) + : : : +- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] + : : +- Shift +51 + : : +- Project [c_custkey#0] + : : +- Filter isnotnull(c_custkey#0) + : : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] (shared #2) + : +- Project [l_orderkey#33] + : +- Filter (sum(l_quantity)#49 > 300.00) + : +- Aggregate [l_orderkey#33], [l_orderkey#33, sum(l_quantity#37) AS sum(l_quantity)#49] + : +- Shift +16 + : +- Filter isnotnull(l_orderkey#17) + : +- Shared #1, printed above + +- Filter isnotnull(c_custkey#0) + +- Shared #2, printed above diff --git a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap index 05ba32028..5cc366c5f 100644 --- a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap +++ b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap @@ -1,36 +1,45 @@ --- source: crates/krishiv-optimizer/tests/it/join_order.rs -assertion_line: 112 expression: optimized.root().to_string() --- Limit 100 +- Sort [numwait#68 DESC NULLS LAST, s_name#1 ASC NULLS FIRST], true +- Aggregate [s_name#1], [s_name#1, count(1) AS numwait#68] +- Project [s_name#1] - +- Join LeftAnti, ((l_orderkey#52 == l_orderkey#7) and (NOT (l_suppkey#54 == l_suppkey#9))) - :- Join LeftSemi, ((l_orderkey#36 == l_orderkey#7) and (NOT (l_suppkey#38 == l_suppkey#9))) - : :- Project [l_orderkey#7, l_suppkey#9, s_name#1] - : : +- Join Inner, (o_orderkey#23 == l_orderkey#7) - : : :- Project [o_orderkey#23] - : : : +- Filter ((o_orderstatus#25 == 'F') and isnotnull(o_orderkey#23)) - : : : +- Scan spark_catalog.default.orders [o_orderkey#23, o_orderstatus#25] - : : +- Project [l_orderkey#7, l_suppkey#9, s_name#1] - : : +- Join Inner, (s_suppkey#0 == l_suppkey#9) - : : :- Project [l_orderkey#7, l_suppkey#9] - : : : +- Filter (((l_receiptdate#19 > l_commitdate#18) and isnotnull(l_suppkey#9)) and isnotnull(l_orderkey#7)) - : : : +- Scan spark_catalog.default.lineitem [l_orderkey#7, l_suppkey#9, l_commitdate#18, l_receiptdate#19] (shared #1) - : : +- Project [s_suppkey#0, s_name#1] - : : +- Join Inner, (s_nationkey#3 == n_nationkey#32) - : : :- Filter (isnotnull(s_suppkey#0) and isnotnull(s_nationkey#3)) - : : : +- Scan spark_catalog.default.supplier [s_suppkey#0, s_name#1, s_nationkey#3] - : : +- Project [n_nationkey#32] - : : +- Filter ((n_name#33 == 'SAUDI ARABIA') and isnotnull(n_nationkey#32)) - : : +- Scan spark_catalog.default.nation [n_nationkey#32, n_name#33] - : +- Project [l_orderkey#36, l_suppkey#38] - : +- Shift +29 - : +- Filter isnotnull(l_orderkey#7) - : +- Shared #1, printed above - +- Project [l_orderkey#52, l_suppkey#54] - +- Shift +45 - +- Filter ((l_receiptdate#19 > l_commitdate#18) and isnotnull(l_orderkey#7)) - +- Shared #1, printed above + +- Join Inner, (s_nationkey#3 == n_nationkey#32) + :- Project [s_name#1, s_nationkey#3] + : +- Join Inner, (s_suppkey#0 == l_suppkey#9) + : :- Filter (isnotnull(s_suppkey#0) and isnotnull(s_nationkey#3)) + : : +- Scan spark_catalog.default.supplier [s_suppkey#0, s_name#1, s_nationkey#3] (shared #1) + : +- Project [l_suppkey#9] + : +- Join LeftAnti, ((l_orderkey#52 == l_orderkey#7) and (NOT (l_suppkey#54 == l_suppkey#9))) + : :- Join LeftSemi, ((l_orderkey#36 == l_orderkey#7) and (NOT (l_suppkey#38 == l_suppkey#9))) + : : :- Project [l_orderkey#7, l_suppkey#9] + : : : +- Join Inner, (o_orderkey#23 == l_orderkey#7) + : : : :- Project [o_orderkey#23] + : : : : +- Filter ((o_orderstatus#25 == 'F') and isnotnull(o_orderkey#23)) + : : : : +- Scan spark_catalog.default.orders [o_orderkey#23, o_orderstatus#25] + : : : +- Join LeftSemi, (s_suppkey#69 == l_suppkey#9) + : : : :- Project [l_orderkey#7, l_suppkey#9] + : : : : +- Filter (((l_receiptdate#19 > l_commitdate#18) and isnotnull(l_suppkey#9)) and isnotnull(l_orderkey#7)) + : : : : +- Scan spark_catalog.default.lineitem [l_orderkey#7, l_suppkey#9, l_commitdate#18, l_receiptdate#19] (shared #2) + : : : +- Shift +69 + : : : +- Project [s_suppkey#0] + : : : +- Join Inner, (s_nationkey#3 == n_nationkey#32) + : : : :- Project [s_suppkey#0, s_nationkey#3] + : : : : +- Filter (isnotnull(s_suppkey#0) and isnotnull(s_nationkey#3)) + : : : : +- Shared #1, printed above + : : : +- Project [n_nationkey#32] + : : : +- Filter ((n_name#33 == 'SAUDI ARABIA') and isnotnull(n_nationkey#32)) + : : : +- Scan spark_catalog.default.nation [n_nationkey#32, n_name#33] (shared #3) + : : +- Project [l_orderkey#36, l_suppkey#38] + : : +- Shift +29 + : : +- Filter isnotnull(l_orderkey#7) + : : +- Shared #2, printed above + : +- Project [l_orderkey#52, l_suppkey#54] + : +- Shift +45 + : +- Filter ((l_receiptdate#19 > l_commitdate#18) and isnotnull(l_orderkey#7)) + : +- Shared #2, printed above + +- Project [n_nationkey#32] + +- Filter ((n_name#33 == 'SAUDI ARABIA') and isnotnull(n_nationkey#32)) + +- Shared #3, printed above diff --git a/crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap b/crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap new file mode 100644 index 000000000..bb783622b --- /dev/null +++ b/crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap @@ -0,0 +1,22 @@ +--- +source: crates/krishiv-optimizer/tests/it/late_payload.rs +expression: changed +--- +[ + ( + 35, + 1, + ), + ( + 58, + 3, + ), + ( + 72, + 1, + ), + ( + 83, + 3, + ), +] diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap index bd7baa9db..70b828fa6 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap @@ -1,69 +1,77 @@ --- source: crates/krishiv-session/tests/explain_analyze_tpch.rs -assertion_line: 59 expression: text --- == Physical Plan (analyzed) == -wall , 8 pipelines, drivers, threads (most in one pipeline), 1 worker +wall , 9 pipelines, drivers, threads (most in one pipeline), 1 worker memory: peak reserved , limit hash probe: % of driver busy time TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] : · sink #1 TopN p1: in 5 rows (1 batch); compute (max , drivers); peak mem -: · source #24 TopN p0: out 5 rows (1 batch); compute (max , drivers); io (max ) +: · source #28 TopN p0: out 5 rows (1 batch); compute (max , drivers); io (max ) +- Exchange SinglePartition +- LocalTopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] : · sink #2 TopN p2: in 5 rows (1 batch); compute (max , drivers); peak mem - : · source #23 TopN p1: out 5 rows (1 batch); compute (max , drivers); io (max ) - +- Project [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21)#4611686018427387946 AS sum(l_quantity)#50] - : · operator #22 Project p2: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) + : · source #27 TopN p1: out 5 rows (1 batch); compute (max , drivers); io (max ) + +- Project [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21)#4611686018427387952 AS sum(l_quantity)#50] + : · operator #26 Project p2: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) +- HashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] - : · source #21 HashAggregate p2: out 5 rows (1 batch); compute (max , drivers); io (max ) + : · source #25 HashAggregate p2: out 5 rows (1 batch); compute (max , drivers); io (max ) +- Exchange hashpartitioning(c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11) +- PartialHashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] : · sink #3 HashAggregate p3: in 20 rows (1 batch); compute (max , drivers); peak mem ; groups=5 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=5 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= - +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey - : · sink #4 HashJoinBuild p4: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain - : · operator #20 HashJoinProbe p3: in 160 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= - :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : : · operator #19 Project p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers) - : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey - : : · sink #11 HashJoinBuild p6: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain - : : · operator #18 HashJoinProbe p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= - : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] - : : · source #17 MemoryScan p3: out 160 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [l_orderkey]: range, tested 160, dropped 0 - : +- BroadcastExchange - : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : : · operator #16 Project p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) - : +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey - : : · sink #12 HashJoinBuild p7: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain - : : · operator #15 HashJoinProbe p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= - : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12], PushedFilters: [(isnotnull(o_custkey#9) and isnotnull(o_orderkey#8))] - : : · source #14 MemoryScan p6: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [o_custkey]: range, tested 40, dropped 0, #3 [o_orderkey]: small set of 5, tested 0, dropped 0 - : +- BroadcastExchange - : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1], PushedFilters: [isnotnull(c_custkey#0)] - : · source #13 MemoryScan p7: out 20 rows (1 batch); compute (max , drivers); io (max ) - +- BroadcastExchange - +- Project [l_orderkey#33] - : · operator #10 Project p4: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) - +- Filter (sum(l_quantity)#49 > 300.00) - : · operator #9 Filter p4: in 40 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) - +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387943 AS sum(l_quantity)#49] - : · operator #8 Project p4: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) - +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - : · source #7 HashAggregate p4: out 40 rows (1 batch); compute (max , drivers); io (max ) - +- Exchange hashpartitioning(l_orderkey#33) - +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - : · sink #5 HashAggregate p5: in 160 rows (1 batch); compute (max , drivers); peak mem ; groups=40 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=40 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= - +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] (ids +16) - · source #6 MemoryScan p5: out 160 rows (1 batch); compute (max , drivers); io (max ) + +- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] + : · operator #24 Project p3: in 20 rows (1 batch); out 20 rows (1 batch); compute (max , drivers) + +- HashJoin Inner, [c_custkey#0] = [o_custkey#9], RuntimeFilter #4: c_custkey#0 -> spark_catalog.default.customer.c_custkey + : · sink #4 HashJoinBuild p4: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · operator #23 HashJoinProbe p3: in 20 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=5 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + :- Scan spark_catalog.default.customer [c_custkey#0, c_name#1], PushedFilters: [isnotnull(c_custkey#0)] + : · source #22 MemoryScan p3: out 20 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=20 runtime_filter_rows_dropped=15 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#4 [c_custkey]: small set of 5, tested 20, dropped 15 + +- BroadcastExchange + +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey + : · sink #5 HashJoinBuild p5: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · operator #21 HashJoinProbe p4: in 160 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, o_custkey#9] + : : · operator #20 Project p4: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers) + : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey + : : · sink #12 HashJoinBuild p7: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · operator #19 HashJoinProbe p4: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] + : : · source #18 MemoryScan p4: out 160 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [l_orderkey]: range, tested 160, dropped 0 + : +- BroadcastExchange + : +- HashJoin LeftSemi, [o_custkey#9] = [c_custkey#51], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey + : : · sink #13 HashJoinBuild p8: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · operator #17 HashJoinProbe p7: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12], PushedFilters: [(isnotnull(o_custkey#9) and isnotnull(o_orderkey#8))] + : : · source #16 MemoryScan p7: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [o_custkey]: range, tested 40, dropped 0, #3 [o_orderkey]: small set of 5, tested 0, dropped 0 + : +- BroadcastExchange + : +- Project [c_custkey#0] (ids +51) + : : · operator #15 Project p8: in 20 rows (1 batch); out 20 rows (1 batch); compute (max , drivers) + : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1], PushedFilters: [isnotnull(c_custkey#0)] + : · source #14 MemoryScan p8: out 20 rows (1 batch); compute (max , drivers); io (max ) + +- BroadcastExchange + +- Project [l_orderkey#33] + : · operator #11 Project p5: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) + +- Filter (sum(l_quantity)#49 > 300.00) + : · operator #10 Filter p5: in 40 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) + +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387949 AS sum(l_quantity)#49] + : · operator #9 Project p5: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) + +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : · source #8 HashAggregate p5: out 40 rows (1 batch); compute (max , drivers); io (max ) + +- Exchange hashpartitioning(l_orderkey#33) + +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : · sink #6 HashAggregate p6: in 160 rows (1 batch); compute (max , drivers); peak mem ; groups=40 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=40 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= + +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21], PushedFilters: [isnotnull(l_orderkey#17)] (ids +16) + · source #7 MemoryScan p6: out 160 rows (1 batch); compute (max , drivers); io (max ) Result: sink #0 Result p0: in 5 rows (1 batch); compute (max , drivers) Pipelines: - p0: wall , drivers, threads: #24 TopN -> #0 Result - p1: wall , drivers, threads: #23 TopN -> #1 TopN - p2: wall , drivers, threads: #21 HashAggregate -> #22 Project -> #2 TopN - p3: wall , drivers, threads: #17 MemoryScan -> #18 HashJoinProbe -> #19 Project -> #20 HashJoinProbe -> #3 HashAggregate - p4: wall , drivers, threads: #7 HashAggregate -> #8 Project -> #9 Filter -> #10 Project -> #4 HashJoinBuild - p5: wall , drivers, threads: #6 MemoryScan -> #5 HashAggregate - p6: wall , drivers, threads: #14 MemoryScan -> #15 HashJoinProbe -> #16 Project -> #11 HashJoinBuild - p7: wall , drivers, threads: #13 MemoryScan -> #12 HashJoinBuild + p0: wall , drivers, threads: #28 TopN -> #0 Result + p1: wall , drivers, threads: #27 TopN -> #1 TopN + p2: wall , drivers, threads: #25 HashAggregate -> #26 Project -> #2 TopN + p3: wall , drivers, threads: #22 MemoryScan -> #23 HashJoinProbe -> #24 Project -> #3 HashAggregate + p4: wall , drivers, threads: #18 MemoryScan -> #19 HashJoinProbe -> #20 Project -> #21 HashJoinProbe -> #4 HashJoinBuild + p5: wall , drivers, threads: #8 HashAggregate -> #9 Project -> #10 Filter -> #11 Project -> #5 HashJoinBuild + p6: wall , drivers, threads: #7 MemoryScan -> #6 HashAggregate + p7: wall , drivers, threads: #16 MemoryScan -> #17 HashJoinProbe -> #12 HashJoinBuild + p8: wall , drivers, threads: #14 MemoryScan -> #15 Project -> #13 HashJoinBuild diff --git a/docs/reference/config/index.md b/docs/reference/config/index.md index 646fe656b..ffc88f6ef 100644 --- a/docs/reference/config/index.md +++ b/docs/reference/config/index.md @@ -40,7 +40,7 @@ Keys under `krishiv.experimental.` are experimental and off by default. Spark al | `krishiv.connect.session.idleTimeout` | duration | `1h` | cluster | startup | 0.3.0 | | A Spark Connect session no RPC used for this long is released (its id then answers `INVALID_HANDLE.SESSION_CLOSED`); a session running a query is kept. `0s` never expires (Spark's `spark.connect.session.manager.defaultSessionTimeout`, whose `-1` this is). | | `krishiv.connect.session.maintenanceInterval` | duration (bare number: ms) | `30s` | cluster | startup | 0.3.0 | `spark.connect.session.manager.maintenanceInterval` | How often the server looks for idle Spark Connect sessions to release. | | `krishiv.delta.logThreads` | positive int | `4` | cluster | startup | 0.2.0 | | Experimental. Threads of the one process-wide pool on which Delta log work runs (log replay, snapshots, commits, checkpoints): at most this many Delta kernel calls run at once across all queries, and a query waits for a free thread (a cancellable wait, measured by the `delta.log_pool.wait_ms` metric). Read when the pool first starts. | -| `krishiv.deterministicOrder` | bool | `false` | session | user | 0.1.0 | | Make a query's result reproducible, aiming at the same bytes at any thread count (a float sum under a window, a sort-merge join or a very large grouping can still differ, because those size their buckets by the thread count). A scan pipeline whose source can be split deterministically runs one driver per logical partition (up to 16), morsels are dealt to them by ordinal, and their partial states are combined in partition order, so partial states (a decimal `sum` or `avg`, whose overflow can depend on the order of the merges) merge the same way and a query without `ORDER BY` returns its rows in the same order on every run, byte for byte, for the engine's own scans. Any other pipeline (a sink that is not lane-safe, a source that reports no positions, a pushed `LIMIT`, an operator with work after its input, a memory limit under 4 GiB) runs on one driver. The key hashes take a fixed seed, so an operator that spills splits its rows the same way on every run. Costs parallelism where it falls back to one driver: meant for reproducing a result, comparing two builds, and tests. Memory pressure from other queries in the process can still change when an operator spills, and so the order of what it returns. The branches of a `UNION ALL` under an operator that must see all of them (a hash join probing a union) take turns in branch order, so they lose their overlap too. Off by default. | +| `krishiv.deterministicOrder` | bool | `false` | session | user | 0.1.0 | | Make a query's result reproducible, aiming at the same bytes at any thread count (a float sum under a window, a sort-merge join or a very large grouping can still differ, because those size their buckets by the thread count). A scan pipeline whose source can be split deterministically runs one driver per logical partition (up to 16), morsels are dealt to them by ordinal, and their partial states are combined in partition order, so partial states (a decimal `sum` or `avg`, whose overflow can depend on the order of the merges) merge the same way and a query without `ORDER BY` returns its rows in the same order on every run, byte for byte, for the engine's own scans. Any other pipeline (a sink that is not lane-safe, a source that reports no positions, a pushed `LIMIT`, an operator with work after its input, a memory limit under 4 GiB) runs on one driver. The key hashes take a fixed seed, so an operator that spills splits its rows the same way on every run. Costs parallelism where it falls back to one driver: meant for reproducing a result, comparing two builds, and tests. Memory pressure from other queries in the process can still change when an operator spills, or whether a hash join build copies its rows into blocks as they arrive, and so the order of what it returns. The branches of a `UNION ALL` under an operator that must see all of them (a hash join probing a union) take turns in branch order, so they lose their overlap too. Off by default. | | `krishiv.exec.agg.directKeys` | bool | `true` | session | user | 0.2.0 | | Internal: a grouped aggregation whose key columns in a batch are all one-byte strings or small integers without NULLs finds each row's group through a table indexed by the packed key, and hashes only each new key's first row. Results are the same either way; false hashes every row. | | `krishiv.exec.agg.rowLayout` | bool | `true` | session | user | 0.1.0 | | Internal: grouped aggregations keep their keys as fixed-width rows in radix-partitioned tables and merge the partitions in parallel. Results are the same either way; aggregations with nested or dictionary keys keep the columnar path. | | `krishiv.exec.batchSize` | int | `4096` | session | admin | 0.1.0 | | Rows per batch passed between operators. Operators do about a millisecond of work per batch, so the scheduler can switch tasks between batches. | @@ -99,6 +99,7 @@ Keys under `krishiv.experimental.` are experimental and off by default. Spark al | `krishiv.optimizer.joinCost` | enum (`rows`, `bytes`) | `rows` | session | user | 0.1.0 | | What the join optimizer minimises when it orders a tree of inner joins, and which input of each join it builds. `rows`: the rows of the intermediate results; the input with fewer rows is built; when two orders cost the same rows (to a relative 1e-9), the one whose builds hold fewer bytes wins. `bytes`: each join's result rows times 48 bytes plus the bytes of its build input (rows times the estimated width of the columns still read, Spark's default size per type); the input with fewer rows is built unless it holds more than twice the other's bytes. Results are the same either way. | | `krishiv.optimizer.joinDpLimit` | positive int | `10` | session | user | 0.1.0 | | Most inputs of an inner-join tree the `dpsub` join enumerator plans exactly; larger trees are planned greedily. Values above 14 count as 14: the work grows as 3 to the power of the inputs. Results are the same either way. | | `krishiv.optimizer.joinEnumerator` | enum (`dpsub`, `greedy`) | `dpsub` | session | user | 0.1.0 | | How the join optimizer orders each tree of inner joins. `dpsub`: dynamic programming over every split of every subset of the inputs, for trees of at most `krishiv.optimizer.joinDpLimit` inputs, greedy above. `greedy`: always join the connected pair with the fewest result rows. Results are the same either way; for comparing plans and planning time. | +| `krishiv.optimizer.latePayloadJoin` | bool | `true` | session | user | 0.1.0 | | Join a dimension's payload columns (such as a name read only by the aggregate on top) after the semi, anti and outer joins that never read them, and leave a semi join on the dimension's keys where the join was, so those joins build and probe narrower rows. Applies when statistics show the payload is wide and the dimension is the smaller side. Results are the same either way. | | `krishiv.optimizer.ndvSampling.enabled` | bool | `true` | session | user | 0.1.0 | | Before a query is optimized, count the distinct values of the file columns an equality or IN compares with values, when the files store no distinct count, from the first rows of the table's files. The count steers join ordering and build-side choice; results are the same either way. Samples are kept per file version. | | `krishiv.optimizer.ndvSampling.rows` | positive int | `131072` | session | user | 0.1.0 | | The most rows a distinct-value sample of a table reads, from its first files in order. A table no larger than this is counted exactly. | | `krishiv.optimizer.ndvSampling.timeout` | duration | `5s` | session | user | 0.1.0 | | The time one query's distinct-value samples may take together: its tables are sampled in turn, each with what the ones before it left. A table that runs out of time, or gets none, keeps its default estimates; EXPLAIN COST and EXPLAIN VERBOSE say which ("Distinct-Value Samples"). The row limit is what bounds a sample; this only guards against slow storage, so that how fast the machine is does not decide the plan. | diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index 1d3cf4302..dbf18a533 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -81,9 +81,10 @@ so it lists no operators of its own. | `runtime_filter_*`, `runtime_filters` | Rows a scan's runtime filters tested and dropped, units and splits they pruned, and per filter the form its producer chose (`range`, `small set of N`, `hash set of N`, `bloom N B`). | | `hash_entries`, `hash_buckets` (load), `hash_index` | The operator's hash tables: distinct keys, buckets and the index kind (a join's probe side reports the tables its grace phase built after spilling as `grace_hash_entries` and `grace_hash_buckets`) (`swiss`: an open-addressing table of keys; `swiss+chain`: the same with duplicate build rows chained; `row-flat+chain` and `row-salted+chain`: the row-layout join index, flat or salted by size). | | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | -| `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent concatenating rows and indexing them (summed over the tasks, which run in parallel), and the longest single task. | -| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the buffer bytes of the finalized row arenas (Arrow's `get_array_memory_size`, capacities included), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). The operator's `peak_reserved_bytes` also counts the transient concatenation copy `finalize` makes per partition. | -| `max_finalize_copy_bytes`, `max_coalesce_copy_bytes` | A hash join build's two transient copies: the most bytes of concatenation copies `finalize` held reserved at once, and the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). | +| `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent setting up the rows' blocks (whole blocks sliced in place, the chunks' tails copied) and indexing them (summed over the tasks, which run in parallel), and the longest single task. | +| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the bytes of the finalized row arenas (each block's rows' share of its buffers, Arrow's `get_slice_memory_size`), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). | +| `max_coalesce_copy_bytes` | A hash join build's transient copy: the largest set of small chunks one build-phase step copied into a block of 16,384 rows (each copy reserves those chunks again while it runs), also over a grace partition's frames as they are read back. Finalize reads the blocks in place and copies only the chunks' tails. | +| `max_finalize_copy_bytes` | The most bytes of tail copies `finalize` held reserved at once (partitions copying in parallel). | | `broadcast_shared`, `broadcast_fallback` | A broadcast hash join on a multi-task run: 1 when the task probed the worker's shared build table, or 1 when the shared build did not fit `krishiv.exec.join.sharedBroadcastBytes` and the task built its own table instead. | | Pipelines | Per pipeline: wall time from its first driver's start to its last driver's end, drivers, threads that ran them, and its operators from source to sink. |