From 260c9428068a16c3a00dbb41f56b201cf9bf868c Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:17:47 +0000 Subject: [PATCH 01/25] exec: hash join build memory counters (steady reservation, arena bytes by hash/key/payload, index bytes) in EXPLAIN ANALYZE and PROFILE (X3' step 1) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 14 ++++++++++++++ crates/krishiv-exec/src/join/mod.rs | 20 +++++++++++++++++-- crates/krishiv-exec/src/join/table.rs | 28 +++++++++++++++++++++++++++ docs/reference/explain-analyze.md | 1 + 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 0df26846f..87101be46 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -843,6 +843,15 @@ impl Sink for BuildSink { .saturating_sub(table.index_memory()) .min(index.size()), ); + let split = table.arena().memory_split(); + let n = &self.0.counters; + n.add( + Counter::ArenaBytes, + (split.hash + split.key + split.other) as u64, + ); + n.add(Counter::ArenaHashBytes, split.hash as u64); + n.add(Counter::ArenaKeyBytes, split.key as u64); + n.add(Counter::IndexBytes, table.index_memory() as u64); let (keys, buckets, kind) = table.index_size(); self.0.counters.or(Counter::IndexKinds, kind as u64); self.0.counters.add(Counter::HashEntries, keys as u64); @@ -878,6 +887,11 @@ impl Sink for BuildSink { let mut grace = Grace::new(g.hashing.fanout0()); grace.keep_unprobed = s.join_type.keeps_null_build_keys(); for (p, part) in c.parts.iter_mut().enumerate() { + if spilled & (1 << p) == 0 { + self.0 + .counters + .add(Counter::SteadyReservedBytes, part.bytes() as u64); + } memory.append(&mut part.memory); if spilled & (1 << p) != 0 { grace.build[p].files = std::mem::take(&mut part.files); diff --git a/crates/krishiv-exec/src/join/mod.rs b/crates/krishiv-exec/src/join/mod.rs index ef50f072d..91c786638 100644 --- a/crates/krishiv-exec/src/join/mod.rs +++ b/crates/krishiv-exec/src/join/mod.rs @@ -379,11 +379,27 @@ pub(crate) enum Counter { ProbeTableNs, /// Spilled partitions that would not split, joined by sort-merge. MergeFallbacks, + /// What the build's in-memory partitions keep reserved for their rows + /// once finalized (the chunks' reservations, which the probe lives with). + SteadyReservedBytes, + /// The exact buffer bytes of the finalized arenas (all columns). + ArenaBytes, + /// Of those, the `hash` column's. + ArenaHashBytes, + /// Of those, the key columns'. + ArenaKeyBytes, + /// The bytes of the finalized indexes (and match bitmaps). + IndexBytes, } /// The build sink's counters, by name. -const BUILD_COUNTERS: [(Counter, &str); 9] = [ +const BUILD_COUNTERS: [(Counter, &str); 14] = [ (Counter::BuildRows, "build_rows"), + (Counter::SteadyReservedBytes, "steady_reserved_bytes"), + (Counter::ArenaBytes, "arena_bytes"), + (Counter::ArenaHashBytes, "arena_hash_bytes"), + (Counter::ArenaKeyBytes, "arena_key_bytes"), + (Counter::IndexBytes, "index_bytes"), (Counter::HashEntries, "hash_entries"), (Counter::HashBuckets, "hash_buckets"), (Counter::SpilledPartitions, "spilled_partitions"), @@ -415,7 +431,7 @@ const PROBE_DETAIL_COUNTERS: [(Counter, &str); 3] = [ /// Counters updated by every driver (relaxed: read once the query ended). #[derive(Debug, Default)] -pub(crate) struct Counters([std::sync::atomic::AtomicU64; 24]); +pub(crate) struct Counters([std::sync::atomic::AtomicU64; 29]); impl Counters { pub(crate) fn add(&self, c: Counter, v: u64) { diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index b33add71d..9fbb8686c 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -213,6 +213,15 @@ impl Cols { } } +/// An arena's buffer bytes by part ([`RowArena::memory_split`]). +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct ArenaBytes { + pub(crate) hash: usize, + pub(crate) key: usize, + /// Payload and NULL-key columns. + pub(crate) other: usize, +} + /// One partition's build rows, coalesced into a single batch. #[derive(Debug)] pub(crate) struct RowArena { @@ -260,6 +269,25 @@ impl RowArena { self.batch.num_rows() } + /// The arena's buffer bytes by part: the `hash` column, the key + /// columns (`key*`), and everything else (payload, NULL-key flags). + pub(crate) fn memory_split(&self) -> ArenaBytes { + let data = self.batch.data(); + let mut out = ArenaBytes::default(); + for (field, column) in data.schema().fields().iter().zip(data.columns()) { + let bytes = column.get_array_memory_size(); + let name = field.name(); + if name == "hash" { + out.hash += bytes; + } else if name.starts_with("key") { + out.key += bytes; + } else { + out.other += bytes; + } + } + out + } + pub(crate) fn hash(&self, row: u32) -> u64 { self.hashes.value(row as usize) } diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index 3bc4ada57..29b2ed61c 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -78,6 +78,7 @@ so it lists no operators of its own. | `hash_entries`, `hash_buckets` (load), `hash_index` | The operator's hash tables: distinct keys, buckets and the index kind (a join's probe side reports the tables its grace phase built after spilling as `grace_hash_entries` and `grace_hash_buckets`) (`swiss`: an open-addressing table of keys; `swiss+chain`: the same with duplicate build rows chained; `row-flat+chain` and `row-salted+chain`: the row-layout join index, flat or salted by size). | | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | | `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent concatenating rows and indexing them (summed over the tasks, which run in parallel), and the longest single task. | +| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over its in-memory partitions: what their rows keep reserved through the probe, the exact buffer bytes of the finalized row arenas, of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). The operator's `peak_reserved_bytes` also counts the transient concatenation copy `finalize` makes per partition. | | Pipelines | Per pipeline: wall time from its first driver's start to its last driver's end, drivers, threads that ran them, and its operators from source to sink. | Other operator-specific counters (an aggregate's `groups` and spill runs, From 2f6db8ca9a7a354212f2f0646f1a017a2598e290 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:57:20 +0000 Subject: [PATCH 02/25] exec: finalize-copy and coalesce-copy maxima among the build counters; a test of the build memory counters; snapshots (X3' step 1) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 10 +++++++++- crates/krishiv-exec/src/join/grace.rs | 19 +++++++++++------- crates/krishiv-exec/src/join/mod.rs | 20 ++++++++++++++----- .../krishiv-session/tests/explain_analyze.rs | 13 ++++++++++++ ...plain_analyze__analyze_join_aggregate.snap | 2 +- ...ain_graph__profile_join_aggregate_dot.snap | 2 +- ...graph__profile_join_aggregate_mermaid.snap | 2 +- docs/reference/explain-analyze.md | 1 + 8 files changed, 53 insertions(+), 16 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 87101be46..779feed99 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -625,7 +625,8 @@ impl Sink for BuildSink { } for (p, c) in chunks { l.parts[p].chunks.push(c); - l.parts[p].coalesce(s.arena_schema(&g.hashing))?; + let merged = l.parts[p].coalesce(s.arena_schema(&g.hashing))?; + s.counters.max(Counter::MaxCoalesceCopyBytes, merged as u64); } if let Some((sketches, _)) = &mut l.sketches { for (sketch, rows) in sketches.iter_mut().zip(&by_part) { @@ -762,6 +763,8 @@ impl Sink for BuildSink { // are asked for through arbitration when nothing else runs. let spill = s.spill.is_some(); let mut spare = held_copy; + // The concatenation copies reserved at once (for the counter). + let mut copying = 0usize; let mut jobs = std::collections::VecDeque::from(jobs); let mut running = futures::stream::FuturesUnordered::new(); let mut tables = Vec::with_capacity(jobs.len()); @@ -796,6 +799,10 @@ impl Sink for BuildSink { .. }) = jobs.pop_front_if(|_| funded) { + copying = copying.saturating_add(copy); + self.0 + .counters + .max(Counter::MaxFinalizeCopyBytes, copying as u64); let shared = Arc::clone(&self.0); let hashing = g.hashing.clone(); running.push(async move { @@ -836,6 +843,7 @@ impl Sink for BuildSink { break; }; let (p, table, copy, index_bytes) = done?; + copying = copying.saturating_sub(copy); // The index was reserved for one key per row: keep what it // holds. index.shrink( diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index c8d99a440..1d8b8c2b4 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -87,13 +87,17 @@ impl Part { /// count reached the fan-out copied the whole partition again every 15 /// pushes (quadratic in its size); tiered, a row is copied about once /// per tier it climbs. - pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result<()> { + /// + /// Returns the bytes of the largest run merged (each merge reserves the + /// run again while it copies: the transient peak), 0 when none was. + pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result { const FANOUT: usize = 16; let size = |c: &Batch| c.data().get_array_memory_size(); let tier = |c: &Batch| size(c).max(1).ilog(FANOUT); + let mut largest = 0; while self.chunks.len() >= FANOUT { let Some(top) = self.chunks.last().map(tier) else { - return Ok(()); + return Ok(largest); }; let run = self .chunks @@ -102,15 +106,15 @@ impl Part { .take_while(|c| tier(c) <= top) .count(); if run < FANOUT { - return Ok(()); + return Ok(largest); } let from = self.chunks.len() - run; let old: usize = self.chunks[from..].iter().map(size).sum(); let Some(m) = self.memory.first_mut().filter(|m| m.size() >= old) else { - return Ok(()); + return Ok(largest); }; if m.try_grow(old).is_err() { - return Ok(()); + return Ok(largest); } let merged = Batch::concat(schema, &self.chunks[from..]).map_err(|e| internal(e.to_string())); @@ -118,14 +122,15 @@ impl Part { Ok(b) if size(&b) <= old => b, other => { m.shrink(old); - return other.map(drop); + return other.map(|_| largest); } }; + largest = largest.max(old); m.shrink(old + old - size(&merged)); self.chunks.truncate(from); self.chunks.push(merged); } - Ok(()) + Ok(largest) } pub(crate) fn absorb(&mut self, other: Self) { diff --git a/crates/krishiv-exec/src/join/mod.rs b/crates/krishiv-exec/src/join/mod.rs index 91c786638..0c8390972 100644 --- a/crates/krishiv-exec/src/join/mod.rs +++ b/crates/krishiv-exec/src/join/mod.rs @@ -390,16 +390,24 @@ pub(crate) enum Counter { ArenaKeyBytes, /// The bytes of the finalized indexes (and match bitmaps). IndexBytes, + /// The most bytes of concatenation copies `finalize` held reserved at + /// once (partitions copying in parallel). + MaxFinalizeCopyBytes, + /// The largest run of chunks a build-phase coalesce merged (each merge + /// reserves the run again while it copies). + MaxCoalesceCopyBytes, } /// The build sink's counters, by name. -const BUILD_COUNTERS: [(Counter, &str); 14] = [ +const BUILD_COUNTERS: [(Counter, &str); 16] = [ (Counter::BuildRows, "build_rows"), (Counter::SteadyReservedBytes, "steady_reserved_bytes"), (Counter::ArenaBytes, "arena_bytes"), (Counter::ArenaHashBytes, "arena_hash_bytes"), (Counter::ArenaKeyBytes, "arena_key_bytes"), (Counter::IndexBytes, "index_bytes"), + (Counter::MaxFinalizeCopyBytes, "max_finalize_copy_bytes"), + (Counter::MaxCoalesceCopyBytes, "max_coalesce_copy_bytes"), (Counter::HashEntries, "hash_entries"), (Counter::HashBuckets, "hash_buckets"), (Counter::SpilledPartitions, "spilled_partitions"), @@ -431,7 +439,7 @@ const PROBE_DETAIL_COUNTERS: [(Counter, &str); 3] = [ /// Counters updated by every driver (relaxed: read once the query ended). #[derive(Debug, Default)] -pub(crate) struct Counters([std::sync::atomic::AtomicU64; 29]); +pub(crate) struct Counters([std::sync::atomic::AtomicU64; 31]); impl Counters { pub(crate) fn add(&self, c: Counter, v: u64) { @@ -456,9 +464,11 @@ impl Counters { .map(|&(c, name)| { let v = self.get(c); let v = match c { - Counter::RecursionDepth | Counter::MaxProbeLength | Counter::MaxPartitionNs => { - Metric::Max(v) - } + Counter::RecursionDepth + | Counter::MaxProbeLength + | Counter::MaxPartitionNs + | Counter::MaxFinalizeCopyBytes + | Counter::MaxCoalesceCopyBytes => Metric::Max(v), _ => Metric::Count(v), }; (std::borrow::Cow::Borrowed(name), v) diff --git a/crates/krishiv-session/tests/explain_analyze.rs b/crates/krishiv-session/tests/explain_analyze.rs index 148d5ca65..7fce49d3d 100644 --- a/crates/krishiv-session/tests/explain_analyze.rs +++ b/crates/krishiv-session/tests/explain_analyze.rs @@ -396,6 +396,19 @@ fn row_layout_joins_report_their_index() { Some(&Metric::Text("row-flat+chain".to_owned())) ); assert_eq!(build.extra("hash_entries"), Some(&Metric::Count(4))); + // Its memory once finalized: one 8-byte hash per row, key and payload + // columns, an index, and the reservation the probe lives with. + let count = |name| build.extra(name).and_then(Metric::value).unwrap_or(0); + let rows_built = count("build_rows"); + assert!(rows_built > 0, "{build:?}"); + assert!(count("arena_hash_bytes") >= 8 * rows_built, "{build:?}"); + assert!(count("arena_key_bytes") > 0, "{build:?}"); + assert!( + count("arena_bytes") >= count("arena_hash_bytes") + count("arena_key_bytes"), + "{build:?}" + ); + assert!(count("index_bytes") > 0, "{build:?}"); + assert!(count("steady_reserved_bytes") > 0, "{build:?}"); // The row-layout index counts probe lengths too: every probe row with // a key looks up once (2,000 orders, filtered to those over 10). let probe = profile diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap b/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap index 09b087b32..69e200a3c 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap @@ -20,7 +20,7 @@ Sort [name#3 ASC NULLS FIRST] +- Project [amount#1, name#3] : · operator #7 Project p2: in 1150 rows (4 batches); out 1150 rows (4 batches); compute (max , drivers) +- HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -> spark_catalog.default.orders.cust - : · sink #3 HashJoinBuild p3: in 4 rows (1 batch); compute (max , drivers); peak mem ; build_rows=4 hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #3 HashJoinBuild p3: in 4 rows (1 batch); compute (max , drivers); peak mem ; build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #6 HashJoinProbe p2: in 1150 rows (4 batches); out 1150 rows (4 batches); compute (max , drivers); probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.orders [cust#0, amount#1], PushedFilters: [(amount#1 > 10L)] : · source #5 MemoryScan p2: out 1150 rows (4 batches); compute (max , drivers); io (max ); runtime_filter_rows_tested=1450 runtime_filter_rows_dropped=300 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [cust]: range, tested 1450, dropped 300 diff --git a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap index 50dc82ce2..fe45a7b20 100644 --- a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap +++ b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap @@ -10,7 +10,7 @@ digraph { n4 [label="Exchange hashpartitioning(name#3)"]; n5 [label="PartialHashAggregate keys=[name#3], functions=[count(1), sum(amount#1)]\n#2 HashAggregate sink [p2]: in_rows=1150 in_batches=4 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 groups=4 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=4 hash_buckets= hash_resizes=1 hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows=4 exchange_bytes= exchange_keys=4 exchange_map_tasks=1 exchange_partitions=3 exchange_max_partition_rows=2"]; n6 [label="Project [amount#1, name#3]\n#7 Project operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"]; - n7 [label="HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -> spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="]; + n7 [label="HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -> spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="]; n8 [label="Scan spark_catalog.default.orders [cust#0, amount#1], PushedFilters: [(amount#1 > 10L)]\n#5 MemoryScan source [p2]: in_rows=0 in_batches=0 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 runtime_filter_rows_tested= runtime_filter_rows_dropped= runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [cust]: range, tested , dropped "]; n9 [label="BroadcastExchange"]; n10 [label="Scan spark_catalog.default.customers [id#2, name#3]\n#4 MemoryScan source [p3]: in_rows=0 in_batches=0 out_rows=4 out_batches=1 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"]; diff --git a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap index d0bfa1e66..275e210c7 100644 --- a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap +++ b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap @@ -10,7 +10,7 @@ flowchart TD n4["Exchange hashpartitioning(name#3)"] n5["PartialHashAggregate keys=[name#3], functions=[count(1), sum(amount#1)]\n#2 HashAggregate sink [p2]: in_rows=1150 in_batches=4 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 groups=4 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=4 hash_buckets= hash_resizes=1 hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows=4 exchange_bytes= exchange_keys=4 exchange_map_tasks=1 exchange_partitions=3 exchange_max_partition_rows=2"] n6["Project [amount#1, name#3]\n#7 Project operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"] - n7["HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -#gt; spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="] + n7["HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -#gt; spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="] n8["Scan spark_catalog.default.orders [cust#0, amount#1], PushedFilters: [(amount#1 #gt; 10L)]\n#5 MemoryScan source [p2]: in_rows=0 in_batches=0 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 runtime_filter_rows_tested= runtime_filter_rows_dropped= runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [cust]: range, tested , dropped "] n9["BroadcastExchange"] n10["Scan spark_catalog.default.customers [id#2, name#3]\n#4 MemoryScan source [p3]: in_rows=0 in_batches=0 out_rows=4 out_batches=1 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"] diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index 29b2ed61c..b4f0fff4c 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -79,6 +79,7 @@ so it lists no operators of its own. | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | | `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent concatenating rows and indexing them (summed over the tasks, which run in parallel), and the longest single task. | | `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over its in-memory partitions: what their rows keep reserved through the probe, the exact buffer bytes of the finalized row arenas, of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). The operator's `peak_reserved_bytes` also counts the transient concatenation copy `finalize` makes per partition. | +| `max_finalize_copy_bytes`, `max_coalesce_copy_bytes` | A hash join build's two transient copies: the most bytes of concatenation copies `finalize` held reserved at once, and the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). | | Pipelines | Per pipeline: wall time from its first driver's start to its last driver's end, drivers, threads that ran them, and its operators from source to sink. | Other operator-specific counters (an aggregate's `groups` and spill runs, From 59b0606b70a53ff618e95c51d1449b6c54de1c07 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:03:20 +0000 Subject: [PATCH 03/25] exec: coalesce's merged-run bytes tested; build counter docs name what they cover (review nits) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/grace.rs | 12 +++++++++--- docs/reference/explain-analyze.md | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index 1d8b8c2b4..8a9c33a11 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -1645,15 +1645,21 @@ mod tests { let c = chunk(i); reserve_chunks(parts, &[(0, c.clone())]).unwrap(); parts[0].chunks.push(c); - parts[0].coalesce(&schema).unwrap(); + parts[0].coalesce(&schema).unwrap() }; + let sizes: usize = (0..16) + .map(|i| chunk(i).data().get_array_memory_size()) + .sum(); + let mut merged_run = 0; for i in 0..16 { - push(&mut parts, i); + merged_run = push(&mut parts, i); } assert_eq!(parts[0].chunks.len(), 1, "16 small chunks merge"); + // It reports the run it merged: the 16 chunks' bytes. + assert_eq!(merged_run, sizes); let merged = Arc::clone(parts[0].chunks[0].data().column(0)); for i in 16..31 { - push(&mut parts, i); + assert_eq!(push(&mut parts, i), 0, "no merge below the fan-out"); } assert_eq!(parts[0].chunks.len(), 16); let first = parts[0].chunks[0].data().column(0); diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index b4f0fff4c..fd4a42291 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -78,7 +78,7 @@ so it lists no operators of its own. | `hash_entries`, `hash_buckets` (load), `hash_index` | The operator's hash tables: distinct keys, buckets and the index kind (a join's probe side reports the tables its grace phase built after spilling as `grace_hash_entries` and `grace_hash_buckets`) (`swiss`: an open-addressing table of keys; `swiss+chain`: the same with duplicate build rows chained; `row-flat+chain` and `row-salted+chain`: the row-layout join index, flat or salted by size). | | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | | `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent concatenating rows and indexing them (summed over the tasks, which run in parallel), and the longest single task. | -| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over its in-memory partitions: what their rows keep reserved through the probe, the exact buffer bytes of the finalized row arenas, of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). The operator's `peak_reserved_bytes` also counts the transient concatenation copy `finalize` makes per partition. | +| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the buffer bytes of the finalized row arenas (Arrow's `get_array_memory_size`, capacities included), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). The operator's `peak_reserved_bytes` also counts the transient concatenation copy `finalize` makes per partition. | | `max_finalize_copy_bytes`, `max_coalesce_copy_bytes` | A hash join build's two transient copies: the most bytes of concatenation copies `finalize` held reserved at once, and the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). | | Pipelines | Per pipeline: wall time from its first driver's start to its last driver's end, drivers, threads that ran them, and its operators from source to sink. | From eee345acd01648927859f7c8f5ac29de768290c2 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:17:57 +0000 Subject: [PATCH 04/25] exec, session: build counter docs name get_array_memory_size; the counter test asserts the build rows exactly and bounds the hash bytes (review of #586) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/mod.rs | 3 ++- crates/krishiv-exec/src/join/table.rs | 4 ++-- crates/krishiv-session/tests/explain_analyze.rs | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/krishiv-exec/src/join/mod.rs b/crates/krishiv-exec/src/join/mod.rs index 0c8390972..6f45d5128 100644 --- a/crates/krishiv-exec/src/join/mod.rs +++ b/crates/krishiv-exec/src/join/mod.rs @@ -382,7 +382,8 @@ pub(crate) enum Counter { /// What the build's in-memory partitions keep reserved for their rows /// once finalized (the chunks' reservations, which the probe lives with). SteadyReservedBytes, - /// The exact buffer bytes of the finalized arenas (all columns). + /// The finalized arenas' bytes (all columns), as Arrow's `get_array_memory_size` counts them: buffer + /// capacities and the arrays' own structs, the measure the build's reservations use. ArenaBytes, /// Of those, the `hash` column's. ArenaHashBytes, diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index 9fbb8686c..f34402c02 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -213,7 +213,7 @@ impl Cols { } } -/// An arena's buffer bytes by part ([`RowArena::memory_split`]). +/// An arena's bytes by part ([`RowArena::memory_split`]), as `get_array_memory_size` counts them. #[derive(Debug, Default, Clone, Copy)] pub(crate) struct ArenaBytes { pub(crate) hash: usize, @@ -269,7 +269,7 @@ impl RowArena { self.batch.num_rows() } - /// The arena's buffer bytes by part: the `hash` column, the key + /// The arena's bytes by part (`get_array_memory_size`, the measure its reservation uses): the `hash` column, the key /// columns (`key*`), and everything else (payload, NULL-key flags). pub(crate) fn memory_split(&self) -> ArenaBytes { let data = self.batch.data(); diff --git a/crates/krishiv-session/tests/explain_analyze.rs b/crates/krishiv-session/tests/explain_analyze.rs index 7fce49d3d..bc2cbc438 100644 --- a/crates/krishiv-session/tests/explain_analyze.rs +++ b/crates/krishiv-session/tests/explain_analyze.rs @@ -399,9 +399,13 @@ fn row_layout_joins_report_their_index() { // Its memory once finalized: one 8-byte hash per row, key and payload // columns, an index, and the reservation the probe lives with. let count = |name| build.extra(name).and_then(Metric::value).unwrap_or(0); + // The build side is the 4 customers (`hash_entries` above: 4 distinct keys). let rows_built = count("build_rows"); - assert!(rows_built > 0, "{build:?}"); - assert!(count("arena_hash_bytes") >= 8 * rows_built, "{build:?}"); + assert_eq!(rows_built, 4, "{build:?}"); + // `get_array_memory_size`: 8 bytes per row in a 64-byte-aligned buffer plus the array's own struct, never + // more than a kilobyte for 4 rows (an inflated count fails here). + let hash = count("arena_hash_bytes"); + assert!((8 * rows_built..=1024).contains(&hash), "{build:?}"); assert!(count("arena_key_bytes") > 0, "{build:?}"); assert!( count("arena_bytes") >= count("arena_hash_bytes") + count("arena_key_bytes"), From 55c9059c35f689dd2421ae33c507579e2ff0b138 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:44:52 +0000 Subject: [PATCH 05/25] exec: a hash join build's arena reads its chunks in place, no concatenation copy at finalize or grace reload; a per-row chunk table for the probe's lookups (X3' step 3, option B) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 10 +- crates/krishiv-exec/src/join/grace.rs | 12 +- crates/krishiv-exec/src/join/table.rs | 234 ++++++++++++++++++++------ 3 files changed, 185 insertions(+), 71 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 6d37dd05a..7750e84ce 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -406,7 +406,7 @@ impl BuildSink { /// What indexing a partition's in-memory rows reserves: `(index, copy)`, /// its [`JoinTable`] index (and match bitmap when `marks`) and its -/// transient concatenation copy (none for a single chunk); `None` if no +/// concatenation copy (always 0: the arena reads its chunks in place); `None` if no /// rows are in memory. fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize)> { if part.chunks.is_empty() { @@ -418,12 +418,8 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz if marks { index = index.saturating_add(JoinTable::marks_bytes(rows)); } - let copy = if part.chunks.len() > 1 { - part.bytes() - } else { - 0 - }; - Some((index, copy)) + // The arena reads the chunks in place (`RowArena::concat` copies nothing): no concatenation copy to fund. + Some((index, 0)) } /// A duration in whole nanoseconds, saturating. diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index 8a9c33a11..54094e703 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -1228,9 +1228,9 @@ impl Loading { Some(b) => self.pending = Some(b?), } } - // Every row read: the table's index, the concatenated copy of the - // rows and the probe rows' read-back headroom (exact, so a - // partition that loads can always be joined). + // Every row read: the table's index and the probe rows' read-back + // headroom (exact, so a partition that loads can always be joined). + // The arena reads the chunks in place: no concatenated copy. let chunks: Vec = self .task .build @@ -1240,11 +1240,7 @@ impl Loading { .cloned() .collect(); let rows = chunks.iter().map(Batch::num_rows).sum(); - let concat = if chunks.len() > 1 { - self.memory.size() + self.task.build.bytes() - } else { - 0 - }; + let concat = 0; let headroom = self.task.probe.read_headroom(); let marks = if shared.join_type.builds_preserved() { JoinTable::marks_bytes(rows) diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index f34402c02..d7f2561e9 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -222,9 +222,9 @@ pub(crate) struct ArenaBytes { pub(crate) other: usize, } -/// One partition's build rows, coalesced into a single batch. +/// One chunk of an arena: a batch of build rows and its columns, read in place. #[derive(Debug)] -pub(crate) struct RowArena { +struct ArenaPart { batch: Batch, hashes: UInt64Array, keys: Cols, @@ -234,13 +234,8 @@ pub(crate) struct RowArena { null_key: Option, } -impl RowArena { - /// Concatenates arena chunks (same schema). - pub(crate) fn concat(schema: &SchemaRef, chunks: &[Batch]) -> Result { - let batch = match chunks { - [one] => one.clone(), - _ => Batch::concat(schema, chunks).map_err(|e| internal(e.to_string()))?, - }; +impl ArenaPart { + fn of(batch: Batch) -> Result { let bad = || internal("malformed join row arena".to_owned()); let hashes = batch.data().column_by_name("hash").ok_or_else(bad)?; let hashes = hashes @@ -253,9 +248,6 @@ impl RowArena { Some(c) => Some(c.as_boolean_opt().ok_or_else(bad)?.clone()), None => None, }; - if u32::try_from(batch.num_rows()).is_err() { - return Err(internal("a join partition has over 2^32 rows".to_owned())); - } Ok(Self { batch, hashes, @@ -264,40 +256,127 @@ impl RowArena { null_key, }) } +} + +/// One partition's build rows: its chunks read in place (X3' step 3, design +/// `P2-X3-copy-free-join-finalize`; no concatenation copy at finalize). Rows +/// are numbered across the chunks in order; `starts[i]` is chunk `i`'s first +/// row id. +#[derive(Debug)] +pub(crate) struct RowArena { + parts: Vec, + starts: Vec, + /// Each row's chunk, when there are 2 to 256 chunks: one load instead of a + /// search over `starts` on the probe's path (1 byte per row). + chunk_of: Vec, + len: usize, +} + +impl RowArena { + /// The arena over `chunks` (same schema, in order), without copying their + /// rows. + pub(crate) fn concat(_schema: &SchemaRef, chunks: &[Batch]) -> Result { + let mut parts = Vec::with_capacity(chunks.len().max(1)); + let mut starts = Vec::with_capacity(chunks.len().max(1)); + let mut len = 0usize; + for chunk in chunks.iter().filter(|c| c.num_rows() > 0) { + let start = u32::try_from(len) + .map_err(|_| internal("a join partition has over 2^32 rows".to_owned()))?; + starts.push(start); + len = len.saturating_add(chunk.num_rows()); + parts.push(ArenaPart::of(chunk.clone())?); + } + if u32::try_from(len).is_err() { + return Err(internal("a join partition has over 2^32 rows".to_owned())); + } + let mut chunk_of = Vec::new(); + if (2..=256).contains(&parts.len()) { + chunk_of + .try_reserve_exact(len) + .map_err(|e| internal(e.to_string()))?; + for (i, part) in parts.iter().enumerate() { + let id = u8::try_from(i).unwrap_or(u8::MAX); + chunk_of.extend(std::iter::repeat_n(id, part.batch.num_rows())); + } + } + Ok(Self { + parts, + starts, + chunk_of, + len, + }) + } + + /// The bytes of the per-row chunk table (part of the index's cost). + pub(crate) fn chunk_table_bytes(&self) -> usize { + self.chunk_of.capacity() + } + + /// The chunk holding row `row` and the row's index in it (`None` past + /// the end). + #[inline] + fn at(&self, row: u32) -> Option<(&ArenaPart, u32)> { + if let [one] = self.parts.as_slice() { + return ((row as usize) < self.len).then_some((one, row)); + } + let i = match self.chunk_of.get(row as usize) { + Some(&i) => usize::from(i), + // Past 256 chunks: the last chunk starting at or before `row`. + None if self.chunk_of.is_empty() => { + self.starts.partition_point(|&s| s <= row).checked_sub(1)? + } + None => return None, + }; + let part = self.parts.get(i)?; + let local = row.checked_sub(*self.starts.get(i)?)?; + ((local as usize) < part.batch.num_rows()).then_some((part, local)) + } pub(crate) fn len(&self) -> usize { - self.batch.num_rows() + self.len } /// The arena's bytes by part (`get_array_memory_size`, the measure its reservation uses): the `hash` column, the key /// columns (`key*`), and everything else (payload, NULL-key flags). pub(crate) fn memory_split(&self) -> ArenaBytes { - let data = self.batch.data(); let mut out = ArenaBytes::default(); - for (field, column) in data.schema().fields().iter().zip(data.columns()) { - let bytes = column.get_array_memory_size(); - let name = field.name(); - if name == "hash" { - out.hash += bytes; - } else if name.starts_with("key") { - out.key += bytes; - } else { - out.other += bytes; + for part in &self.parts { + let data = part.batch.data(); + for (field, column) in data.schema().fields().iter().zip(data.columns()) { + let bytes = column.get_array_memory_size(); + let name = field.name(); + if name == "hash" { + out.hash += bytes; + } else if name.starts_with("key") { + out.key += bytes; + } else { + out.other += bytes; + } } } out } pub(crate) fn hash(&self, row: u32) -> u64 { - self.hashes.value(row as usize) + self.at(row).map_or(0, |(p, i)| p.hashes.value(i as usize)) } - /// The arrow-row key bytes, if the keys are in that format. - fn arrow_keys(&self) -> Option<&LargeBinaryArray> { - match &self.keys { - Cols::Arrow(k) => Some(k), - Cols::Rows(_) => None, - } + /// Whether the keys are arrow-row bytes (not row-layout rows). + fn has_arrow_keys(&self) -> bool { + self.parts + .first() + .is_some_and(|p| matches!(p.keys, Cols::Arrow(_))) + } + + /// The arrow-row key bytes of all rows, for sizing an index. + fn arrow_key_bytes(&self) -> usize { + self.parts + .iter() + .map(|p| match &p.keys { + Cols::Arrow(k) => k.values().len(), + Cols::Rows(_) => 0, + }) + .sum() } /// The row's key bytes, comparable byte for byte with another row's of @@ -305,8 +384,9 @@ impl RowArena { /// row-layout row. `None` for a layout with string views, whose rows /// point into a heap. pub(crate) fn key_bytes(&self, row: u32) -> Option<&[u8]> { + let (part, row) = self.at(row)?; let row = row as usize; - match &self.keys { + match &part.keys { Cols::Arrow(k) => Some(k.value(row)), Cols::Rows(RowCols { fixed, @@ -320,8 +400,14 @@ impl RowArena { /// The row's arrow-row payload bytes (empty if the arena has none or /// holds payload rows). pub(crate) fn payload(&self, row: u32) -> &[u8] { - match &self.payload { - Some(Cols::Arrow(p)) if (row as usize) < p.len() => p.value(row as usize), + match self.at(row) { + Some(( + ArenaPart { + payload: Some(Cols::Arrow(p)), + .. + }, + i, + )) if (i as usize) < p.len() => p.value(i as usize), _ => &[], } } @@ -329,52 +415,78 @@ impl RowArena { /// The row's payload row and its heap (empty if the arena has none or /// holds arrow-row payload). pub(crate) fn payload_row(&self, row: u32) -> (&[u8], &[u8]) { - match &self.payload { - Some(Cols::Rows(p)) => p.row(row), + match self.at(row) { + Some(( + ArenaPart { + payload: Some(Cols::Rows(p)), + .. + }, + i, + )) => p.row(i), _ => (&[], &[]), } } /// The row's weight. pub(crate) fn weight(&self, row: u32) -> i64 { - match self.batch.weights() { - Weights::Const(w) => *w, - Weights::Column(c) => c.value(row as usize), + match self.at(row) { + Some((p, i)) => match p.batch.weights() { + Weights::Const(w) => *w, + Weights::Column(c) => c.value(i as usize), + }, + None => 0, } } /// Whether every weight is 1. pub(crate) fn is_unweighted(&self) -> bool { - self.batch.is_unweighted() + self.parts.iter().all(|p| p.batch.is_unweighted()) } } impl RowSource for RowArena { #[inline] fn prefetch(&self, id: u32) { - if let Cols::Rows(k) = &self.keys { - k.prefetch(id); + if let Some(( + ArenaPart { + keys: Cols::Rows(k), + .. + }, + i, + )) = self.at(id) + { + k.prefetch(i); } } #[inline] fn row(&self, id: u32) -> (&[u8], &[u8]) { - match &self.keys { - Cols::Rows(k) => k.row(id), - Cols::Arrow(_) => (&[], &[]), + match self.at(id) { + Some(( + ArenaPart { + keys: Cols::Rows(k), + .. + }, + i, + )) => k.row(i), + _ => (&[], &[]), } } #[inline] fn hash(&self, id: u32) -> u64 { - self.hashes.values().get(id as usize).copied().unwrap_or(0) + self.at(id) + .and_then(|(p, i)| p.hashes.values().get(i as usize).copied()) + .unwrap_or(0) } #[inline] fn indexed(&self, id: u32) -> bool { - self.null_key - .as_ref() - .is_none_or(|f| (id as usize) >= f.len() || !f.value(id as usize)) + self.at(id).is_none_or(|(p, i)| { + p.null_key + .as_ref() + .is_none_or(|f| (i as usize) >= f.len() || !f.value(i as usize)) + }) } } @@ -441,6 +553,11 @@ impl JoinTable { /// per row. Row layout: the [`JoinIndex`] sized for one key per row and /// a link per row (it copies no keys and holds the heads itself). pub(crate) fn index_bytes(rows: usize, key_bytes: usize, row_layout: bool) -> usize { + // Plus the arena's chunk table (`RowArena::chunk_table_bytes`, at most a byte per row). + Self::index_only_bytes(rows, key_bytes, row_layout).saturating_add(rows) + } + + fn index_only_bytes(rows: usize, key_bytes: usize, row_layout: bool) -> usize { if row_layout { IndexChoice::Adaptive .bytes(rows) @@ -500,7 +617,10 @@ impl JoinTable { } Index::Rows { index, .. } => index.memory_size(), }; - index + self.next.capacity() * size_of::() + self.marks.capacity() * size_of::() + index + + self.next.capacity() * size_of::() + + self.marks.capacity() * size_of::() + + self.arena.chunk_table_bytes() } /// Bytes of the match bitmap of a table of `rows` rows @@ -591,15 +711,17 @@ impl JoinTable { let oom = |e: std::collections::TryReserveError| internal(e.to_string()); next.try_reserve_exact(n).map_err(oom)?; next.resize(n, NONE); - let index = match (arena.arrow_keys(), layout) { - (Some(arrow), _) => { + let index = match (arena.has_arrow_keys(), layout) { + (true, _) => { let mut keys = KeyTable::new(); - keys.reserve(n, arrow.values().len()); + keys.reserve(n, arena.arrow_key_bytes()); let mut heads: Vec = Vec::new(); heads.try_reserve_exact(n).map_err(oom)?; for row in (0..u32::try_from(n).unwrap_or(0)).filter(|&r| arena.indexed(r)) { - let (id, new) = - keys.insert_hashed(arrow.value(row as usize), arena.hash(row))?; + let Some(key) = arena.key_bytes(row) else { + continue; + }; + let (id, new) = keys.insert_hashed(key, arena.hash(row))?; if new { heads.push(row); } else if let Some(head) = heads.get_mut(id as usize) { @@ -610,7 +732,7 @@ impl JoinTable { } Index::Arrow { keys, heads } } - (None, Some(layout)) => Index::Rows { + (false, Some(layout)) => Index::Rows { index: JoinIndex::build_expecting( layout, &arena, @@ -622,7 +744,7 @@ impl JoinTable { )?, layout: Arc::clone(layout), }, - (None, None) => return Err(internal("join key rows without a layout".to_owned())), + (false, None) => return Err(internal("join key rows without a layout".to_owned())), }; Ok(Self { arena, From d0bb25e7fb666422ea78e79831bfb3b26c197de2 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:33:54 +0000 Subject: [PATCH 06/25] Merge origin/main into p2b/opus1-x3-build-bytes; the TPC-H EXPLAIN ANALYZE snapshots show the build counters; grace.rs file-size line Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .../snapshots/explain_analyze_tpch__analyze_q09.snap | 10 +++++----- .../snapshots/explain_analyze_tpch__analyze_q18.snap | 6 +++--- xtask/file-size-baseline.txt | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap index fdc187c7d..98b76839a 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap @@ -20,7 +20,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [n_name#47 AS nation#50, extract('year', o_orderdate#41) AS o_year#51, ((l_extendedprice#21 * (1 - l_discount#22)) - (ps_supplycost#35 * l_quantity#20)) AS amount#52] : · operator #24 Project p2: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [ps_suppkey#33, ps_partkey#32] = [l_suppkey#18, l_partkey#17], RuntimeFilter #5: ps_suppkey#33 -> spark_catalog.default.partsupp.ps_suppkey, RuntimeFilter #6: ps_partkey#32 -> spark_catalog.default.partsupp.ps_partkey - : · sink #3 HashJoinBuild p3: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #3 HashJoinBuild p3: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #23 HashJoinProbe p2: in 20 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=20 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.partsupp [ps_partkey#32, ps_suppkey#33, ps_supplycost#35] : · source #22 MemoryScan p2: out 20 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=80 runtime_filter_rows_dropped=20 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#5 [ps_suppkey]: range, tested 40, dropped 0, #6 [ps_partkey]: hash set of 10, tested 40, dropped 20 @@ -28,7 +28,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [o_orderdate#41, n_name#47, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #21 Project p3: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [o_orderkey#37] = [l_orderkey#16], RuntimeFilter #4: o_orderkey#37 -> spark_catalog.default.orders.o_orderkey - : · sink #4 HashJoinBuild p4: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #4 HashJoinBuild p4: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #20 HashJoinProbe p3: in 40 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.orders [o_orderkey#37, o_orderdate#41] : · source #19 MemoryScan p3: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#4 [o_orderkey]: range, tested 40, dropped 0 @@ -36,7 +36,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [n_name#47, l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #18 Project p4: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [n_nationkey#46] = [s_nationkey#12], RuntimeFilter #3: n_nationkey#46 -> spark_catalog.default.nation.n_nationkey - : · sink #5 HashJoinBuild p5: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #5 HashJoinBuild p5: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #17 HashJoinProbe p4: in 10 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=10 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.nation [n_nationkey#46, n_name#47] : · source #16 MemoryScan p4: out 10 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=25 runtime_filter_rows_dropped=15 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#3 [n_nationkey]: range, tested 25, dropped 15 @@ -44,7 +44,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [s_nationkey#12, l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #15 Project p5: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [s_suppkey#9] = [l_suppkey#18], RuntimeFilter #2: s_suppkey#9 -> spark_catalog.default.supplier.s_suppkey - : · sink #6 HashJoinBuild p6: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #6 HashJoinBuild p6: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #14 HashJoinProbe p5: in 10 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=10 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.supplier [s_suppkey#9, s_nationkey#12] : · source #13 MemoryScan p5: out 10 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=10 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [s_suppkey]: range, tested 10, dropped 0 @@ -52,7 +52,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #12 Project p6: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [l_partkey#17] = [p_partkey#0], RuntimeFilter #1: l_partkey#17 -> spark_catalog.default.lineitem.l_partkey - : · sink #7 HashJoinBuild p7: in 10 rows (1 batch); compute (max , drivers); peak mem ; build_rows=10 hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #7 HashJoinBuild p7: in 10 rows (1 batch); compute (max , drivers); peak mem ; build_rows=10 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #11 HashJoinProbe p6: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=80 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.lineitem [l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · source #10 MemoryScan p6: out 80 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=80 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [l_partkey]: hash set of 10, tested 160, dropped 80 diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap index 8c7fd0274..c891dbd34 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap @@ -21,12 +21,12 @@ TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] +- PartialHashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] : · sink #3 HashAggregate p3: in 20 rows (1 batch); compute (max , drivers); peak mem ; groups=5 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=5 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey - : · sink #4 HashJoinBuild p4: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #4 HashJoinBuild p4: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #20 HashJoinProbe p3: in 160 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] : : · operator #19 Project p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers) : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey - : : · sink #11 HashJoinBuild p6: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · sink #11 HashJoinBuild p6: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : : · operator #18 HashJoinProbe p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] : : · source #17 MemoryScan p3: out 160 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [l_orderkey]: range, tested 160, dropped 0 @@ -34,7 +34,7 @@ TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] : : · operator #16 Project p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) : +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey - : : · sink #12 HashJoinBuild p7: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · sink #12 HashJoinBuild p7: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : : · operator #15 HashJoinProbe p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] : : · source #14 MemoryScan p6: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [o_custkey]: range, tested 40, dropped 0, #3 [o_orderkey]: small set of 5, tested 0, dropped 0 diff --git a/xtask/file-size-baseline.txt b/xtask/file-size-baseline.txt index f44fc73a4..b41df9423 100644 --- a/xtask/file-size-baseline.txt +++ b/xtask/file-size-baseline.txt @@ -8,7 +8,8 @@ # +1 line (and in physical/instantiate.rs, window/tests.rs): the `udafs` spec field and the `Udf` aggregate variant (ADR 0045, T060a) # +37 lines over main: rows of batches that take the batch-local key map equal the columnar tables' (E2 step 2) 3277 crates/krishiv-exec/src/aggregate/tests.rs -2182 crates/krishiv-exec/src/join/grace.rs +# +11: the coalesce test asserts the merged-run bytes it now returns (X3' step 1, #586) +2193 crates/krishiv-exec/src/join/grace.rs 2003 crates/krishiv-exec/src/operators/sort/sort.rs 2215 crates/krishiv-exec/src/physical/instantiate.rs 2139 crates/krishiv-exec/src/planner/build.rs From a66923eb934b926957f23713ec323410027e8dd6 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:58:49 +0000 Subject: [PATCH 07/25] exec: finalize's copy plumbing removed (build_need reserves the index only; max_finalize_copy_bytes dropped); a test that a chunked arena reads and indexes as one batch, past the 256-chunk table too Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 86 ++++++------------- crates/krishiv-exec/src/join/grace.rs | 8 +- crates/krishiv-exec/src/join/mod.rs | 9 +- crates/krishiv-exec/src/join/table.rs | 55 ++++++++++++ ...plain_analyze__analyze_join_aggregate.snap | 2 +- .../explain_analyze_tpch__analyze_q09.snap | 10 +-- .../explain_analyze_tpch__analyze_q18.snap | 6 +- ...ain_graph__profile_join_aggregate_dot.snap | 2 +- ...graph__profile_join_aggregate_mermaid.snap | 2 +- docs/reference/explain-analyze.md | 4 +- 10 files changed, 98 insertions(+), 86 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 7750e84ce..f3aefe615 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -404,11 +404,10 @@ impl BuildSink { } } -/// What indexing a partition's in-memory rows reserves: `(index, copy)`, -/// its [`JoinTable`] index (and match bitmap when `marks`) and its -/// concatenation copy (always 0: the arena reads its chunks in place); `None` if no -/// rows are in memory. -fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize)> { +/// What indexing a partition's in-memory rows reserves: its [`JoinTable`] +/// index (and match bitmap when `marks`); `None` if no rows are in memory. +/// The arena reads the chunks in place, so no copy of the rows is reserved. +fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option { if part.chunks.is_empty() { return None; } @@ -418,8 +417,7 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz if marks { index = index.saturating_add(JoinTable::marks_bytes(rows)); } - // The arena reads the chunks in place (`RowArena::concat` copies nothing): no concatenation copy to fund. - Some((index, 0)) + Some(index) } /// A duration in whole nanoseconds, saturating. @@ -434,7 +432,7 @@ struct Job { bytes: usize, chunks: Vec, /// What indexing it reserves (see [`build_need`]). - need: (usize, usize), + need: usize, /// Its estimated distinct keys (row layout, when sketched). keys: Option, } @@ -711,13 +709,11 @@ impl Sink for BuildSink { let s = &self.0; let filter_memory = self.publish_filters(g, ctx); // Write what is left of spilled partitions; reserve the index of - // the rest, spilling the largest while it does not fit. A - // partition's table is built only while its concatenation copy - // is reserved, so the copies never need the build's size again. + // the rest, spilling the largest while it does not fit. The arenas + // read the chunks in place, so finalize needs no copy of the rows. let mut index = ctx.consumer().reservation(); - let mut held_copy = 0; if s.spill.is_some() { - held_copy = loop { + loop { // Once memory ran short, every partition goes to disk: the // probe phase then holds one partition's table per driver, // never tables that fill the budget and cannot spill. @@ -736,20 +732,20 @@ impl Sink for BuildSink { self.spill_collected(g, 0, ctx, true).await?; continue; } - let (indexes, copy) = { + let indexes = { let c = lock(&g.collect); let rows = g.hashing.rows(); let marks = s.join_type.builds_preserved(); - let needs = c.parts.iter().filter_map(|p| build_need(p, rows, marks)); - needs.fold((0usize, 0usize), |(i, m), (index, copy)| { - (i.saturating_add(index), usize::max(m, copy)) - }) + c.parts + .iter() + .filter_map(|p| build_need(p, rows, marks)) + .fold(0usize, usize::saturating_add) }; - if index.try_grow(indexes.saturating_add(copy)).is_ok() { - break copy; + if index.try_grow(indexes).is_ok() { + break; } self.spill_collected(g, 1, ctx, true).await?; - }; + } } let mut jobs: Vec = { let mut c = lock(&g.collect); @@ -780,34 +776,20 @@ impl Sink for BuildSink { // Nothing in memory is spillable any more. ctx.consumer().signal().set_spillable_bytes(0); // One blocking task per partition (the executor's blocking pool - // bounds how many run at once), each started only once its - // concatenation copy is reserved. A spilling build's reservation - // already funds the largest copy (`spare`), so one partition can - // always run; without a spill scope a partition's index and copy - // are asked for through arbitration when nothing else runs. + // bounds how many run at once). A spilling build reserved every + // index already; without a spill scope a partition's index is + // asked for through arbitration when nothing else runs. let spill = s.spill.is_some(); - let mut spare = held_copy; - // The concatenation copies reserved at once (for the counter). - let mut copying = 0usize; let mut jobs = std::collections::VecDeque::from(jobs); let mut running = futures::stream::FuturesUnordered::new(); let mut tables = Vec::with_capacity(jobs.len()); loop { if let Some(&Job { - need: (index_bytes, copy), - .. + need: index_bytes, .. }) = jobs.front() { - // A spilling build reserved every index already. - let grow = if spill { - copy - } else { - index_bytes.saturating_add(copy) - }; - let funded = if spill && copy <= spare { - spare -= copy; - true - } else if index.try_grow(grow).is_ok() { + let grow = if spill { 0 } else { index_bytes }; + let funded = if grow == 0 || index.try_grow(grow).is_ok() { true } else if running.is_empty() { index.grow(grow).await.map_err(KrishivError::from)?; @@ -818,15 +800,11 @@ impl Sink for BuildSink { if let Some(Job { part: p, chunks, - need: (index_bytes, copy), + need: index_bytes, keys, .. }) = jobs.pop_front_if(|_| funded) { - copying = copying.saturating_add(copy); - self.0 - .counters - .max(Counter::MaxFinalizeCopyBytes, copying as u64); let shared = Arc::clone(&self.0); let hashing = g.hashing.clone(); running.push(async move { @@ -858,7 +836,7 @@ impl Sink for BuildSink { table }) .await?; - Ok::<_, KrishivError>((p, table, copy, index_bytes)) + Ok::<_, KrishivError>((p, table, index_bytes)) }); continue; } @@ -866,8 +844,7 @@ impl Sink for BuildSink { let Some(done) = futures::StreamExt::next(&mut running).await else { break; }; - let (p, table, copy, index_bytes) = done?; - copying = copying.saturating_sub(copy); + let (p, table, index_bytes) = done?; // The index was reserved for one key per row: keep what it // holds. index.shrink( @@ -889,18 +866,7 @@ impl Sink for BuildSink { self.0.counters.add(Counter::HashEntries, keys as u64); self.0.counters.add(Counter::HashBuckets, buckets as u64); tables.push((p, table)); - if spill { - // Keep the largest copy funded; return the rest. - spare = spare.saturating_add(copy); - if spare > held_copy { - index.shrink(spare - held_copy); - spare = held_copy; - } - } else { - index.shrink(copy); - } } - index.shrink(spare); let mut c = lock(&g.collect); // Spilled partitions that hold rows: the probe defers only these // (an empty partition matches nothing, spilled or not). diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index 54094e703..13ebfbc0d 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -1240,24 +1240,20 @@ impl Loading { .cloned() .collect(); let rows = chunks.iter().map(Batch::num_rows).sum(); - let concat = 0; let headroom = self.task.probe.read_headroom(); let marks = if shared.join_type.builds_preserved() { JoinTable::marks_bytes(rows) } else { 0 }; - let need = JoinTable::index_bytes(rows, key_bytes(&chunks), hashing.rows()) - + marks - + concat - + headroom; + let need = + JoinTable::index_bytes(rows, key_bytes(&chunks), hashing.rows()) + marks + headroom; if self.memory.try_grow(need).is_err() { return Ok(Loaded::Need(need)); } let arena = RowArena::concat(shared.arena_schema(hashing), &chunks)?; drop(chunks); self.chunks.clear(); - self.memory.shrink(concat); let layout = shared.keys.layout().filter(|_| hashing.rows()); let mut table = JoinTable::build(arena, layout, hashing.g_seed())?; if shared.join_type.builds_preserved() { diff --git a/crates/krishiv-exec/src/join/mod.rs b/crates/krishiv-exec/src/join/mod.rs index 6f45d5128..49b551c6f 100644 --- a/crates/krishiv-exec/src/join/mod.rs +++ b/crates/krishiv-exec/src/join/mod.rs @@ -391,23 +391,19 @@ pub(crate) enum Counter { ArenaKeyBytes, /// The bytes of the finalized indexes (and match bitmaps). IndexBytes, - /// The most bytes of concatenation copies `finalize` held reserved at - /// once (partitions copying in parallel). - MaxFinalizeCopyBytes, /// The largest run of chunks a build-phase coalesce merged (each merge /// reserves the run again while it copies). MaxCoalesceCopyBytes, } /// The build sink's counters, by name. -const BUILD_COUNTERS: [(Counter, &str); 16] = [ +const BUILD_COUNTERS: [(Counter, &str); 15] = [ (Counter::BuildRows, "build_rows"), (Counter::SteadyReservedBytes, "steady_reserved_bytes"), (Counter::ArenaBytes, "arena_bytes"), (Counter::ArenaHashBytes, "arena_hash_bytes"), (Counter::ArenaKeyBytes, "arena_key_bytes"), (Counter::IndexBytes, "index_bytes"), - (Counter::MaxFinalizeCopyBytes, "max_finalize_copy_bytes"), (Counter::MaxCoalesceCopyBytes, "max_coalesce_copy_bytes"), (Counter::HashEntries, "hash_entries"), (Counter::HashBuckets, "hash_buckets"), @@ -440,7 +436,7 @@ const PROBE_DETAIL_COUNTERS: [(Counter, &str); 3] = [ /// Counters updated by every driver (relaxed: read once the query ended). #[derive(Debug, Default)] -pub(crate) struct Counters([std::sync::atomic::AtomicU64; 31]); +pub(crate) struct Counters([std::sync::atomic::AtomicU64; 30]); impl Counters { pub(crate) fn add(&self, c: Counter, v: u64) { @@ -468,7 +464,6 @@ impl Counters { Counter::RecursionDepth | Counter::MaxProbeLength | Counter::MaxPartitionNs - | Counter::MaxFinalizeCopyBytes | Counter::MaxCoalesceCopyBytes => Metric::Max(v), _ => Metric::Count(v), }; diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index d7f2561e9..2b0ed2330 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -960,6 +960,61 @@ mod tests { } } + /// An arena over several chunks (no copy, X3' step 3) reads every row as the one batch of the same rows does: + /// with the per-row chunk table (2-256 chunks), past it (the search over chunk starts), with empty chunks + /// skipped, and the index built over it chains the same rows. + #[test] + fn a_chunked_arena_reads_and_indexes_as_one_batch() { + let schema = + arena_schema(ArenaFormat::Arrow { payload: false }).unwrap_or_else(|e| panic!("{e}")); + let n = 600usize; + let key = |i: usize| format!("k{}", i % 97).into_bytes(); + let batch = |rows: std::ops::Range| { + Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(UInt64Array::from_iter_values( + rows.clone().map(|i| (i % 97) as u64), + )), + Arc::new(LargeBinaryArray::from_iter_values(rows.map(key))), + ], + ) + .unwrap_or_else(|e| panic!("{e}")), + ) + }; + let one = RowArena::concat(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")); + // Uneven chunks with empty ones between; then 300 chunks of 2 rows (past the 256-chunk table). + let uneven: Vec = [0..1, 1..1, 1..250, 250..251, 251..251, 251..n] + .into_iter() + .map(batch) + .collect(); + let many: Vec = (0..n / 2).map(|c| batch(2 * c..2 * c + 2)).collect(); + for chunks in [uneven, many] { + let arena = RowArena::concat(&schema, &chunks).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(arena.len(), n); + for row in 0..u32::try_from(n).unwrap_or(0) { + assert_eq!(arena.hash(row), one.hash(row), "row {row}"); + assert_eq!(arena.key_bytes(row), one.key_bytes(row), "row {row}"); + assert_eq!(arena.weight(row), 1, "row {row}"); + } + // Past the end: nothing, not a panic. + let end = u32::try_from(n).unwrap_or(0); + assert_eq!(arena.hash(end), 0); + assert_eq!(arena.key_bytes(end), None); + assert!(arena.memory_split().hash > 0); + let chained = JoinTable::build(arena, None, 0).unwrap_or_else(|e| panic!("{e}")); + let single = JoinTable::build( + RowArena::concat(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")), + None, + 0, + ) + .unwrap_or_else(|e| panic!("{e}")); + assert_eq!(chained.next(), single.next()); + assert_eq!(chained.index_size().0, single.index_size().0); + } + } + #[test] fn malformed_arenas_are_internal_errors() { let schema = Arc::new(Schema::new(vec![ diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap b/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap index 69e200a3c..d74888f39 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze__analyze_join_aggregate.snap @@ -20,7 +20,7 @@ Sort [name#3 ASC NULLS FIRST] +- Project [amount#1, name#3] : · operator #7 Project p2: in 1150 rows (4 batches); out 1150 rows (4 batches); compute (max , drivers) +- HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -> spark_catalog.default.orders.cust - : · sink #3 HashJoinBuild p3: in 4 rows (1 batch); compute (max , drivers); peak mem ; build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #3 HashJoinBuild p3: in 4 rows (1 batch); compute (max , drivers); peak mem ; build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #6 HashJoinProbe p2: in 1150 rows (4 batches); out 1150 rows (4 batches); compute (max , drivers); probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.orders [cust#0, amount#1], PushedFilters: [(amount#1 > 10L)] : · source #5 MemoryScan p2: out 1150 rows (4 batches); compute (max , drivers); io (max ); runtime_filter_rows_tested=1450 runtime_filter_rows_dropped=300 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [cust]: range, tested 1450, dropped 300 diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap index 98b76839a..8d0681833 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q09.snap @@ -20,7 +20,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [n_name#47 AS nation#50, extract('year', o_orderdate#41) AS o_year#51, ((l_extendedprice#21 * (1 - l_discount#22)) - (ps_supplycost#35 * l_quantity#20)) AS amount#52] : · operator #24 Project p2: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [ps_suppkey#33, ps_partkey#32] = [l_suppkey#18, l_partkey#17], RuntimeFilter #5: ps_suppkey#33 -> spark_catalog.default.partsupp.ps_suppkey, RuntimeFilter #6: ps_partkey#32 -> spark_catalog.default.partsupp.ps_partkey - : · sink #3 HashJoinBuild p3: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #3 HashJoinBuild p3: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #23 HashJoinProbe p2: in 20 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=20 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.partsupp [ps_partkey#32, ps_suppkey#33, ps_supplycost#35] : · source #22 MemoryScan p2: out 20 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=80 runtime_filter_rows_dropped=20 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#5 [ps_suppkey]: range, tested 40, dropped 0, #6 [ps_partkey]: hash set of 10, tested 40, dropped 20 @@ -28,7 +28,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [o_orderdate#41, n_name#47, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #21 Project p3: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [o_orderkey#37] = [l_orderkey#16], RuntimeFilter #4: o_orderkey#37 -> spark_catalog.default.orders.o_orderkey - : · sink #4 HashJoinBuild p4: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #4 HashJoinBuild p4: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #20 HashJoinProbe p3: in 40 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.orders [o_orderkey#37, o_orderdate#41] : · source #19 MemoryScan p3: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#4 [o_orderkey]: range, tested 40, dropped 0 @@ -36,7 +36,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [n_name#47, l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #18 Project p4: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [n_nationkey#46] = [s_nationkey#12], RuntimeFilter #3: n_nationkey#46 -> spark_catalog.default.nation.n_nationkey - : · sink #5 HashJoinBuild p5: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #5 HashJoinBuild p5: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #17 HashJoinProbe p4: in 10 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=10 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.nation [n_nationkey#46, n_name#47] : · source #16 MemoryScan p4: out 10 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=25 runtime_filter_rows_dropped=15 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#3 [n_nationkey]: range, tested 25, dropped 15 @@ -44,7 +44,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [s_nationkey#12, l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #15 Project p5: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [s_suppkey#9] = [l_suppkey#18], RuntimeFilter #2: s_suppkey#9 -> spark_catalog.default.supplier.s_suppkey - : · sink #6 HashJoinBuild p6: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #6 HashJoinBuild p6: in 80 rows (1 batch); compute (max , drivers); peak mem ; build_rows=80 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #14 HashJoinProbe p5: in 10 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=10 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.supplier [s_suppkey#9, s_nationkey#12] : · source #13 MemoryScan p5: out 10 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=10 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [s_suppkey]: range, tested 10, dropped 0 @@ -52,7 +52,7 @@ Sort [nation#50 ASC NULLS FIRST, o_year#51 DESC NULLS LAST] +- Project [l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · operator #12 Project p6: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers) +- HashJoin Inner, [l_partkey#17] = [p_partkey#0], RuntimeFilter #1: l_partkey#17 -> spark_catalog.default.lineitem.l_partkey - : · sink #7 HashJoinBuild p7: in 10 rows (1 batch); compute (max , drivers); peak mem ; build_rows=10 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #7 HashJoinBuild p7: in 10 rows (1 batch); compute (max , drivers); peak mem ; build_rows=10 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=10 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #11 HashJoinProbe p6: in 80 rows (1 batch); out 80 rows (1 batch); compute (max , drivers); probe_rows=80 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Scan spark_catalog.default.lineitem [l_orderkey#16, l_partkey#17, l_suppkey#18, l_quantity#20, l_extendedprice#21, l_discount#22] : · source #10 MemoryScan p6: out 80 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=80 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [l_partkey]: hash set of 10, tested 160, dropped 80 diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap index c891dbd34..918b9a923 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap @@ -21,12 +21,12 @@ TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] +- PartialHashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] : · sink #3 HashAggregate p3: in 20 rows (1 batch); compute (max , drivers); peak mem ; groups=5 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=5 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey - : · sink #4 HashJoinBuild p4: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · sink #4 HashJoinBuild p4: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : · operator #20 HashJoinProbe p3: in 160 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] : : · operator #19 Project p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers) : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey - : : · sink #11 HashJoinBuild p6: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · sink #11 HashJoinBuild p6: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : : · operator #18 HashJoinProbe p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] : : · source #17 MemoryScan p3: out 160 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [l_orderkey]: range, tested 160, dropped 0 @@ -34,7 +34,7 @@ TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] : : · operator #16 Project p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) : +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey - : : · sink #12 HashJoinBuild p7: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · sink #12 HashJoinBuild p7: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain : : · operator #15 HashJoinProbe p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] : : · source #14 MemoryScan p6: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [o_custkey]: range, tested 40, dropped 0, #3 [o_orderkey]: small set of 5, tested 0, dropped 0 diff --git a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap index fe45a7b20..c71e480af 100644 --- a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap +++ b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_dot.snap @@ -10,7 +10,7 @@ digraph { n4 [label="Exchange hashpartitioning(name#3)"]; n5 [label="PartialHashAggregate keys=[name#3], functions=[count(1), sum(amount#1)]\n#2 HashAggregate sink [p2]: in_rows=1150 in_batches=4 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 groups=4 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=4 hash_buckets= hash_resizes=1 hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows=4 exchange_bytes= exchange_keys=4 exchange_map_tasks=1 exchange_partitions=3 exchange_max_partition_rows=2"]; n6 [label="Project [amount#1, name#3]\n#7 Project operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"]; - n7 [label="HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -> spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="]; + n7 [label="HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -> spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="]; n8 [label="Scan spark_catalog.default.orders [cust#0, amount#1], PushedFilters: [(amount#1 > 10L)]\n#5 MemoryScan source [p2]: in_rows=0 in_batches=0 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 runtime_filter_rows_tested= runtime_filter_rows_dropped= runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [cust]: range, tested , dropped "]; n9 [label="BroadcastExchange"]; n10 [label="Scan spark_catalog.default.customers [id#2, name#3]\n#4 MemoryScan source [p3]: in_rows=0 in_batches=0 out_rows=4 out_batches=1 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"]; diff --git a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap index 275e210c7..779a0d698 100644 --- a/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap +++ b/crates/krishiv-session/tests/snapshots/explain_graph__profile_join_aggregate_mermaid.snap @@ -10,7 +10,7 @@ flowchart TD n4["Exchange hashpartitioning(name#3)"] n5["PartialHashAggregate keys=[name#3], functions=[count(1), sum(amount#1)]\n#2 HashAggregate sink [p2]: in_rows=1150 in_batches=4 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 groups=4 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=4 hash_buckets= hash_resizes=1 hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows=4 exchange_bytes= exchange_keys=4 exchange_map_tasks=1 exchange_partitions=3 exchange_max_partition_rows=2"] n6["Project [amount#1, name#3]\n#7 Project operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"] - n7["HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -#gt; spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="] + n7["HashJoin Inner, [cust#0] = [id#2], RuntimeFilter #1: cust#0 -#gt; spark_catalog.default.orders.cust\n#3 HashJoinBuild sink [p3]: in_rows=4 in_batches=1 out_rows=0 out_batches=0 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 build_rows=4 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_coalesce_copy_bytes= hash_entries=4 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain\n#6 HashJoinProbe operator [p2]: in_rows=1150 in_batches=4 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=0 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 probe_rows=1150 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length="] n8["Scan spark_catalog.default.orders [cust#0, amount#1], PushedFilters: [(amount#1 #gt; 10L)]\n#5 MemoryScan source [p2]: in_rows=0 in_batches=0 out_rows=1150 out_batches=4 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1 runtime_filter_rows_tested= runtime_filter_rows_dropped= runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [cust]: range, tested , dropped "] n9["BroadcastExchange"] n10["Scan spark_catalog.default.customers [id#2, name#3]\n#4 MemoryScan source [p3]: in_rows=0 in_batches=0 out_rows=4 out_batches=1 compute_ns= max_compute_ns= io_ns= max_io_ns= mem_wait_ns= sink_wait_ns= morsels=1 peak_bytes= spills=0 spilled_bytes= spill_files=0 spill_write_bytes= drivers=1"] diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index 97ff1e847..5c97ca46f 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -78,8 +78,8 @@ so it lists no operators of its own. | `hash_entries`, `hash_buckets` (load), `hash_index` | The operator's hash tables: distinct keys, buckets and the index kind (a join's probe side reports the tables its grace phase built after spilling as `grace_hash_entries` and `grace_hash_buckets`) (`swiss`: an open-addressing table of keys; `swiss+chain`: the same with duplicate build rows chained; `row-flat+chain` and `row-salted+chain`: the row-layout join index, flat or salted by size). | | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | | `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent concatenating rows and indexing them (summed over the tasks, which run in parallel), and the longest single task. | -| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the buffer bytes of the finalized row arenas (Arrow's `get_array_memory_size`, capacities included), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). The operator's `peak_reserved_bytes` also counts the transient concatenation copy `finalize` makes per partition. | -| `max_finalize_copy_bytes`, `max_coalesce_copy_bytes` | A hash join build's two transient copies: the most bytes of concatenation copies `finalize` held reserved at once, and the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). | +| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the buffer bytes of the finalized row arenas (Arrow's `get_array_memory_size`, capacities included), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). | +| `max_coalesce_copy_bytes` | A hash join build's transient copy: the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). Finalize makes no copy: the arena reads a partition's chunks in place. | | `broadcast_shared`, `broadcast_fallback` | A broadcast hash join on a multi-task run: 1 when the task probed the worker's shared build table, or 1 when the shared build did not fit `krishiv.exec.join.sharedBroadcastBytes` and the task built its own table instead. | | Pipelines | Per pipeline: wall time from its first driver's start to its last driver's end, drivers, threads that ran them, and its operators from source to sink. | From 5951fdb6777590bdedaa316663d52fb363764418 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:01:11 +0000 Subject: [PATCH 08/25] exec: a join arena finds a row's chunk through a block table (4 bytes per 64 rows, in cache) instead of a byte per row; an arena of no rows keeps an empty chunk, so an empty grace build partition still builds; a joined grace partition keeps its build part's reservations (review of X3' step 3) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 2 +- crates/krishiv-exec/src/join/fallback.rs | 2 +- crates/krishiv-exec/src/join/grace.rs | 10 +- crates/krishiv-exec/src/join/table.rs | 155 +++++++++++++++-------- docs/reference/explain-analyze.md | 2 +- 5 files changed, 115 insertions(+), 56 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index f3aefe615..e8fc5b64c 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -812,7 +812,7 @@ impl Sink for BuildSink { .run_blocking(move || { let t0 = std::time::Instant::now(); let schema = shared.arena_schema(&hashing); - let arena = RowArena::concat(schema, &chunks)?; + let arena = RowArena::over_chunks(schema, &chunks)?; drop(chunks); let t1 = std::time::Instant::now(); let layout = shared.keys.layout().filter(|_| hashing.rows()); diff --git a/crates/krishiv-exec/src/join/fallback.rs b/crates/krishiv-exec/src/join/fallback.rs index 15bff3719..1d1c2f9a0 100644 --- a/crates/krishiv-exec/src/join/fallback.rs +++ b/crates/krishiv-exec/src/join/fallback.rs @@ -323,7 +323,7 @@ fn build_rows( schema: &SchemaRef, ) -> Result { let data = chunk.data(); - let arena = RowArena::concat(shared.arena_schema(hashing), std::slice::from_ref(chunk))?; + let arena = RowArena::over_chunks(shared.arena_schema(hashing), std::slice::from_ref(chunk))?; let rows = u32::try_from(arena.len()).map_err(|_| internal("a chunk too large".to_owned()))?; let picks: Vec<(usize, u32)> = (0..rows).map(|r| (0, r)).collect(); let mut columns = shared.build_columns(hashing.rows(), &picks, |_| Some(&arena))?; diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index 13ebfbc0d..c41dfbd53 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -447,6 +447,8 @@ struct Active { table: JoinTable, _memory: Reservation, chunks: Vec, + /// The build part's own reservations (its in-memory chunks, if any). + _build_memory: Vec, _chunk_memory: Vec, files: VecDeque, reader: Option, @@ -664,7 +666,7 @@ fn reverse_blocking( // The stream runs in one blocking call: a cancelled query stops it // at the next chunk. query.check_cancelled()?; - let arena = RowArena::concat(schema, std::slice::from_ref(&chunk))?; + let arena = RowArena::over_chunks(schema, std::slice::from_ref(&chunk))?; let rows = u32::try_from(arena.len()).map_err(|_| internal("build chunk too large".to_owned()))?; // (probe row, build row of this chunk), evaluated before it is dropped. @@ -1172,10 +1174,12 @@ impl Loading { ctx.consumer().consumer().release_credit(); self.memory.shrink_to_credit(headroom); let Task { build, probe, .. } = self.task; - drop(build); Active { table, _memory: self.memory, + // The arena reads its chunks in place: any build chunks the task + // still held stay reserved while the table may read them. + _build_memory: build.memory, chunks: probe.chunks, _chunk_memory: probe.memory, files: probe.files.into(), @@ -1251,7 +1255,7 @@ impl Loading { if self.memory.try_grow(need).is_err() { return Ok(Loaded::Need(need)); } - let arena = RowArena::concat(shared.arena_schema(hashing), &chunks)?; + let arena = RowArena::over_chunks(shared.arena_schema(hashing), &chunks)?; drop(chunks); self.chunks.clear(); let layout = shared.keys.layout().filter(|_| hashing.rows()); diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index 2b0ed2330..caba527ec 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -27,7 +27,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use arrow_array::cast::AsArray; use arrow_array::types::UInt64Type; -use arrow_array::{Array, LargeBinaryArray, UInt64Array}; +use arrow_array::{Array, LargeBinaryArray, RecordBatch, UInt64Array}; use arrow_buffer::Buffer; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use krishiv_types::batch::{Batch, Weights}; @@ -261,75 +261,99 @@ impl ArenaPart { /// One partition's build rows: its chunks read in place (X3' step 3, design /// `P2-X3-copy-free-join-finalize`; no concatenation copy at finalize). Rows /// are numbered across the chunks in order; `starts[i]` is chunk `i`'s first -/// row id. +/// row id and `starts[parts.len()]` is the row count. #[derive(Debug)] pub(crate) struct RowArena { parts: Vec, starts: Vec, - /// Each row's chunk, when there are 2 to 256 chunks: one load instead of a - /// search over `starts` on the probe's path (1 byte per row). - chunk_of: Vec, + /// For each block of [`ARENA_BLOCK`] rows, the chunk holding the block's + /// first row, when there are two or more chunks: a row's chunk is that + /// one or a few after it (4 bytes per block, small enough to stay in + /// cache, unlike a byte per row). + block_chunk: Vec, len: usize, } +/// Rows per entry of [`RowArena::block_chunk`] (`1 << ARENA_BLOCK_SHIFT`). +const ARENA_BLOCK_SHIFT: u32 = 6; +const ARENA_BLOCK: usize = 1 << ARENA_BLOCK_SHIFT; + impl RowArena { /// The arena over `chunks` (same schema, in order), without copying their - /// rows. - pub(crate) fn concat(_schema: &SchemaRef, chunks: &[Batch]) -> Result { + /// rows. Empty chunks are skipped; an arena of no rows keeps one empty + /// chunk of `schema`, so its key format is still known. + pub(crate) fn over_chunks(schema: &SchemaRef, chunks: &[Batch]) -> Result { let mut parts = Vec::with_capacity(chunks.len().max(1)); - let mut starts = Vec::with_capacity(chunks.len().max(1)); + let mut starts = Vec::with_capacity(chunks.len() + 1); let mut len = 0usize; for chunk in chunks.iter().filter(|c| c.num_rows() > 0) { - let start = u32::try_from(len) - .map_err(|_| internal("a join partition has over 2^32 rows".to_owned()))?; - starts.push(start); + starts.push(Self::row_id(len)?); len = len.saturating_add(chunk.num_rows()); parts.push(ArenaPart::of(chunk.clone())?); } - if u32::try_from(len).is_err() { - return Err(internal("a join partition has over 2^32 rows".to_owned())); + if parts.is_empty() { + let empty = match chunks.first() { + Some(c) => c.clone(), + None => Batch::new(RecordBatch::new_empty(Arc::clone(schema))), + }; + starts.push(0); + parts.push(ArenaPart::of(empty)?); } - let mut chunk_of = Vec::new(); - if (2..=256).contains(&parts.len()) { - chunk_of - .try_reserve_exact(len) + starts.push(Self::row_id(len)?); + let mut block_chunk = Vec::new(); + if parts.len() > 1 { + block_chunk + .try_reserve_exact(len.div_ceil(ARENA_BLOCK)) .map_err(|e| internal(e.to_string()))?; - for (i, part) in parts.iter().enumerate() { - let id = u8::try_from(i).unwrap_or(u8::MAX); - chunk_of.extend(std::iter::repeat_n(id, part.batch.num_rows())); + let mut chunk = 0usize; + for first in (0..len).step_by(ARENA_BLOCK) { + while starts.get(chunk + 1).is_some_and(|&s| s as usize <= first) { + chunk += 1; + } + block_chunk.push(Self::row_id(chunk)?); } } Ok(Self { parts, starts, - chunk_of, + block_chunk, len, }) } - /// The bytes of the per-row chunk table (part of the index's cost). + fn row_id(n: usize) -> Result { + u32::try_from(n).map_err(|_| internal("a join partition has over 2^32 rows".to_owned())) + } + + /// The bytes of the block-to-chunk table (part of the index's cost). pub(crate) fn chunk_table_bytes(&self) -> usize { - self.chunk_of.capacity() + self.block_chunk.capacity() * size_of::() + } + + /// The bytes [`chunk_table_bytes`](Self::chunk_table_bytes) is at most + /// for `rows` rows. + pub(crate) fn chunk_table_bound(rows: usize) -> usize { + rows.div_ceil(ARENA_BLOCK) * size_of::() } /// The chunk holding row `row` and the row's index in it (`None` past /// the end). #[inline] fn at(&self, row: u32) -> Option<(&ArenaPart, u32)> { + if (row as usize) >= self.len { + return None; + } if let [one] = self.parts.as_slice() { - return ((row as usize) < self.len).then_some((one, row)); + return Some((one, row)); + } + let mut i = *self.block_chunk.get((row >> ARENA_BLOCK_SHIFT) as usize)? as usize; + // The block's chunk starts at or before `row`; a later one may too + // (chunks shorter than a block). + while self.starts.get(i + 1).is_some_and(|&s| s <= row) { + i += 1; } - let i = match self.chunk_of.get(row as usize) { - Some(&i) => usize::from(i), - // Past 256 chunks: the last chunk starting at or before `row`. - None if self.chunk_of.is_empty() => { - self.starts.partition_point(|&s| s <= row).checked_sub(1)? - } - None => return None, - }; let part = self.parts.get(i)?; - let local = row.checked_sub(*self.starts.get(i)?)?; - ((local as usize) < part.batch.num_rows()).then_some((part, local)) + Some((part, row.checked_sub(*self.starts.get(i)?)?)) } pub(crate) fn len(&self) -> usize { @@ -553,8 +577,9 @@ impl JoinTable { /// per row. Row layout: the [`JoinIndex`] sized for one key per row and /// a link per row (it copies no keys and holds the heads itself). pub(crate) fn index_bytes(rows: usize, key_bytes: usize, row_layout: bool) -> usize { - // Plus the arena's chunk table (`RowArena::chunk_table_bytes`, at most a byte per row). - Self::index_only_bytes(rows, key_bytes, row_layout).saturating_add(rows) + // Plus the arena's chunk table (`RowArena::chunk_table_bytes`). + Self::index_only_bytes(rows, key_bytes, row_layout) + .saturating_add(RowArena::chunk_table_bound(rows)) } fn index_only_bytes(rows: usize, key_bytes: usize, row_layout: bool) -> usize { @@ -861,7 +886,7 @@ mod tests { ], ) .unwrap_or_else(|e| panic!("{e}")); - RowArena::concat(&schema, &[Batch::new(data)]).unwrap_or_else(|e| panic!("{e}")) + RowArena::over_chunks(&schema, &[Batch::new(data)]).unwrap_or_else(|e| panic!("{e}")) } #[test] @@ -930,8 +955,8 @@ mod tests { ) }; let (a, b) = valid.split_at(valid.len() / 3); - let arena = - RowArena::concat(&schema, &[chunk(a), chunk(b)]).unwrap_or_else(|e| panic!("{e}")); + let arena = RowArena::over_chunks(&schema, &[chunk(a), chunk(b)]) + .unwrap_or_else(|e| panic!("{e}")); let l = layout.as_ref().filter(|_| rows); let t = JoinTable::build_with(arena, l, hashing.g_seed(), choice, None) .unwrap_or_else(|e| panic!("{e}")); @@ -961,8 +986,8 @@ mod tests { } /// An arena over several chunks (no copy, X3' step 3) reads every row as the one batch of the same rows does: - /// with the per-row chunk table (2-256 chunks), past it (the search over chunk starts), with empty chunks - /// skipped, and the index built over it chains the same rows. + /// through the block-to-chunk table, with chunks longer and shorter than a block, past 256 chunks, with empty + /// chunks skipped, and the index built over it chains the same rows. #[test] fn a_chunked_arena_reads_and_indexes_as_one_batch() { let schema = @@ -983,15 +1008,25 @@ mod tests { .unwrap_or_else(|e| panic!("{e}")), ) }; - let one = RowArena::concat(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")); - // Uneven chunks with empty ones between; then 300 chunks of 2 rows (past the 256-chunk table). - let uneven: Vec = [0..1, 1..1, 1..250, 250..251, 251..251, 251..n] - .into_iter() - .map(batch) - .collect(); + let one = RowArena::over_chunks(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")); + // Uneven chunks with empty ones between, a block-aligned edge (64, 128); then 300 chunks of 2 rows + // (many chunks per block, more than 256 chunks). + let uneven: Vec = [ + 0..1, + 1..1, + 1..64, + 64..128, + 128..250, + 250..251, + 251..251, + 251..n, + ] + .into_iter() + .map(batch) + .collect(); let many: Vec = (0..n / 2).map(|c| batch(2 * c..2 * c + 2)).collect(); for chunks in [uneven, many] { - let arena = RowArena::concat(&schema, &chunks).unwrap_or_else(|e| panic!("{e}")); + let arena = RowArena::over_chunks(&schema, &chunks).unwrap_or_else(|e| panic!("{e}")); assert_eq!(arena.len(), n); for row in 0..u32::try_from(n).unwrap_or(0) { assert_eq!(arena.hash(row), one.hash(row), "row {row}"); @@ -1005,7 +1040,7 @@ mod tests { assert!(arena.memory_split().hash > 0); let chained = JoinTable::build(arena, None, 0).unwrap_or_else(|e| panic!("{e}")); let single = JoinTable::build( - RowArena::concat(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")), + RowArena::over_chunks(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")), None, 0, ) @@ -1015,6 +1050,26 @@ mod tests { } } + /// An arena of no rows (a grace partition whose build side is empty, its + /// chunks all empty or none at all) keeps its key format, builds a table and + /// finds nothing (Fable review of X3' step 3, blocker 1). + #[test] + fn an_empty_arena_builds_and_matches_nothing() { + let schema = + arena_schema(ArenaFormat::Arrow { payload: false }).unwrap_or_else(|e| panic!("{e}")); + let empty = Batch::new(RecordBatch::new_empty(Arc::clone(&schema))); + for chunks in [vec![], vec![empty.clone()], vec![empty.clone(), empty]] { + let arena = RowArena::over_chunks(&schema, &chunks).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(arena.len(), 0); + assert!(arena.has_arrow_keys()); + assert_eq!(arena.hash(0), 0); + assert_eq!(arena.key_bytes(0), None); + assert_eq!(arena.chunk_table_bytes(), 0); + let table = JoinTable::build(arena, None, 0).unwrap_or_else(|e| panic!("{e}")); + assert!(table.next().is_empty()); + } + } + #[test] fn malformed_arenas_are_internal_errors() { let schema = Arc::new(Schema::new(vec![ @@ -1029,7 +1084,7 @@ mod tests { ], ) .unwrap_or_else(|e| panic!("{e}")); - assert!(RowArena::concat(&schema, &[Batch::new(data)]).is_err()); + assert!(RowArena::over_chunks(&schema, &[Batch::new(data)]).is_err()); let rows = arena(vec![1], vec![b"a"]); assert_eq!(RowSource::row(&rows, 0), (&[][..], &[][..])); assert_eq!(RowSource::hash(&rows, 9), 0); diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index 5c97ca46f..bbaeb9f43 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -77,7 +77,7 @@ so it lists no operators of its own. | `runtime_filter_*`, `runtime_filters` | Rows a scan's runtime filters tested and dropped, units and splits they pruned, and per filter the form its producer chose (`range`, `small set of N`, `hash set of N`, `bloom N B`). | | `hash_entries`, `hash_buckets` (load), `hash_index` | The operator's hash tables: distinct keys, buckets and the index kind (a join's probe side reports the tables its grace phase built after spilling as `grace_hash_entries` and `grace_hash_buckets`) (`swiss`: an open-addressing table of keys; `swiss+chain`: the same with duplicate build rows chained; `row-flat+chain` and `row-salted+chain`: the row-layout join index, flat or salted by size). | | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | -| `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent concatenating rows and indexing them (summed over the tasks, which run in parallel), and the longest single task. | +| `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent setting up the rows' chunks (no copy) and indexing them (summed over the tasks, which run in parallel), and the longest single task. | | `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the buffer bytes of the finalized row arenas (Arrow's `get_array_memory_size`, capacities included), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). | | `max_coalesce_copy_bytes` | A hash join build's transient copy: the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). Finalize makes no copy: the arena reads a partition's chunks in place. | | `broadcast_shared`, `broadcast_fallback` | A broadcast hash join on a multi-task run: 1 when the task probed the worker's shared build table, or 1 when the shared build did not fit `krishiv.exec.join.sharedBroadcastBytes` and the task built its own table instead. | From ab45aa5244df64d98fe6d1ce66c516e350711ce8 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:09:42 +0000 Subject: [PATCH 09/25] exec: a join arena in blocks of 2^14 rows (X3' step 3 option C): a row is row id & mask of block id >> 14, with no chunk table or search; build chunks are copied once into blocks as they come, a grace reload blocks its frames as it reads them, finalize slices whole blocks in place and copies only the tails (reserved; the tails beside chunks that live on stay reserved) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Study 10 §4: option B's dependent chunk-table load before each row fired the probe_table +20% kill line. The tiered coalesce stays for the probe side; both move to join/block.rs (grace.rs under its file-size line). An arena's arrow key bytes count each block's own range. Opus review ops/reviews/2026-11-07-opus-1-x3-option-c-diff.md, fixes listed. Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/CLAUDE.md | 3 +- crates/krishiv-exec/src/join/block.rs | 342 ++++++++++++++++++++++++++ crates/krishiv-exec/src/join/build.rs | 20 +- crates/krishiv-exec/src/join/grace.rs | 154 ++---------- crates/krishiv-exec/src/join/mod.rs | 1 + crates/krishiv-exec/src/join/table.rs | 326 +++++++++++++++++------- docs/reference/explain-analyze.md | 6 +- 7 files changed, 614 insertions(+), 238 deletions(-) create mode 100644 crates/krishiv-exec/src/join/block.rs diff --git a/crates/krishiv-exec/CLAUDE.md b/crates/krishiv-exec/CLAUDE.md index 69054dcbc..d77f04132 100644 --- a/crates/krishiv-exec/CLAUDE.md +++ b/crates/krishiv-exec/CLAUDE.md @@ -28,7 +28,8 @@ executor: `mock_exec` in `krishiv-testkit/tests/file_format_mock.rs`. `join/` hash join (P2-T052a): `mod.rs` API + Spark semantics · `keys.rs` row-format keys, seeded foldhash, radix partitions · `table.rs` row arena + `JoinTable` (shared `KeyTable`, E02) · `build.rs` build sink (also the runtime filter producer, P2-T163: per-driver partials, published at finalize, `Disabled` on drop) · -`probe.rs` probe operator + match cursor · `grace.rs` spill partitions, flush-phase grace join · `broadcast.rs` +`probe.rs` probe operator + match cursor · `grace.rs` spill partitions, flush-phase grace join · `block.rs` a +partition's chunks kept few: the probe side's tiered merge, the build side's cut into the arena's 2^14-row blocks · `broadcast.rs` (P2-T177 unit 3) `BroadcastTables`, a worker resource (`WorkerContext::with_broadcasts`): a broadcast join over a cut broadcast exchange builds its table once per job and worker (its own query over the resource's pool, no spill, its reservations never abort a query) and every task probes it through a `PrebuiltSink` that republishes its runtime diff --git a/crates/krishiv-exec/src/join/block.rs b/crates/krishiv-exec/src/join/block.rs new file mode 100644 index 000000000..8a193d59d --- /dev/null +++ b/crates/krishiv-exec/src/join/block.rs @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Keeping a partition's in-memory chunks few: the probe side's size-tiered +//! merge ([`Part::coalesce`]) and the build side's cut into the row arena's +//! blocks of [`ARENA_ROWS`] rows ([`Part::block`], [`block_chunks`]; X3' +//! step 3 option C). + +use arrow_schema::SchemaRef; +use krishiv_memory::Reservation; +use krishiv_types::batch::Batch; + +use super::grace::Part; +use super::table::ARENA_ROWS; +use crate::error::{Result, internal}; + +impl Part { + /// Merges runs of small in-memory chunks (each chunk carries fixed array + /// overhead), if each merged copy can be reserved first. + /// + /// Size-tiered, like an LSM tree: a chunk's tier is `log16` of its bytes, + /// and only a trailing run of at least `FANOUT` chunks no higher in tier + /// than the newest one is merged. Merging every chunk held whenever the + /// count reached the fan-out copied the whole partition again every 15 + /// pushes (quadratic in its size); tiered, a row is copied about once + /// per tier it climbs. + /// + /// Returns the bytes of the largest run merged (each merge reserves the + /// run again while it copies: the transient peak), 0 when none was. + pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result { + const FANOUT: usize = 16; + let size = |c: &Batch| c.data().get_array_memory_size(); + let tier = |c: &Batch| size(c).max(1).ilog(FANOUT); + let mut largest = 0; + while self.chunks.len() >= FANOUT { + let Some(top) = self.chunks.last().map(tier) else { + return Ok(largest); + }; + let run = self + .chunks + .iter() + .rev() + .take_while(|c| tier(c) <= top) + .count(); + if run < FANOUT { + return Ok(largest); + } + let from = self.chunks.len() - run; + let old: usize = self.chunks[from..].iter().map(size).sum(); + let Some(m) = self.memory.first_mut().filter(|m| m.size() >= old) else { + return Ok(largest); + }; + if m.try_grow(old).is_err() { + return Ok(largest); + } + let merged = + Batch::concat(schema, &self.chunks[from..]).map_err(|e| internal(e.to_string())); + let merged = match merged { + Ok(b) if size(&b) <= old => b, + other => { + m.shrink(old); + return other.map(|_| largest); + } + }; + largest = largest.max(old); + m.shrink(old + old - size(&merged)); + self.chunks.truncate(from); + self.chunks.push(merged); + } + Ok(largest) + } + + /// [`block_chunks`] over the part's chunks, the copies reserved in its + /// first reservation (the one its chunks are reserved in). + pub(crate) fn block(&mut self, schema: &SchemaRef) -> Result { + match self.memory.first_mut() { + Some(m) => block_chunks(&mut self.chunks, m, schema), + None => Ok(0), + } + } +} + +/// Cuts build rows into the arena's blocks as they come (X3' step 3 +/// option C): once the trailing chunks shorter than a block hold a +/// block's rows, they are copied into one block of exactly +/// [`ARENA_ROWS`] rows (the last chunk split, its tail left as a +/// zero-copy slice), if the copy can be reserved first. Chunks of a block +/// or more stay as they are: the arena slices them. Each row is copied at +/// most once here, and finalize copies only the tails left. +/// +/// Returns the bytes of the largest copy (each one reserves its chunks +/// again while it copies: the transient peak), 0 when none was made. +pub(crate) fn block_chunks( + chunks: &mut Vec, + m: &mut Reservation, + schema: &SchemaRef, +) -> Result { + let size = |c: &Batch| c.data().get_array_memory_size(); + let mut largest = 0; + loop { + let run = chunks + .iter() + .rev() + .take_while(|c| c.num_rows() < ARENA_ROWS) + .count(); + let from = chunks.len() - run; + // The chunks that make up the block: whole ones, then the head of + // the last. + let mut held = 0usize; + let Some(last) = chunks[from..].iter().position(|c| { + held += c.num_rows(); + held >= ARENA_ROWS + }) else { + return Ok(largest); + }; + let last = from + last; + let head = ARENA_ROWS - (held - chunks[last].num_rows()); + let whole = if head == chunks[last].num_rows() { + last + 1 + } else { + last + }; + let freed: usize = chunks[from..whole].iter().map(size).sum(); + let old = freed + + if whole == last { + size(&chunks[last]) + } else { + 0 + }; + if m.size() < old { + return Ok(largest); + } + if m.try_grow(old).is_err() { + return Ok(largest); + } + let pieces = chunks[from..last] + .iter() + .cloned() + .map(Ok) + .chain(std::iter::once(chunks[last].slice(0, head))) + .collect::, _>>() + .and_then(|p| Batch::concat(schema, &p)) + .map_err(|e| internal(e.to_string())); + let n = chunks[last].num_rows(); + let tail = (whole == last).then(|| { + chunks[last] + .slice(head, n - head) + .map_err(|e| internal(e.to_string())) + }); + let (block, tail) = match (pieces, tail.transpose()) { + (Ok(b), Ok(t)) => (b, t), + (Err(e), _) | (_, Err(e)) => { + m.shrink(old); + return Err(e); + } + }; + // A copy larger than its sources (buffer rounding) is kept if the + // excess can be reserved; dropping it would redo it on every push. + let new = size(&block); + let excess = new.saturating_sub(old); + if excess > 0 && m.try_grow(excess).is_err() { + m.shrink(old); + return Ok(largest); + } + largest = largest.max(old); + // Held now: the block, and the last chunk while its tail lives + // (`old + excess >= new`). + m.shrink(old + excess + freed - new); + chunks.truncate(from); + chunks.push(block); + chunks.extend(tail); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::cast::AsArray; + use arrow_array::{Int64Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use krishiv_memory::{MemoryConfig, MemoryPool, TreePool}; + + use super::*; + use crate::join::grace::reserve_chunks; + + /// Coalescing is size-tiered: a merged chunk is not copied again by the + /// next run of small chunks (merging all chunks every 16 pushes was + /// quadratic), the chunk count stays bounded, and no row is lost. + #[test] + fn coalescing_does_not_recopy_merged_chunks() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("coalesce", None); + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("s", DataType::Utf8, false), + ])); + let chunk = |i: i64| { + let k = Int64Array::from_iter_values(i * 300..(i + 1) * 300); + let s = StringArray::from_iter_values((0..300).map(|j| format!("row-{i}-{j}"))); + Batch::new( + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k), Arc::new(s)]).unwrap(), + ) + }; + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let push = |parts: &mut Vec, i: i64| { + let c = chunk(i); + reserve_chunks(parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].coalesce(&schema).unwrap() + }; + let sizes: usize = (0..16) + .map(|i| chunk(i).data().get_array_memory_size()) + .sum(); + let mut merged_run = 0; + for i in 0..16 { + merged_run = push(&mut parts, i); + } + assert_eq!(parts[0].chunks.len(), 1, "16 small chunks merge"); + // It reports the run it merged: the 16 chunks' bytes. + assert_eq!(merged_run, sizes); + let merged = Arc::clone(parts[0].chunks[0].data().column(0)); + for i in 16..31 { + assert_eq!(push(&mut parts, i), 0, "no merge below the fan-out"); + } + assert_eq!(parts[0].chunks.len(), 16); + let first = parts[0].chunks[0].data().column(0); + assert!( + Arc::ptr_eq(first, &merged), + "the merged chunk was copied again" + ); + let mut most = 0; + for i in 31..1024 { + push(&mut parts, i); + most = most.max(parts[0].chunks.len()); + } + assert!(most < 3 * 16, "{most} chunks held"); + assert_eq!(parts[0].mem_rows(), 1024 * 300); + let mut keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + keys.sort_unstable(); + assert!( + keys.into_iter().eq(0..1024 * 300), + "rows lost or duplicated" + ); + drop(parts); + assert_eq!(pool.reserved(), 0); + } + + /// Build chunks are cut into the arena's blocks as they come: small + /// chunks are copied once into blocks of exactly `ARENA_ROWS` rows (the + /// chunk that crosses a block's end split, its tail kept), chunks of a + /// block or more are left alone, no row is lost and the reservation + /// follows what is held. + #[test] + fn build_chunks_are_cut_into_blocks_once() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("block", None); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); + let chunk = |from: i64, n: i64| { + Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from_iter_values(from..from + n))], + ) + .unwrap(), + ) + }; + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + let mut push = |parts: &mut Vec, n: i64| { + let c = chunk(next, n); + next += n; + reserve_chunks(parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].block(&schema).unwrap() + }; + let block = ARENA_ROWS as i64; + // 1,000-row chunks: a block once 17 of them are held, the 17th split. + let mut copied = 0; + for _ in 0..16 { + copied = copied.max(push(&mut parts, 1000)); + } + assert_eq!(copied, 0); + assert!(push(&mut parts, 1000) > 0); + let rows: Vec = parts[0].chunks.iter().map(Batch::num_rows).collect(); + assert_eq!(rows, [ARENA_ROWS, 17 * 1000 - ARENA_ROWS]); + // A chunk of a block or more is not copied, and ends the run before it. + assert_eq!(push(&mut parts, 2 * block + 7), 0); + let blocked = Arc::clone(parts[0].chunks[0].data().column(0)); + for _ in 0..40 { + push(&mut parts, 1000); + } + assert!(Arc::ptr_eq(parts[0].chunks[0].data().column(0), &blocked)); + assert!( + parts[0] + .chunks + .iter() + .filter(|c| c.num_rows() < ARENA_ROWS) + .count() + <= 17 + ); + let held: usize = parts[0] + .chunks + .iter() + .map(|c| c.data().get_array_memory_size()) + .sum(); + assert!( + parts[0].bytes() >= held, + "{} reserved, {held} held", + parts[0].bytes() + ); + let mut keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + keys.sort_unstable(); + assert!(keys.into_iter().eq(0..next), "rows lost or duplicated"); + drop(parts); + assert_eq!(pool.reserved(), 0); + } +} diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index e8fc5b64c..a33f4217e 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -405,8 +405,9 @@ impl BuildSink { } /// What indexing a partition's in-memory rows reserves: its [`JoinTable`] -/// index (and match bitmap when `marks`); `None` if no rows are in memory. -/// The arena reads the chunks in place, so no copy of the rows is reserved. +/// index (and match bitmap when `marks`), and the copy of the chunk tails +/// the arena joins into blocks ([`RowArena::reblock_bytes`]); `None` if no +/// rows are in memory. fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option { if part.chunks.is_empty() { return None; @@ -417,7 +418,7 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option { if marks { index = index.saturating_add(JoinTable::marks_bytes(rows)); } - Some(index) + Some(index.saturating_add(RowArena::reblock_bytes(&part.chunks, 0).0)) } /// A duration in whole nanoseconds, saturating. @@ -433,6 +434,9 @@ struct Job { chunks: Vec, /// What indexing it reserves (see [`build_need`]). need: usize, + /// Of that, the arena's copied tails that stay beside chunks it also + /// slices: reserved through the probe ([`RowArena::reblock_bytes`]). + kept: usize, /// Its estimated distinct keys (row layout, when sketched). keys: Option, } @@ -647,7 +651,7 @@ impl Sink for BuildSink { } for (p, c) in chunks { l.parts[p].chunks.push(c); - let merged = l.parts[p].coalesce(s.arena_schema(&g.hashing))?; + let merged = l.parts[p].block(s.arena_schema(&g.hashing))?; s.counters.max(Counter::MaxCoalesceCopyBytes, merged as u64); } if let Some((sketches, _)) = &mut l.sketches { @@ -710,7 +714,7 @@ impl Sink for BuildSink { let filter_memory = self.publish_filters(g, ctx); // Write what is left of spilled partitions; reserve the index of // the rest, spilling the largest while it does not fit. The arenas - // read the chunks in place, so finalize needs no copy of the rows. + // read the blocks in place; finalize copies only the chunk tails. let mut index = ctx.consumer().reservation(); if s.spill.is_some() { loop { @@ -764,6 +768,7 @@ impl Sink for BuildSink { .map(|(i, need, p, keys)| Job { part: i, bytes: p.bytes(), + kept: RowArena::reblock_bytes(&p.chunks, 0).1, chunks: std::mem::take(&mut p.chunks), need, keys, @@ -802,6 +807,7 @@ impl Sink for BuildSink { chunks, need: index_bytes, keys, + kept, .. }) = jobs.pop_front_if(|_| funded) { @@ -836,7 +842,7 @@ impl Sink for BuildSink { table }) .await?; - Ok::<_, KrishivError>((p, table, index_bytes)) + Ok::<_, KrishivError>((p, table, index_bytes.saturating_sub(kept))) }); continue; } @@ -846,7 +852,7 @@ impl Sink for BuildSink { }; let (p, table, index_bytes) = done?; // The index was reserved for one key per row: keep what it - // holds. + // holds (and the arena's copied tails that stay). index.shrink( index_bytes .saturating_sub(table.index_memory()) diff --git a/crates/krishiv-exec/src/join/grace.rs b/crates/krishiv-exec/src/join/grace.rs index c41dfbd53..ad0954efb 100644 --- a/crates/krishiv-exec/src/join/grace.rs +++ b/crates/krishiv-exec/src/join/grace.rs @@ -33,6 +33,7 @@ use krishiv_memory::{ }; use krishiv_types::batch::Batch; +use super::block::block_chunks; use super::build::BuildGlobal; use super::keys::{Hashing, MAX_DEPTH, PARTITIONS}; use super::probe::{Cursor, Tables}; @@ -78,61 +79,6 @@ impl Part { self.chunks.iter().map(Batch::num_rows).sum() } - /// Merges runs of small in-memory chunks (each chunk carries fixed array - /// overhead), if each merged copy can be reserved first. - /// - /// Size-tiered, like an LSM tree: a chunk's tier is `log16` of its bytes, - /// and only a trailing run of at least `FANOUT` chunks no higher in tier - /// than the newest one is merged. Merging every chunk held whenever the - /// count reached the fan-out copied the whole partition again every 15 - /// pushes (quadratic in its size); tiered, a row is copied about once - /// per tier it climbs. - /// - /// Returns the bytes of the largest run merged (each merge reserves the - /// run again while it copies: the transient peak), 0 when none was. - pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result { - const FANOUT: usize = 16; - let size = |c: &Batch| c.data().get_array_memory_size(); - let tier = |c: &Batch| size(c).max(1).ilog(FANOUT); - let mut largest = 0; - while self.chunks.len() >= FANOUT { - let Some(top) = self.chunks.last().map(tier) else { - return Ok(largest); - }; - let run = self - .chunks - .iter() - .rev() - .take_while(|c| tier(c) <= top) - .count(); - if run < FANOUT { - return Ok(largest); - } - let from = self.chunks.len() - run; - let old: usize = self.chunks[from..].iter().map(size).sum(); - let Some(m) = self.memory.first_mut().filter(|m| m.size() >= old) else { - return Ok(largest); - }; - if m.try_grow(old).is_err() { - return Ok(largest); - } - let merged = - Batch::concat(schema, &self.chunks[from..]).map_err(|e| internal(e.to_string())); - let merged = match merged { - Ok(b) if size(&b) <= old => b, - other => { - m.shrink(old); - return other.map(|_| largest); - } - }; - largest = largest.max(old); - m.shrink(old + old - size(&merged)); - self.chunks.truncate(from); - self.chunks.push(merged); - } - Ok(largest) - } - pub(crate) fn absorb(&mut self, other: Self) { self.chunks.extend(other.chunks); self.files.extend(other.files); @@ -1199,6 +1145,16 @@ impl Loading { return Ok(Loaded::Need(bytes)); } self.chunks.push(b); + // Spill frames are slices of 64 KiB: joined into blocks as + // they come, one block's copy at a time, not all at the end. + let copied = block_chunks( + &mut self.chunks, + &mut self.memory, + shared.arena_schema(hashing), + )?; + shared + .counters + .max(Counter::MaxCoalesceCopyBytes, copied as u64); } if self.reader.is_none() { let Some(file) = self.task.build.files.get(self.file) else { @@ -1234,7 +1190,8 @@ impl Loading { } // Every row read: the table's index and the probe rows' read-back // headroom (exact, so a partition that loads can always be joined). - // The arena reads the chunks in place: no concatenated copy. + // The arena reads the chunks in place, copying only blocks that span + // chunks. let chunks: Vec = self .task .build @@ -1250,14 +1207,20 @@ impl Loading { } else { 0 }; - let need = - JoinTable::index_bytes(rows, key_bytes(&chunks), hashing.rows()) + marks + headroom; + // Blocks that span chunks are copied: reserved, and the part beside + // chunks that live on (the task's own build chunks) stays reserved. + let (copy, kept) = RowArena::reblock_bytes(&chunks, self.task.build.chunks.len()); + let need = JoinTable::index_bytes(rows, key_bytes(&chunks), hashing.rows()) + + marks + + headroom + + copy; if self.memory.try_grow(need).is_err() { return Ok(Loaded::Need(need)); } let arena = RowArena::over_chunks(shared.arena_schema(hashing), &chunks)?; drop(chunks); self.chunks.clear(); + self.memory.shrink(copy - kept); let layout = shared.keys.layout().filter(|_| hashing.rows()); let mut table = JoinTable::build(arena, layout, hashing.g_seed())?; if shared.join_type.builds_preserved() { @@ -1615,81 +1578,6 @@ mod tests { } } - /// Coalescing is size-tiered: a merged chunk is not copied again by the - /// next run of small chunks (merging all chunks every 16 pushes was - /// quadratic), the chunk count stays bounded, and no row is lost. - #[test] - fn coalescing_does_not_recopy_merged_chunks() { - let pool: Arc = TreePool::new(&MemoryConfig::default()); - let id = pool.register("coalesce", None); - let schema = Arc::new(Schema::new(vec![ - Field::new("k", DataType::Int64, false), - Field::new("s", DataType::Utf8, false), - ])); - let chunk = |i: i64| { - let k = Int64Array::from_iter_values(i * 300..(i + 1) * 300); - let s = StringArray::from_iter_values((0..300).map(|j| format!("row-{i}-{j}"))); - Batch::new( - RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k), Arc::new(s)]).unwrap(), - ) - }; - let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( - Arc::clone(&pool), - id, - ))]; - let push = |parts: &mut Vec, i: i64| { - let c = chunk(i); - reserve_chunks(parts, &[(0, c.clone())]).unwrap(); - parts[0].chunks.push(c); - parts[0].coalesce(&schema).unwrap() - }; - let sizes: usize = (0..16) - .map(|i| chunk(i).data().get_array_memory_size()) - .sum(); - let mut merged_run = 0; - for i in 0..16 { - merged_run = push(&mut parts, i); - } - assert_eq!(parts[0].chunks.len(), 1, "16 small chunks merge"); - // It reports the run it merged: the 16 chunks' bytes. - assert_eq!(merged_run, sizes); - let merged = Arc::clone(parts[0].chunks[0].data().column(0)); - for i in 16..31 { - assert_eq!(push(&mut parts, i), 0, "no merge below the fan-out"); - } - assert_eq!(parts[0].chunks.len(), 16); - let first = parts[0].chunks[0].data().column(0); - assert!( - Arc::ptr_eq(first, &merged), - "the merged chunk was copied again" - ); - let mut most = 0; - for i in 31..1024 { - push(&mut parts, i); - most = most.max(parts[0].chunks.len()); - } - assert!(most < 3 * 16, "{most} chunks held"); - assert_eq!(parts[0].mem_rows(), 1024 * 300); - let mut keys: Vec = parts[0] - .chunks - .iter() - .flat_map(|c| { - let k = c - .data() - .column(0) - .as_primitive::(); - k.values().to_vec() - }) - .collect(); - keys.sort_unstable(); - assert!( - keys.into_iter().eq(0..1024 * 300), - "rows lost or duplicated" - ); - drop(parts); - assert_eq!(pool.reserved(), 0); - } - /// Probe rows deferred in many small chunks are coalesced, spilled in /// slices, and read back by the grace phase with exactly the headroom /// [`load`] reserves: with the pool otherwise full, no read reaches it. diff --git a/crates/krishiv-exec/src/join/mod.rs b/crates/krishiv-exec/src/join/mod.rs index 49b551c6f..3efa52564 100644 --- a/crates/krishiv-exec/src/join/mod.rs +++ b/crates/krishiv-exec/src/join/mod.rs @@ -154,6 +154,7 @@ //! the tables: two clock reads per batch, for //! [`QueryMetrics::hash_probe_share`](crate::QueryMetrics::hash_probe_share)). +mod block; mod broadcast; mod build; mod fallback; diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index caba527ec..b96e06c67 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -213,7 +213,7 @@ impl Cols { } } -/// An arena's bytes by part ([`RowArena::memory_split`]), as `get_array_memory_size` counts them. +/// An arena's bytes by part ([`RowArena::memory_split`]): each block's rows' share of its buffers. #[derive(Debug, Default, Clone, Copy)] pub(crate) struct ArenaBytes { pub(crate) hash: usize, @@ -222,9 +222,11 @@ pub(crate) struct ArenaBytes { pub(crate) other: usize, } -/// One chunk of an arena: a batch of build rows and its columns, read in place. +/// One block of an arena: a batch of build rows and its columns, read in place. #[derive(Debug)] struct ArenaPart { + /// `batch.num_rows()`, kept beside the columns `at` reads. + rows: u32, batch: Batch, hashes: UInt64Array, keys: Cols, @@ -237,6 +239,7 @@ struct ArenaPart { impl ArenaPart { fn of(batch: Batch) -> Result { let bad = || internal("malformed join row arena".to_owned()); + let rows = u32::try_from(batch.num_rows()).map_err(|_| bad())?; let hashes = batch.data().column_by_name("hash").ok_or_else(bad)?; let hashes = hashes .as_primitive_opt::() @@ -249,6 +252,7 @@ impl ArenaPart { None => None, }; Ok(Self { + rows, batch, hashes, keys, @@ -258,116 +262,199 @@ impl ArenaPart { } } -/// One partition's build rows: its chunks read in place (X3' step 3, design -/// `P2-X3-copy-free-join-finalize`; no concatenation copy at finalize). Rows -/// are numbered across the chunks in order; `starts[i]` is chunk `i`'s first -/// row id and `starts[parts.len()]` is the row count. +/// Rows per arena block: `1 << ARENA_SHIFT` (X3' step 3 option C, study 10 +/// §4: 16,384 rows, about 1-6 MB of build rows). +pub(crate) const ARENA_SHIFT: u32 = 14; +/// [`ARENA_SHIFT`] as a row count. +pub(crate) const ARENA_ROWS: usize = 1 << ARENA_SHIFT; + +/// One partition's build rows, read in place (X3' step 3, design +/// `P2-X3-copy-free-join-finalize` option C): blocks of exactly +/// `1 << shift` rows, the last one shorter, so row `id` is row +/// `id & mask` of block `id >> shift`, found without a table or a search. +/// Rows are numbered across the blocks in order. #[derive(Debug)] pub(crate) struct RowArena { parts: Vec, - starts: Vec, - /// For each block of [`ARENA_BLOCK`] rows, the chunk holding the block's - /// first row, when there are two or more chunks: a row's chunk is that - /// one or a few after it (4 bytes per block, small enough to stay in - /// cache, unlike a byte per row). - block_chunk: Vec, + shift: u32, len: usize, } -/// Rows per entry of [`RowArena::block_chunk`] (`1 << ARENA_BLOCK_SHIFT`). -const ARENA_BLOCK_SHIFT: u32 = 6; -const ARENA_BLOCK: usize = 1 << ARENA_BLOCK_SHIFT; +/// How [`RowArena::over_chunks`] cuts chunks into blocks: each entry is a +/// block of pieces `(chunk, offset, rows)`; a block of one piece is a +/// zero-copy slice, one of several is concatenated. Every whole block a +/// chunk holds comes first, in order; then the tails (chunks shorter than a +/// block, and what is left of longer ones) are joined into blocks, the last +/// one shorter. A partition's chunks are its drivers' lists one after +/// another, each ending in a tail: in row order, the first tail would shift +/// every later block across two chunks and copy it. +fn block_plan(chunks: &[Batch], block: usize) -> Vec> { + let mut blocks = Vec::new(); + let mut tails = Vec::new(); + for (c, chunk) in chunks.iter().enumerate() { + let n = chunk.num_rows(); + let whole = n - n % block; + blocks.extend((0..whole).step_by(block).map(|at| vec![(c, at, block)])); + if whole < n { + tails.push((c, whole, n - whole)); + } + } + let mut pending: Vec<(usize, usize, usize)> = Vec::new(); + let mut held = 0usize; + for (c, mut at, mut n) in tails { + while n > 0 { + let take = (block - held).min(n); + pending.push((c, at, take)); + held += take; + at += take; + n -= take; + if held == block { + blocks.push(std::mem::take(&mut pending)); + held = 0; + } + } + } + if !pending.is_empty() { + blocks.push(pending); + } + blocks +} + +/// The bytes a slice of `len` rows at `offset` of `batch` takes once copied +/// (its rows' share of the buffers, weights included). +fn slice_bytes(batch: &Batch, offset: usize, len: usize) -> usize { + let data = batch.data(); + let columns: usize = data + .columns() + .iter() + .map(|c| { + let part = c.slice(offset, len).to_data(); + part.get_slice_memory_size() + .unwrap_or_else(|_| part.get_array_memory_size()) + }) + .fold(0, usize::saturating_add); + let weights = match batch.weights() { + Weights::Const(_) => 0, + Weights::Column(_) => len * size_of::(), + }; + columns.saturating_add(weights) +} impl RowArena { - /// The arena over `chunks` (same schema, in order), without copying their - /// rows. Empty chunks are skipped; an arena of no rows keeps one empty - /// chunk of `schema`, so its key format is still known. + /// The arena over `chunks` (same schema). Chunks are cut into blocks of + /// [`ARENA_ROWS`] rows: a block inside one chunk is a zero-copy slice, + /// the chunks' tails are copied into blocks ([`block_plan`] gives the + /// row order; [`reblock_bytes`](Self::reblock_bytes) bounds the copy). + /// One chunk keeps its row order. + /// An arena of no rows keeps one empty chunk of `schema`, so its key + /// format is still known. pub(crate) fn over_chunks(schema: &SchemaRef, chunks: &[Batch]) -> Result { - let mut parts = Vec::with_capacity(chunks.len().max(1)); - let mut starts = Vec::with_capacity(chunks.len() + 1); + Self::with_shift(schema, chunks, ARENA_SHIFT) + } + + fn with_shift(schema: &SchemaRef, chunks: &[Batch], shift: u32) -> Result { + let block = 1usize << shift; + let plan = block_plan(chunks, block); + let mut parts = Vec::with_capacity(plan.len().max(1)); let mut len = 0usize; - for chunk in chunks.iter().filter(|c| c.num_rows() > 0) { - starts.push(Self::row_id(len)?); - len = len.saturating_add(chunk.num_rows()); - parts.push(ArenaPart::of(chunk.clone())?); + for pieces in plan { + let batch = match pieces.as_slice() { + [(c, at, n)] => match chunks.get(*c) { + Some(chunk) if *at == 0 && *n == chunk.num_rows() => chunk.clone(), + Some(chunk) => chunk.slice(*at, *n).map_err(|e| internal(e.to_string()))?, + None => return Err(internal("join arena block out of range".to_owned())), + }, + _ => { + let slices = pieces + .iter() + .map(|&(c, at, n)| { + chunks + .get(c) + .ok_or_else(|| { + internal("join arena block out of range".to_owned()) + })? + .slice(at, n) + .map_err(|e| internal(e.to_string())) + }) + .collect::>>()?; + Batch::concat(schema, &slices).map_err(|e| internal(e.to_string()))? + } + }; + len = len.saturating_add(batch.num_rows()); + parts.push(ArenaPart::of(batch)?); } if parts.is_empty() { let empty = match chunks.first() { Some(c) => c.clone(), None => Batch::new(RecordBatch::new_empty(Arc::clone(schema))), }; - starts.push(0); parts.push(ArenaPart::of(empty)?); } - starts.push(Self::row_id(len)?); - let mut block_chunk = Vec::new(); - if parts.len() > 1 { - block_chunk - .try_reserve_exact(len.div_ceil(ARENA_BLOCK)) - .map_err(|e| internal(e.to_string()))?; - let mut chunk = 0usize; - for first in (0..len).step_by(ARENA_BLOCK) { - while starts.get(chunk + 1).is_some_and(|&s| s as usize <= first) { - chunk += 1; + Self::row_id(len)?; + Ok(Self { parts, shift, len }) + } + + /// What [`over_chunks`](Self::over_chunks) copies to build the blocks + /// that span chunks: `(copy, kept)`, the bytes of the copy (reserved + /// before it, buffer rounding included) and of the part of it that stays + /// beside its source once the arena is built, because the source chunk + /// lives on: the first `live` chunks (held by their owner), and chunks + /// that also give whole blocks (sliced in place). The rest of the copy + /// replaces chunks that are dropped. + pub(crate) fn reblock_bytes(chunks: &[Batch], live: usize) -> (usize, usize) { + // Each new buffer rounds up to 64 bytes; about three per column. + let rounding = chunks + .first() + .map_or(0, |c| c.data().num_columns().saturating_add(1) * 3 * 64); + let mut copy = 0usize; + let mut kept = 0usize; + for pieces in block_plan(chunks, ARENA_ROWS) + .iter() + .filter(|p| p.len() > 1) + { + copy = copy.saturating_add(rounding); + for &(c, at, n) in pieces { + let Some(b) = chunks.get(c) else { continue }; + let bytes = slice_bytes(b, at, n); + copy = copy.saturating_add(bytes); + if c < live || b.num_rows() >= ARENA_ROWS { + kept = kept.saturating_add(bytes); } - block_chunk.push(Self::row_id(chunk)?); } } - Ok(Self { - parts, - starts, - block_chunk, - len, - }) + (copy, kept) } fn row_id(n: usize) -> Result { u32::try_from(n).map_err(|_| internal("a join partition has over 2^32 rows".to_owned())) } - /// The bytes of the block-to-chunk table (part of the index's cost). - pub(crate) fn chunk_table_bytes(&self) -> usize { - self.block_chunk.capacity() * size_of::() - } - - /// The bytes [`chunk_table_bytes`](Self::chunk_table_bytes) is at most - /// for `rows` rows. - pub(crate) fn chunk_table_bound(rows: usize) -> usize { - rows.div_ceil(ARENA_BLOCK) * size_of::() - } - - /// The chunk holding row `row` and the row's index in it (`None` past + /// The block holding row `row` and the row's index in it (`None` past /// the end). #[inline] fn at(&self, row: u32) -> Option<(&ArenaPart, u32)> { - if (row as usize) >= self.len { - return None; - } - if let [one] = self.parts.as_slice() { - return Some((one, row)); - } - let mut i = *self.block_chunk.get((row >> ARENA_BLOCK_SHIFT) as usize)? as usize; - // The block's chunk starts at or before `row`; a later one may too - // (chunks shorter than a block). - while self.starts.get(i + 1).is_some_and(|&s| s <= row) { - i += 1; - } - let part = self.parts.get(i)?; - Some((part, row.checked_sub(*self.starts.get(i)?)?)) + let part = self.parts.get((row >> self.shift) as usize)?; + let local = row & ((1u32 << self.shift) - 1); + (local < part.rows).then_some((part, local)) } pub(crate) fn len(&self) -> usize { self.len } - /// The arena's bytes by part (`get_array_memory_size`, the measure its reservation uses): the `hash` column, the key - /// columns (`key*`), and everything else (payload, NULL-key flags). + /// The arena's bytes by part (each block's share of its buffers): the `hash` column, the key columns (`key*`), and + /// everything else (payload, NULL-key flags). pub(crate) fn memory_split(&self) -> ArenaBytes { let mut out = ArenaBytes::default(); for part in &self.parts { let data = part.batch.data(); for (field, column) in data.schema().fields().iter().zip(data.columns()) { - let bytes = column.get_array_memory_size(); + // Blocks sliced from one chunk share its buffers: count each + // block's rows only. + let column = column.to_data(); + let bytes = column + .get_slice_memory_size() + .unwrap_or_else(|_| column.get_array_memory_size()); let name = field.name(); if name == "hash" { out.hash += bytes; @@ -397,7 +484,13 @@ impl RowArena { self.parts .iter() .map(|p| match &p.keys { - Cols::Arrow(k) => k.values().len(), + // A block sliced from a chunk shares its values: count the + // block's own range. + Cols::Arrow(k) => { + let o = k.value_offsets(); + let span = o.last().copied().unwrap_or(0) - o.first().copied().unwrap_or(0); + usize::try_from(span).unwrap_or(0) + } Cols::Rows(_) => 0, }) .sum() @@ -577,12 +670,6 @@ impl JoinTable { /// per row. Row layout: the [`JoinIndex`] sized for one key per row and /// a link per row (it copies no keys and holds the heads itself). pub(crate) fn index_bytes(rows: usize, key_bytes: usize, row_layout: bool) -> usize { - // Plus the arena's chunk table (`RowArena::chunk_table_bytes`). - Self::index_only_bytes(rows, key_bytes, row_layout) - .saturating_add(RowArena::chunk_table_bound(rows)) - } - - fn index_only_bytes(rows: usize, key_bytes: usize, row_layout: bool) -> usize { if row_layout { IndexChoice::Adaptive .bytes(rows) @@ -642,10 +729,7 @@ impl JoinTable { } Index::Rows { index, .. } => index.memory_size(), }; - index - + self.next.capacity() * size_of::() - + self.marks.capacity() * size_of::() - + self.arena.chunk_table_bytes() + index + self.next.capacity() * size_of::() + self.marks.capacity() * size_of::() } /// Bytes of the match bitmap of a table of `rows` rows @@ -985,11 +1069,12 @@ mod tests { } } - /// An arena over several chunks (no copy, X3' step 3) reads every row as the one batch of the same rows does: - /// through the block-to-chunk table, with chunks longer and shorter than a block, past 256 chunks, with empty - /// chunks skipped, and the index built over it chains the same rows. + /// An arena over several chunks (X3' step 3 option C) holds every row of the one batch of the same rows, in blocks + /// of `1 << shift` rows: whole blocks of a chunk first, in order (zero-copy), then the tails joined; with chunks + /// longer and shorter than a block, empty ones and many tiny ones. One chunk keeps its order; the index built over + /// the blocks finds the same keys. #[test] - fn a_chunked_arena_reads_and_indexes_as_one_batch() { + fn a_blocked_arena_holds_the_rows_of_one_batch() { let schema = arena_schema(ArenaFormat::Arrow { payload: false }).unwrap_or_else(|e| panic!("{e}")); let n = 600usize; @@ -1000,7 +1085,7 @@ mod tests { Arc::clone(&schema), vec![ Arc::new(UInt64Array::from_iter_values( - rows.clone().map(|i| (i % 97) as u64), + rows.clone().map(|i| i as u64), )), Arc::new(LargeBinaryArray::from_iter_values(rows.map(key))), ], @@ -1008,9 +1093,30 @@ mod tests { .unwrap_or_else(|e| panic!("{e}")), ) }; + let rows_of = |arena: &RowArena| { + let mut out: Vec<(u64, Vec)> = (0..u32::try_from(arena.len()).unwrap_or(0)) + .map(|r| { + ( + arena.hash(r), + arena.key_bytes(r).unwrap_or_default().to_vec(), + ) + }) + .collect(); + out.sort(); + out + }; let one = RowArena::over_chunks(&schema, &[batch(0..n)]).unwrap_or_else(|e| panic!("{e}")); - // Uneven chunks with empty ones between, a block-aligned edge (64, 128); then 300 chunks of 2 rows - // (many chunks per block, more than 256 chunks). + let all = rows_of(&one); + // One chunk, longer than a block: sliced in order, nothing copied. + let sliced = + RowArena::with_shift(&schema, &[batch(0..n)], 4).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(sliced.parts.len(), n.div_ceil(16)); + for row in 0..u32::try_from(n).unwrap_or(0) { + assert_eq!(sliced.hash(row), u64::from(row), "row {row}"); + } + assert_eq!(RowArena::reblock_bytes(&[batch(0..n)], 0), (0, 0)); + // Its key bytes are counted once, not once per block of the chunk. + assert_eq!(sliced.arrow_key_bytes(), one.arrow_key_bytes()); let uneven: Vec = [ 0..1, 1..1, @@ -1026,11 +1132,17 @@ mod tests { .collect(); let many: Vec = (0..n / 2).map(|c| batch(2 * c..2 * c + 2)).collect(); for chunks in [uneven, many] { - let arena = RowArena::over_chunks(&schema, &chunks).unwrap_or_else(|e| panic!("{e}")); + let arena = RowArena::with_shift(&schema, &chunks, 4).unwrap_or_else(|e| panic!("{e}")); assert_eq!(arena.len(), n); + assert_eq!(rows_of(&arena), all); + // Every block but the last is full. + let (last, full) = arena + .parts + .split_last() + .unwrap_or_else(|| panic!("no blocks")); + assert!(full.iter().all(|p| p.rows == 16)); + assert!((1..=16).contains(&last.rows)); for row in 0..u32::try_from(n).unwrap_or(0) { - assert_eq!(arena.hash(row), one.hash(row), "row {row}"); - assert_eq!(arena.key_bytes(row), one.key_bytes(row), "row {row}"); assert_eq!(arena.weight(row), 1, "row {row}"); } // Past the end: nothing, not a panic. @@ -1045,9 +1157,36 @@ mod tests { 0, ) .unwrap_or_else(|e| panic!("{e}")); - assert_eq!(chained.next(), single.next()); assert_eq!(chained.index_size().0, single.index_size().0); } + // Block-aligned chunks with tails between them (a partition's drivers' lists): the blocks are not copied, + // only the tails are. + let block = ARENA_ROWS; + let lists = [ + batch(0..block), + batch(block..block + 5), + batch(block + 5..3 * block + 5), + batch(3 * block + 5..3 * block + 9), + ]; + let arena = RowArena::over_chunks(&schema, &lists).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(arena.parts.len(), 4); + assert_eq!(arena.len(), 3 * block + 9); + // The two tails: one of a chunk that also gives blocks (kept beside it), one of a chunk the copy replaces. + let (copy, kept) = RowArena::reblock_bytes(&lists, 0); + assert!(kept > 0 && kept < copy && copy < 2048, "{copy} {kept}"); + assert_eq!( + RowArena::reblock_bytes(&lists, 4), + (copy, copy - 3 * 3 * 64) + ); + assert_eq!(arena.hash(0), 0); + assert_eq!( + arena.hash(u32::try_from(block).unwrap_or(0)), + (block + 5) as u64 + ); + assert_eq!( + arena.hash(u32::try_from(3 * block).unwrap_or(0)), + block as u64 + ); } /// An arena of no rows (a grace partition whose build side is empty, its @@ -1064,7 +1203,6 @@ mod tests { assert!(arena.has_arrow_keys()); assert_eq!(arena.hash(0), 0); assert_eq!(arena.key_bytes(0), None); - assert_eq!(arena.chunk_table_bytes(), 0); let table = JoinTable::build(arena, None, 0).unwrap_or_else(|e| panic!("{e}")); assert!(table.next().is_empty()); } diff --git a/docs/reference/explain-analyze.md b/docs/reference/explain-analyze.md index bbaeb9f43..eb3630dfa 100644 --- a/docs/reference/explain-analyze.md +++ b/docs/reference/explain-analyze.md @@ -77,9 +77,9 @@ so it lists no operators of its own. | `runtime_filter_*`, `runtime_filters` | Rows a scan's runtime filters tested and dropped, units and splits they pruned, and per filter the form its producer chose (`range`, `small set of N`, `hash set of N`, `bloom N B`). | | `hash_entries`, `hash_buckets` (load), `hash_index` | The operator's hash tables: distinct keys, buckets and the index kind (a join's probe side reports the tables its grace phase built after spilling as `grace_hash_entries` and `grace_hash_buckets`) (`swiss`: an open-addressing table of keys; `swiss+chain`: the same with duplicate build rows chained; `row-flat+chain` and `row-salted+chain`: the row-layout join index, flat or salted by size). | | `probe_lookups`, `probe_key_compares` (avg), `max_probe_length` | Lookups, the keys compared to answer them (the probe length) and the longest probe. Collected only by `EXPLAIN ANALYZE` (`ExecConfig::detailed_metrics`), and not for the row-layout join index, because they cost a counter per lookup; other runs show `(probe lengths not collected)`. | -| `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent setting up the rows' chunks (no copy) and indexing them (summed over the tasks, which run in parallel), and the longest single task. | -| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the buffer bytes of the finalized row arenas (Arrow's `get_array_memory_size`, capacities included), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). | -| `max_coalesce_copy_bytes` | A hash join build's transient copy: the largest run of chunks one build-phase coalesce merged (each merge reserves the run again while it copies). Finalize makes no copy: the arena reads a partition's chunks in place. | +| `finalize_ns`, `finalize_concat_ns`, `finalize_index_ns`, `finalize_max_partition_ns` | A hash join build's `finalize`, the phase one driver runs after every driver combined: its wall time, the time its partition tasks spent setting up the rows' blocks (whole blocks sliced in place, the chunks' tails copied) and indexing them (summed over the tasks, which run in parallel), and the longest single task. | +| `steady_reserved_bytes`, `arena_bytes`, `arena_hash_bytes`, `arena_key_bytes`, `index_bytes` | A hash join build's memory once finalized, summed over the partitions finalized in memory (arenas the grace phase rebuilds from spilled partitions are not counted): what their rows keep reserved through the probe, the bytes of the finalized row arenas (each block's rows' share of its buffers, Arrow's `get_slice_memory_size`), of which the `hash` column and the key columns, and the bytes of the indexes (with match bitmaps). | +| `max_coalesce_copy_bytes` | A hash join build's transient copy: the largest set of small chunks one build-phase step copied into a block of 16,384 rows (each copy reserves those chunks again while it runs), also over a grace partition's frames as they are read back. Finalize reads the blocks in place and copies only the chunks' tails. | | `broadcast_shared`, `broadcast_fallback` | A broadcast hash join on a multi-task run: 1 when the task probed the worker's shared build table, or 1 when the shared build did not fit `krishiv.exec.join.sharedBroadcastBytes` and the task built its own table instead. | | Pipelines | Per pipeline: wall time from its first driver's start to its last driver's end, drivers, threads that ran them, and its operators from source to sink. | From 3b409cf1fd9614d846be486645c5d0f1456cc10e Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:49:23 +0000 Subject: [PATCH 10/25] exec: option C funds each partition's tail copy as main did (a spilling build reserves every index and the largest copy, not every copy at once); the blocked-arena test gets a tail beside its blocks Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 84 ++++++++++++++++++--------- crates/krishiv-exec/src/join/table.rs | 6 +- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index a33f4217e..01da1302f 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -405,10 +405,10 @@ impl BuildSink { } /// What indexing a partition's in-memory rows reserves: its [`JoinTable`] -/// index (and match bitmap when `marks`), and the copy of the chunk tails -/// the arena joins into blocks ([`RowArena::reblock_bytes`]); `None` if no -/// rows are in memory. -fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option { +/// index (and match bitmap when `marks`), and, while it indexes, the copy +/// of the chunk tails the arena joins into blocks +/// ([`RowArena::reblock_bytes`]); `None` if no rows are in memory. +fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize)> { if part.chunks.is_empty() { return None; } @@ -418,7 +418,7 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option { if marks { index = index.saturating_add(JoinTable::marks_bytes(rows)); } - Some(index.saturating_add(RowArena::reblock_bytes(&part.chunks, 0).0)) + Some((index, RowArena::reblock_bytes(&part.chunks, 0).0)) } /// A duration in whole nanoseconds, saturating. @@ -432,8 +432,9 @@ struct Job { /// Bytes of its rows in memory. bytes: usize, chunks: Vec, - /// What indexing it reserves (see [`build_need`]). - need: usize, + /// What indexing it reserves: the index and the tails' copy (see + /// [`build_need`]). + need: (usize, usize), /// Of that, the arena's copied tails that stay beside chunks it also /// slices: reserved through the probe ([`RowArena::reblock_bytes`]). kept: usize, @@ -715,9 +716,13 @@ impl Sink for BuildSink { // Write what is left of spilled partitions; reserve the index of // the rest, spilling the largest while it does not fit. The arenas // read the blocks in place; finalize copies only the chunk tails. + // A spilling build reserves every index and the largest copy, so + // one partition can always run and the copies never need the + // build's size again. let mut index = ctx.consumer().reservation(); + let mut held_copy = 0; if s.spill.is_some() { - loop { + held_copy = loop { // Once memory ran short, every partition goes to disk: the // probe phase then holds one partition's table per driver, // never tables that fill the budget and cannot spill. @@ -736,20 +741,20 @@ impl Sink for BuildSink { self.spill_collected(g, 0, ctx, true).await?; continue; } - let indexes = { + let (indexes, copy) = { let c = lock(&g.collect); let rows = g.hashing.rows(); let marks = s.join_type.builds_preserved(); - c.parts - .iter() - .filter_map(|p| build_need(p, rows, marks)) - .fold(0usize, usize::saturating_add) + let needs = c.parts.iter().filter_map(|p| build_need(p, rows, marks)); + needs.fold((0usize, 0usize), |(i, m), (index, copy)| { + (i.saturating_add(index), usize::max(m, copy)) + }) }; - if index.try_grow(indexes).is_ok() { - break; + if index.try_grow(indexes.saturating_add(copy)).is_ok() { + break copy; } self.spill_collected(g, 1, ctx, true).await?; - } + }; } let mut jobs: Vec = { let mut c = lock(&g.collect); @@ -781,20 +786,32 @@ impl Sink for BuildSink { // Nothing in memory is spillable any more. ctx.consumer().signal().set_spillable_bytes(0); // One blocking task per partition (the executor's blocking pool - // bounds how many run at once). A spilling build reserved every - // index already; without a spill scope a partition's index is - // asked for through arbitration when nothing else runs. + // bounds how many run at once), each started only once its tails' + // copy is reserved. A spilling build's reservation already funds + // the largest copy (`spare`), so one partition can always run; + // without a spill scope a partition's index and copy are asked + // for through arbitration when nothing else runs. let spill = s.spill.is_some(); + let mut spare = held_copy; let mut jobs = std::collections::VecDeque::from(jobs); let mut running = futures::stream::FuturesUnordered::new(); let mut tables = Vec::with_capacity(jobs.len()); loop { if let Some(&Job { - need: index_bytes, .. + need: (index_bytes, copy), + .. }) = jobs.front() { - let grow = if spill { 0 } else { index_bytes }; - let funded = if grow == 0 || index.try_grow(grow).is_ok() { + // A spilling build reserved every index already. + let grow = if spill { + copy + } else { + index_bytes.saturating_add(copy) + }; + let funded = if spill && copy <= spare { + spare -= copy; + true + } else if grow == 0 || index.try_grow(grow).is_ok() { true } else if running.is_empty() { index.grow(grow).await.map_err(KrishivError::from)?; @@ -805,7 +822,7 @@ impl Sink for BuildSink { if let Some(Job { part: p, chunks, - need: index_bytes, + need: (index_bytes, copy), keys, kept, .. @@ -842,7 +859,7 @@ impl Sink for BuildSink { table }) .await?; - Ok::<_, KrishivError>((p, table, index_bytes.saturating_sub(kept))) + Ok::<_, KrishivError>((p, table, index_bytes, copy, kept)) }); continue; } @@ -850,14 +867,28 @@ impl Sink for BuildSink { let Some(done) = futures::StreamExt::next(&mut running).await else { break; }; - let (p, table, index_bytes) = done?; + let (p, table, index_bytes, copy, kept) = done?; // The index was reserved for one key per row: keep what it - // holds (and the arena's copied tails that stay). + // holds. index.shrink( index_bytes .saturating_sub(table.index_memory()) .min(index.size()), ); + // The copied tails beside chunks the arena also slices stay + // reserved through the probe; the rest of the copy replaced + // chunks that are gone now. + let transient = copy.saturating_sub(kept); + if spill { + // Keep the largest copy funded; return the rest. + spare = spare.saturating_add(transient); + if spare > held_copy { + index.shrink((spare - held_copy).min(index.size())); + spare = held_copy; + } + } else { + index.shrink(transient.min(index.size())); + } let split = table.arena().memory_split(); let n = &self.0.counters; n.add( @@ -873,6 +904,7 @@ impl Sink for BuildSink { self.0.counters.add(Counter::HashBuckets, buckets as u64); tables.push((p, table)); } + index.shrink(spare.min(index.size())); let mut c = lock(&g.collect); // Spilled partitions that hold rows: the probe defers only these // (an empty partition matches nothing, spilled or not). diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index b96e06c67..1d1b76243 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -1165,12 +1165,12 @@ mod tests { let lists = [ batch(0..block), batch(block..block + 5), - batch(block + 5..3 * block + 5), - batch(3 * block + 5..3 * block + 9), + batch(block + 5..3 * block + 8), + batch(3 * block + 8..3 * block + 12), ]; let arena = RowArena::over_chunks(&schema, &lists).unwrap_or_else(|e| panic!("{e}")); assert_eq!(arena.parts.len(), 4); - assert_eq!(arena.len(), 3 * block + 9); + assert_eq!(arena.len(), 3 * block + 12); // The two tails: one of a chunk that also gives blocks (kept beside it), one of a chunk the copy replaces. let (copy, kept) = RowArena::reblock_bytes(&lists, 0); assert!(kept > 0 && kept < copy && copy < 2048, "{copy} {kept}"); From 2f30bf2a345a23683edebab101b71a9b39b236e8 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:49:56 +0000 Subject: [PATCH 11/25] exec: a spilling join build reserves the kept tails with the indexes, so the copy it holds for one partition at a time funds transient copies only (kept tails no longer use it up and leave a later partition unfunded); a test over chunks with half-block tails under tight limits Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 45 +++++++++++---------- crates/krishiv-exec/tests/hash_join/main.rs | 27 +++++++++++++ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 01da1302f..afc60ad45 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -404,10 +404,12 @@ impl BuildSink { } } -/// What indexing a partition's in-memory rows reserves: its [`JoinTable`] -/// index (and match bitmap when `marks`), and, while it indexes, the copy -/// of the chunk tails the arena joins into blocks -/// ([`RowArena::reblock_bytes`]); `None` if no rows are in memory. +/// What indexing a partition's in-memory rows reserves, as `(lasting, +/// transient)`: lasting is its [`JoinTable`] index (and match bitmap when +/// `marks`) plus the copied chunk tails that stay beside chunks the arena +/// also slices (held through the probe); transient is the rest of the +/// tails' copy, which replaces chunks dropped once the arena is built +/// ([`RowArena::reblock_bytes`]). `None` if no rows are in memory. fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize)> { if part.chunks.is_empty() { return None; @@ -418,7 +420,8 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz if marks { index = index.saturating_add(JoinTable::marks_bytes(rows)); } - Some((index, RowArena::reblock_bytes(&part.chunks, 0).0)) + let (copy, kept) = RowArena::reblock_bytes(&part.chunks, 0); + Some((index.saturating_add(kept), copy.saturating_sub(kept))) } /// A duration in whole nanoseconds, saturating. @@ -432,11 +435,12 @@ struct Job { /// Bytes of its rows in memory. bytes: usize, chunks: Vec, - /// What indexing it reserves: the index and the tails' copy (see - /// [`build_need`]). + /// What indexing it reserves: lasting (index and kept tails) and the + /// transient copy (see [`build_need`]). need: (usize, usize), - /// Of that, the arena's copied tails that stay beside chunks it also - /// slices: reserved through the probe ([`RowArena::reblock_bytes`]). + /// Of the lasting part, the arena's copied tails that stay beside chunks + /// it also slices: reserved through the probe + /// ([`RowArena::reblock_bytes`]). kept: usize, /// Its estimated distinct keys (row layout, when sketched). keys: Option, @@ -716,9 +720,9 @@ impl Sink for BuildSink { // Write what is left of spilled partitions; reserve the index of // the rest, spilling the largest while it does not fit. The arenas // read the blocks in place; finalize copies only the chunk tails. - // A spilling build reserves every index and the largest copy, so - // one partition can always run and the copies never need the - // build's size again. + // A spilling build reserves every index and kept tail and the + // largest transient copy, so one partition can always run and the + // copies never need the build's size again. let mut index = ctx.consumer().reservation(); let mut held_copy = 0; if s.spill.is_some() { @@ -788,7 +792,7 @@ impl Sink for BuildSink { // One blocking task per partition (the executor's blocking pool // bounds how many run at once), each started only once its tails' // copy is reserved. A spilling build's reservation already funds - // the largest copy (`spare`), so one partition can always run; + // the largest transient copy (`spare`), so one partition can always run; // without a spill scope a partition's index and copy are asked // for through arbitration when nothing else runs. let spill = s.spill.is_some(); @@ -867,20 +871,19 @@ impl Sink for BuildSink { let Some(done) = futures::StreamExt::next(&mut running).await else { break; }; - let (p, table, index_bytes, copy, kept) = done?; + let (p, table, lasting, transient, kept) = done?; // The index was reserved for one key per row: keep what it - // holds. + // holds. The copied tails beside chunks the arena also slices + // (`kept`, part of `lasting`) stay reserved through the probe. index.shrink( - index_bytes + lasting + .saturating_sub(kept) .saturating_sub(table.index_memory()) .min(index.size()), ); - // The copied tails beside chunks the arena also slices stay - // reserved through the probe; the rest of the copy replaced - // chunks that are gone now. - let transient = copy.saturating_sub(kept); + // The transient copy replaced chunks that are gone now. if spill { - // Keep the largest copy funded; return the rest. + // Keep the largest transient copy funded; return the rest. spare = spare.saturating_add(transient); if spare > held_copy { index.shrink((spare - held_copy).min(index.size())); diff --git a/crates/krishiv-exec/tests/hash_join/main.rs b/crates/krishiv-exec/tests/hash_join/main.rs index b4a05be41..63635321c 100644 --- a/crates/krishiv-exec/tests/hash_join/main.rs +++ b/crates/krishiv-exec/tests/hash_join/main.rs @@ -978,3 +978,30 @@ async fn a_build_that_fits_with_one_partition_copy_completes() { ); assert_eq!(spill_files(&scope), 0); } + +/// Partitions whose chunks hold whole arena blocks (2^14 rows) keep the +/// copied tails beside them through the probe. A spilling build reserves +/// those up front with the indexes, so the copy it holds for one partition +/// at a time is never used up by kept tails: under every limit from the +/// in-memory peak down, the build either fits or spills, and never fails. +#[tokio::test(start_paused = true)] +async fn a_spilling_build_with_kept_tails_fits_or_spills_under_any_limit() { + // Two input batches over distinct keys: each of the 8 level-0 + // partitions gets two chunks of about 24,400 rows, a whole block and a + // tail of about 8,000 rows each. The two tails make one copied block, + // kept beside the chunks: a third of the build is kept tails. + let build = skewed(2 * 8 * 24_384, 1 << 30); + let chunk = build.len() / 2; + let (expect, q) = run_self_join(&build, chunk, 512 * MIB, None).await; + let expect = expect.unwrap(); + let single = peak(&q); + for step in 0..12 { + let limit = single - single * step / 32; + let (_manager, scope) = spill(); + let (got, _) = run_self_join(&build, chunk, limit, Some(scope)).await; + match got { + Ok(got) => assert!(got == expect, "the join result differs at {limit}"), + Err(e) => panic!("the build failed at limit {limit} ({single} in memory): {e}"), + } + } +} From 8de49516cd1c2fa644aabc0155f286bf93f16c82 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:15:15 +0000 Subject: [PATCH 12/25] exec: a join table whose every key occurs once ends each chain without reading it (J4, study 15; DuckDB's chains_longer_than_one): the probe and mark_chain skip next() on unique tables, grace tables too Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/probe.rs | 2 +- crates/krishiv-exec/src/join/table.rs | 191 ++++++++++++++++---------- 2 files changed, 121 insertions(+), 72 deletions(-) diff --git a/crates/krishiv-exec/src/join/probe.rs b/crates/krishiv-exec/src/join/probe.rs index 68fd7e650..fa7e3432e 100644 --- a/crates/krishiv-exec/src/join/probe.rs +++ b/crates/krishiv-exec/src/join/probe.rs @@ -230,7 +230,7 @@ impl Cursor { probe.push(u32::try_from(i).unwrap_or(u32::MAX)); build.push((p, self.chain)); } - self.chain = table.map_or(NONE, |t| t.next()[self.chain as usize]); + self.chain = table.map_or(NONE, |t| t.next_of(self.chain)); if self.chain == NONE { self.pos += 1; } diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index 1d1b76243..d2e5541f2 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -652,7 +652,7 @@ enum Index { /// [`KeyTable`] (hashbrown `HashTable`, E02 decision B), or a /// [`JoinIndex`] in the row layout, maps each distinct key to its newest /// row, and older rows with the same key are chained through -/// [`next`](Self::next). +/// [`next_of`](Self::next_of). #[derive(Debug)] pub(crate) struct JoinTable { arena: RowArena, @@ -661,6 +661,10 @@ pub(crate) struct JoinTable { /// One bit per row: a pair with the row matched (empty unless /// [`with_marks`](Self::with_marks)). marks: Vec, + /// No row has an older row with its key (every chain is one row long), + /// so [`next_of`](Self::next_of) never reads the chain (DuckDB's + /// `chains_longer_than_one`). + unique: bool, } impl JoinTable { @@ -784,7 +788,7 @@ impl JoinTable { let mut r = head; while r != NONE { self.mark(r); - r = self.next.get(r as usize).copied().unwrap_or(NONE); + r = self.next_of(r); } } @@ -855,11 +859,13 @@ impl JoinTable { }, (false, None) => return Err(internal("join key rows without a layout".to_owned())), }; + let unique = next.iter().all(|&n| n == NONE); Ok(Self { arena, index, next, marks: Vec::new(), + unique, }) } @@ -943,11 +949,30 @@ impl JoinTable { } } - /// `next()[r]`: the next build row with the same key as `r`, or [`NONE`]. + /// `next()[r]`: the next build row with the same key as `r`, or [`NONE`] + /// (tests; readers go through [`next_of`](Self::next_of)). + #[cfg(test)] pub(crate) fn next(&self) -> &[u32] { &self.next } + /// The row after `row` in its key's chain, or [`NONE`]; on a table whose + /// every key occurs once, `NONE` without reading the chain. + #[inline] + pub(crate) fn next_of(&self, row: u32) -> u32 { + if self.unique { + NONE + } else { + self.next.get(row as usize).copied().unwrap_or(NONE) + } + } + + /// Whether every key occurs once (no chain is longer than one row). + #[cfg(test)] + pub(crate) fn is_unique(&self) -> bool { + self.unique + } + /// The rows. pub(crate) fn arena(&self) -> &RowArena { &self.arena @@ -973,12 +998,30 @@ mod tests { RowArena::over_chunks(&schema, &[Batch::new(data)]).unwrap_or_else(|e| panic!("{e}")) } + #[test] + fn a_table_of_distinct_keys_is_unique_and_ends_every_chain() { + // Equal hashes, distinct keys: one-row chains only. + let t = JoinTable::build(arena(vec![7, 7, 9], vec![b"a", b"b", b"c"]), None, 0); + let t = t.unwrap_or_else(|e| panic!("{e}")); + assert!(t.is_unique()); + assert!((0..3).all(|r| t.next_of(r) == NONE)); + // Marking a head marks only that row. + let t = t.with_marks().unwrap_or_else(|e| panic!("{e}")); + t.mark_chain(1); + assert_eq!( + (0..3).map(|r| t.is_marked(r)).collect::>(), + [false, true, false] + ); + } + #[test] fn duplicates_chain_and_collisions_compare_keys() { // Rows 0 and 2 share a key; row 1 collides on the hash only. let t = JoinTable::build(arena(vec![7, 7, 7], vec![b"a", b"b", b"a"]), None, 0); let t = t.unwrap_or_else(|e| panic!("{e}")); assert_eq!(t.next(), &[NONE, NONE, 0]); + assert!(!t.is_unique()); + assert_eq!((t.next_of(2), t.next_of(0)), (0, NONE)); assert_eq!(t.arena().len(), 3); assert!(JoinTable::index_bytes(3, 3, false) > 0); assert!(JoinTable::index_bytes(3, 0, true) > 0); @@ -996,76 +1039,82 @@ mod tests { use arrow_array::{ArrayRef, Int64Array, StringArray}; let n = 3000usize; - let ints: ArrayRef = Arc::new(Int64Array::from_iter( - (0..n).map(|i| (i % 7 != 3).then_some((i % 500) as i64)), - )); - let strs: ArrayRef = Arc::new(StringArray::from_iter( - (0..n).map(|i| Some(format!("{}-a-long-enough-string", i % 3))), - )); - let cols = [ints, strs]; - let types = [DataType::Int64, DataType::Utf8]; - let codec = - KeyCodec::with_string_rows(&types, vec![false, true]).unwrap_or_else(|e| panic!("{e}")); - assert!( - KeyCodec::new(&types, vec![false, true]) - .unwrap_or_else(|e| panic!("{e}")) - .layout() - .is_none() - ); - let layout = codec.layout().cloned(); - let tables = |rows: bool, choice: IndexChoice| { - let hashing = Hashing::new(Some(3), rows); - let format = match layout.as_deref().filter(|_| rows) { - Some(keys) => ArenaFormat::Rows { - keys, - payload: None, - arrow_payload: false, - }, - None => ArenaFormat::Arrow { payload: false }, - }; - let schema = arena_schema(format).unwrap_or_else(|e| panic!("{e}")); - let keys = codec - .encode(&cols, &hashing) + // Repeated keys chain; distinct keys (every key once) make a unique + // table in every index kind. + for (m, unique) in [(500usize, false), (n, true)] { + let ints: ArrayRef = Arc::new(Int64Array::from_iter( + (0..n).map(|i| (i % 7 != 3).then_some((i % m) as i64)), + )); + let strs: ArrayRef = Arc::new(StringArray::from_iter( + (0..n).map(|i| Some(format!("{}-a-long-enough-string", i % 3))), + )); + let cols = [ints, strs]; + let types = [DataType::Int64, DataType::Utf8]; + let codec = KeyCodec::with_string_rows(&types, vec![false, true]) .unwrap_or_else(|e| panic!("{e}")); - let valid: Vec = (0..n).filter(|&i| keys.is_valid(i)).collect(); - let chunk = |idx: &[usize]| { - let mut c: Vec = vec![Arc::new( - UInt64Array::from_iter_values(idx.iter().map(|&i| keys.hashes()[i])), - )]; - let l = layout.as_deref().filter(|_| rows); - c.extend(keys.arena_columns(l, idx).unwrap_or_else(|e| panic!("{e}"))); - Batch::new( - RecordBatch::try_new(Arc::clone(&schema), c).unwrap_or_else(|e| panic!("{e}")), - ) + assert!( + KeyCodec::new(&types, vec![false, true]) + .unwrap_or_else(|e| panic!("{e}")) + .layout() + .is_none() + ); + let layout = codec.layout().cloned(); + let tables = |rows: bool, choice: IndexChoice| { + let hashing = Hashing::new(Some(3), rows); + let format = match layout.as_deref().filter(|_| rows) { + Some(keys) => ArenaFormat::Rows { + keys, + payload: None, + arrow_payload: false, + }, + None => ArenaFormat::Arrow { payload: false }, + }; + let schema = arena_schema(format).unwrap_or_else(|e| panic!("{e}")); + let keys = codec + .encode(&cols, &hashing) + .unwrap_or_else(|e| panic!("{e}")); + let valid: Vec = (0..n).filter(|&i| keys.is_valid(i)).collect(); + let chunk = |idx: &[usize]| { + let mut c: Vec = vec![Arc::new( + UInt64Array::from_iter_values(idx.iter().map(|&i| keys.hashes()[i])), + )]; + let l = layout.as_deref().filter(|_| rows); + c.extend(keys.arena_columns(l, idx).unwrap_or_else(|e| panic!("{e}"))); + Batch::new( + RecordBatch::try_new(Arc::clone(&schema), c) + .unwrap_or_else(|e| panic!("{e}")), + ) + }; + let (a, b) = valid.split_at(valid.len() / 3); + let arena = RowArena::over_chunks(&schema, &[chunk(a), chunk(b)]) + .unwrap_or_else(|e| panic!("{e}")); + let l = layout.as_ref().filter(|_| rows); + let t = JoinTable::build_with(arena, l, hashing.g_seed(), choice, None) + .unwrap_or_else(|e| panic!("{e}")); + let probe: Vec = (0..u32::try_from(n).unwrap_or(0)) + .filter(|&i| keys.is_valid(i as usize)) + .collect(); + let mut heads = Vec::new(); + t.find_heads(&keys, &probe, &mut heads); + // A probe in the other format finds nothing. + let other = codec + .encode(&cols, &Hashing::new(Some(3), !rows)) + .unwrap_or_else(|e| panic!("{e}")); + let mut none = Vec::new(); + t.find_heads(&other, &probe[..3], &mut none); + assert_eq!(none, [NONE; 3]); + assert_eq!(t.is_unique(), unique, "modulus {m}"); + (heads, t.next().to_vec(), t.flat_index()) }; - let (a, b) = valid.split_at(valid.len() / 3); - let arena = RowArena::over_chunks(&schema, &[chunk(a), chunk(b)]) - .unwrap_or_else(|e| panic!("{e}")); - let l = layout.as_ref().filter(|_| rows); - let t = JoinTable::build_with(arena, l, hashing.g_seed(), choice, None) - .unwrap_or_else(|e| panic!("{e}")); - let probe: Vec = (0..u32::try_from(n).unwrap_or(0)) - .filter(|&i| keys.is_valid(i as usize)) - .collect(); - let mut heads = Vec::new(); - t.find_heads(&keys, &probe, &mut heads); - // A probe in the other format finds nothing. - let other = codec - .encode(&cols, &Hashing::new(Some(3), !rows)) - .unwrap_or_else(|e| panic!("{e}")); - let mut none = Vec::new(); - t.find_heads(&other, &probe[..3], &mut none); - assert_eq!(none, [NONE; 3]); - (heads, t.next().to_vec(), t.flat_index()) - }; - let (arrow_heads, arrow_next, kind) = tables(false, IndexChoice::Adaptive); - assert_eq!(kind, None); - assert!(arrow_heads.iter().all(|&h| h != NONE)); - for choice in [IndexChoice::Flat, IndexChoice::Salted] { - let (heads, next, kind) = tables(true, choice); - assert_eq!(kind, Some(choice == IndexChoice::Flat)); - assert_eq!(heads, arrow_heads, "{choice:?}"); - assert_eq!(next, arrow_next, "{choice:?}"); + let (arrow_heads, arrow_next, kind) = tables(false, IndexChoice::Adaptive); + assert_eq!(kind, None); + assert!(arrow_heads.iter().all(|&h| h != NONE)); + for choice in [IndexChoice::Flat, IndexChoice::Salted] { + let (heads, next, kind) = tables(true, choice); + assert_eq!(kind, Some(choice == IndexChoice::Flat)); + assert_eq!(heads, arrow_heads, "{choice:?}"); + assert_eq!(next, arrow_next, "{choice:?}"); + } } } From 27b2579797842677959a2b3bd465ea83737f611a Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:06:06 +0000 Subject: [PATCH 13/25] =?UTF-8?q?exec:=20a=20built=20join=20table=20keeps?= =?UTF-8?q?=20reserved=20what=20its=20arena=20holds=20(study=2021=20=C2=A7?= =?UTF-8?q?4.4,=20option=20C's=20block=20slack):=20the=20chunks=20were=20r?= =?UTF-8?q?eserved=20at=20their=20allocated=20size,=20builder=20slack=20an?= =?UTF-8?q?d=20tails=20charged=20at=20the=20chunk=20they=20were=20sliced?= =?UTF-8?q?=20from;=20finalize=20copies=20the=20tails=20into=20exact=20blo?= =?UTF-8?q?cks=20and=20drops=20the=20chunks,=20then=20trues=20each=20parti?= =?UTF-8?q?tion's=20reservation=20down=20to=20the=20arena's=20buffers=20(R?= =?UTF-8?q?owArena::held=5Fbytes,=20each=20buffer's=20capacity=20counted?= =?UTF-8?q?=20once)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A build of 256-row batches over 40,000 keys kept 3.24 MB reserved for a 2.40 MB arena (+35%); trued up it keeps <= 5% over (test a_built_table_keeps_reserved_what_its_arena_holds). Study 21 measured ~27 MB of such slack per chunked build at 12 threads on q21 SF10 (query peak +17% against main, a spill at 300 MiB where main has none). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 25 ++++++++++++++ crates/krishiv-exec/src/join/table.rs | 36 +++++++++++++++++++++ crates/krishiv-exec/tests/hash_join/main.rs | 22 +++++++++++++ 3 files changed, 83 insertions(+) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index afc60ad45..45a00a431 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -424,6 +424,22 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz Some((index.saturating_add(kept), copy.saturating_sub(kept))) } +/// Returns what a built partition's chunks reserved beyond its arena: the +/// chunks were reserved at their allocated size (builder slack, and tails +/// charged at the chunk they were sliced from), the arena copied the tails +/// into exact blocks and the chunks are gone, so the reservation keeps +/// `bytes`, the arena's allocated bytes ([`RowArena::held_bytes`]), less +/// the `kept` tails reserved with the index. Never below zero, and only +/// ever shrinks. +fn trim_to_arena(part: &mut Part, kept: usize, bytes: usize) { + let mut excess = part.bytes().saturating_sub(bytes.saturating_sub(kept)); + for m in part.memory.iter_mut() { + let cut = excess.min(m.size()); + m.shrink(cut); + excess -= cut; + } +} + /// A duration in whole nanoseconds, saturating. pub(super) fn nanos(d: std::time::Duration) -> u64 { u64::try_from(d.as_nanos()).unwrap_or(u64::MAX) @@ -800,6 +816,9 @@ impl Sink for BuildSink { let mut jobs = std::collections::VecDeque::from(jobs); let mut running = futures::stream::FuturesUnordered::new(); let mut tables = Vec::with_capacity(jobs.len()); + // Per built partition: its kept tails and its arena's allocated + // bytes, to true up its chunks' reservation once they are gone. + let mut held = Vec::with_capacity(jobs.len()); loop { if let Some(&Job { need: (index_bytes, copy), @@ -905,6 +924,7 @@ impl Sink for BuildSink { self.0.counters.or(Counter::IndexKinds, kind as u64); self.0.counters.add(Counter::HashEntries, keys as u64); self.0.counters.add(Counter::HashBuckets, buckets as u64); + held.push((p, kept, table.arena().held_bytes())); tables.push((p, table)); } index.shrink(spare.min(index.size())); @@ -925,6 +945,11 @@ impl Sink for BuildSink { let mut memory = vec![index, filter_memory]; let mut grace = Grace::new(g.hashing.fanout0()); grace.keep_unprobed = s.join_type.keeps_null_build_keys(); + for (p, kept, bytes) in held { + if let Some(part) = c.parts.get_mut(p) { + trim_to_arena(part, kept, bytes); + } + } for (p, part) in c.parts.iter_mut().enumerate() { if spilled & (1 << p) == 0 { self.0 diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index d2e5541f2..f8bc670f9 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -442,6 +442,42 @@ impl RowArena { self.len } + /// The bytes the arena's blocks keep allocated: the capacity of every + /// buffer they reference, each counted once (blocks sliced from one + /// chunk share its buffers). A build's chunks were reserved at their + /// allocated size, builder slack included; once finalize copies the + /// tails into exact blocks and drops the chunks, this is what the + /// arena still holds (study 21 §4.4: the slack of the tails kept + /// reserved was ~27 MB per build at 12 threads). + pub(crate) fn held_bytes(&self) -> usize { + fn walk( + data: &arrow_data::ArrayData, + seen: &mut std::collections::HashSet, + sum: &mut usize, + ) { + let nulls = data.nulls().map(arrow_buffer::NullBuffer::buffer); + for b in data.buffers().iter().chain(nulls) { + if seen.insert(b.data_ptr().as_ptr() as usize) { + *sum = sum.saturating_add(b.capacity()); + } + } + for child in data.child_data() { + walk(child, seen, sum); + } + } + let mut seen = std::collections::HashSet::new(); + let mut sum = 0usize; + for part in &self.parts { + for column in part.batch.data().columns() { + walk(&column.to_data(), &mut seen, &mut sum); + } + if let Weights::Column(w) = part.batch.weights() { + walk(&w.to_data(), &mut seen, &mut sum); + } + } + sum + } + /// The arena's bytes by part (each block's share of its buffers): the `hash` column, the key columns (`key*`), and /// everything else (payload, NULL-key flags). pub(crate) fn memory_split(&self) -> ArenaBytes { diff --git a/crates/krishiv-exec/tests/hash_join/main.rs b/crates/krishiv-exec/tests/hash_join/main.rs index 63635321c..d4acc86de 100644 --- a/crates/krishiv-exec/tests/hash_join/main.rs +++ b/crates/krishiv-exec/tests/hash_join/main.rs @@ -1005,3 +1005,25 @@ async fn a_spilling_build_with_kept_tails_fits_or_spills_under_any_limit() { } } } + +/// A build of many small batches leaves every partition's rows in chunk +/// tails shorter than a block, each reserved at its allocated size +/// (builder slack). Finalize copies the tails into exact blocks and drops +/// the chunks, so what the table keeps reserved through the probe is +/// trued up to the arena's allocated bytes, not the chunks' (study 21 +/// §4.4: ~27 MB of such slack per build at 12 threads). +#[tokio::test(start_paused = true)] +async fn a_built_table_keeps_reserved_what_its_arena_holds() { + let build = skewed(40_000, 40_000); + // Batches of 256 rows: about 32 rows per level-0 partition and push. + let (got, q) = run_self_join(&build, 256, 512 * MIB, None).await; + got.unwrap(); + let steady = metric(&q, "HashJoinBuild", "steady_reserved_bytes"); + let arena = metric(&q, "HashJoinBuild", "arena_bytes"); + assert!(arena > 0); + // Exact blocks round each buffer up to 64 bytes, nothing more. + assert!( + steady <= arena + arena / 20, + "steady {steady} B against an arena of {arena} B" + ); +} From 7fcd46c55b01e1c7aa8e0c9fae3d1dccfcdb53fa Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:38:19 +0000 Subject: [PATCH 14/25] exec: review of the arena true-up: an index past its estimate keeps the kept tail bytes it took (the trim leaves them to the chunks' reservation); the test bounds steady from below too; held_bytes sized and documented Review: ops/reviews/2026-11-08-opus-1-x3-arena-trueup-diff.md (Opus, approve with nits). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/build.rs | 15 ++++++++++----- crates/krishiv-exec/src/join/table.rs | 10 ++++++++-- crates/krishiv-exec/tests/hash_join/main.rs | 5 +++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 45a00a431..c2b048a93 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -429,8 +429,9 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz /// charged at the chunk they were sliced from), the arena copied the tails /// into exact blocks and the chunks are gone, so the reservation keeps /// `bytes`, the arena's allocated bytes ([`RowArena::held_bytes`]), less -/// the `kept` tails reserved with the index. Never below zero, and only -/// ever shrinks. +/// the `kept` tail bytes the index reservation still holds (so the two +/// together cover the arena; `kept` may be all of it, leaving the chunks' +/// reservation nothing). Never below zero, and only ever shrinks. fn trim_to_arena(part: &mut Part, kept: usize, bytes: usize) { let mut excess = part.bytes().saturating_sub(bytes.saturating_sub(kept)); for m in part.memory.iter_mut() { @@ -816,8 +817,9 @@ impl Sink for BuildSink { let mut jobs = std::collections::VecDeque::from(jobs); let mut running = futures::stream::FuturesUnordered::new(); let mut tables = Vec::with_capacity(jobs.len()); - // Per built partition: its kept tails and its arena's allocated - // bytes, to true up its chunks' reservation once they are gone. + // Per built partition: the kept tails its index reservation + // holds and its arena's allocated bytes, to true up its chunks' + // reservation once they are gone. let mut held = Vec::with_capacity(jobs.len()); loop { if let Some(&Job { @@ -924,7 +926,10 @@ impl Sink for BuildSink { self.0.counters.or(Counter::IndexKinds, kind as u64); self.0.counters.add(Counter::HashEntries, keys as u64); self.0.counters.add(Counter::HashBuckets, buckets as u64); - held.push((p, kept, table.arena().held_bytes())); + // Of `kept`, what the index reservation still holds once the + // index is paid: an index past its estimate took the rest. + let covered = lasting.saturating_sub(table.index_memory()).min(kept); + held.push((p, covered, table.arena().held_bytes())); tables.push((p, table)); } index.shrink(spare.min(index.size())); diff --git a/crates/krishiv-exec/src/join/table.rs b/crates/krishiv-exec/src/join/table.rs index f8bc670f9..16de606a4 100644 --- a/crates/krishiv-exec/src/join/table.rs +++ b/crates/krishiv-exec/src/join/table.rs @@ -448,7 +448,9 @@ impl RowArena { /// allocated size, builder slack included; once finalize copies the /// tails into exact blocks and drops the chunks, this is what the /// arena still holds (study 21 §4.4: the slack of the tails kept - /// reserved was ~27 MB per build at 12 threads). + /// reserved was ~27 MB per build at 12 threads). Once per partition at + /// finalize: it clones each column's `ArrayData`, so it stays off hot + /// paths. pub(crate) fn held_bytes(&self) -> usize { fn walk( data: &arrow_data::ArrayData, @@ -465,7 +467,11 @@ impl RowArena { walk(child, seen, sum); } } - let mut seen = std::collections::HashSet::new(); + let columns = self + .parts + .first() + .map_or(0, |p| p.batch.data().num_columns()); + let mut seen = std::collections::HashSet::with_capacity(self.parts.len() * columns * 3); let mut sum = 0usize; for part in &self.parts { for column in part.batch.data().columns() { diff --git a/crates/krishiv-exec/tests/hash_join/main.rs b/crates/krishiv-exec/tests/hash_join/main.rs index d4acc86de..f4c06d089 100644 --- a/crates/krishiv-exec/tests/hash_join/main.rs +++ b/crates/krishiv-exec/tests/hash_join/main.rs @@ -1021,9 +1021,10 @@ async fn a_built_table_keeps_reserved_what_its_arena_holds() { let steady = metric(&q, "HashJoinBuild", "steady_reserved_bytes"); let arena = metric(&q, "HashJoinBuild", "arena_bytes"); assert!(arena > 0); - // Exact blocks round each buffer up to 64 bytes, nothing more. + // Exact blocks round each buffer up to 64 bytes, nothing more; and + // the true-up never leaves the arena under-reserved. assert!( - steady <= arena + arena / 20, + steady >= arena - arena / 20 && steady <= arena + arena / 20, "steady {steady} B against an arena of {arena} B" ); } From a4a089b869fd79c87db6c826c263809fc9056e7b Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:00:08 +0000 Subject: [PATCH 15/25] =?UTF-8?q?exec:=20a=20salted=20join=20directory=20i?= =?UTF-8?q?s=20the=20next=20power=20of=20two=20of=205/4=20of=20its=20keys,?= =?UTF-8?q?=20at=20load=20up=20to=204/5=20(study=2021=20=C2=A75-6,=20S5=20?= =?UTF-8?q?lever=203;=20DuckDB's=20load=20factor),=20instead=20of=20twice?= =?UTF-8?q?=20its=20keys=20at=20load=20up=20to=201/2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit next_pow2(2 x keys) landed at 3.7 slots per key on q21's 145 M-key build at SF1000 (5.1 GB, a third of op 6's held table); at 5/4 it saves 2.5 GB on op 6 and 1.3 GB on op 5 and op 4 there, and nothing at SF10, where the rounding lands on the same power. The 16-bit salt keeps a probe from reading a row for all but 1 in 65,536 of the slots it passes; the probe cost at the higher load is what perf-1's A/B against p2b/opus1-x3-copy-free decides (study 15 measured max_probe_length 2 at load 0.35). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/rows/index.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/krishiv-exec/src/rows/index.rs b/crates/krishiv-exec/src/rows/index.rs index d2fef0496..73415a64d 100644 --- a/crates/krishiv-exec/src/rows/index.rs +++ b/crates/krishiv-exec/src/rows/index.rs @@ -7,7 +7,7 @@ //! enough to stay in a core's own cache, where its lower instruction count //! wins; //! - **salted**: a power-of-two directory of `u64` slots, each a 16-bit -//! salt plus `row id + 1`, probed linearly at load ≤ 1/2 (DuckDB's +//! salt plus `row id + 1`, probed linearly at load ≤ 4/5 (DuckDB's //! `ht_entry_t`, with row ids instead of pointers), above that size, //! where a probe costs one directory miss and reads a row only when its //! salt matches. Its lookups prefetch each chunk's slots, then its first @@ -123,9 +123,19 @@ pub fn flat_bytes(n: usize) -> usize { .saturating_add(16) } -/// Slots of a salted directory for `n` keys at load ≤ 1/2. +/// Load of a salted directory at most `SALTED_LOAD_NUM / SALTED_LOAD_DEN` +/// (4/5): the directory is the next power of two of 5/4 of its keys +/// (DuckDB's load factor; study 21 §5: `next_pow2(2 x keys)` landed at +/// 3.7 slots per key on q21's 145 M-key build at SF1000, a third of its +/// table's bytes), so a filled one sits between 2/5 and 4/5. The 16-bit +/// salt keeps a probe from reading a row for all but 1 in 65,536 of the +/// slots it passes. +const SALTED_LOAD_NUM: usize = 4; +const SALTED_LOAD_DEN: usize = 5; + +/// Slots of a salted directory for `n` keys at load ≤ 4/5. fn salted_slots(n: usize) -> usize { - n.saturating_mul(2) + n.saturating_add(n / SALTED_LOAD_NUM) .checked_next_power_of_two() .unwrap_or(usize::MAX) .max(MIN_SLOTS) @@ -244,7 +254,7 @@ impl JoinIndex { slots.resize(len, 0); let mut mask = len - 1; for id in (0..n).filter(|&id| src.indexed(id)) { - if (keys + 1) * 2 > slots.len() { + if (keys + 1) * SALTED_LOAD_DEN > slots.len() * SALTED_LOAD_NUM { slots = grow_salted(&slots, g_of)?; mask = slots.len() - 1; } From 8a32d024230ca1c21d5283c46335771ac43aca95 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:25:43 +0000 Subject: [PATCH 16/25] optimizer, common: a dimension's payload joins above the semi and anti joins that never read it (LatePayloadJoin, P2b S5 lever 2; q21's s_name) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TPC-H q21 joins supplier ⋈ nation('SAUDI ARABIA') to l1 first and then carries s_name (31 of the 79 B of arena a build row) through the join with orders, the EXISTS semi join and the NOT EXISTS anti join, all built on the l1 side; only the aggregate on top reads it (study 21 §4.1, §6). The new rule (late_payload.rs) moves such an inner equi-join to the top of its path and leaves a key-only semi join in its place: Consumer(Path(X ⋈[x = d] D)) => Consumer(Project[out(Path)](Path'(X ⋉[x = d'] Shift(π_d D)) ⋈[x = d] D)) Exact with no uniqueness assumption: every admitted path node (filters, projections, inner joins either side, left semi/anti/existence/outer joins with the path on the left, right outer on the right; none reading the dimension's columns, all deterministic) commutes with an inner join on columns it does not read, so each X row still meets every matching D row; the semi join drops only rows the top join drops. D is computed twice (the semi join's instance under a Shift, as restrict does), so it must be restrict::repeatable. Applies only with at least one non-inner join on the path (join ordering never moves an inner join back across one, so the plan stays a fixed point), when no other parent reads the join or a path node (no copied shared subtree), and with statistics on both join sides (never on default row counts): the carried payload is variable-width or >= 16 B a row, the dimension estimates no more rows than the other side and scans no more rows than it holds (it is computed twice), and some join on the path builds the path's side as the physical planner orients it (an inner join's right input; the left input of a semi, non-null-aware anti or left outer join whose right input costs more than 2x and is not broadcast, orient.rs; the broadcast threshold is the session's). A payload on a probe side only streams. Runs after Prune/Tidy, so the payload is what the path really carries; when it moves a join, join order, eager aggregation and the prune/tidy loop (now prune_and_tidy) run again. Several dimensions under one consumer all move in one pass. EXPLAIN VERBOSE notes each move and each refused gate, once each (LatePayloadJoin). Switch: krishiv.optimizer.latePayloadJoin (session, user, default true), Options::late_payload_join. Design: krishiv-internal design/P2b-S5-dimension-payload-late.md. Plans: TPC-H with the statistics fixture changes q21 (s_name) and q18 (c_name leaves the orders build of the join with lineitem; q18's SF1000 row is the one that spills); TPC-DS with SF1 row counts changes q35, q58, q72 and q83 (listed in a snapshot); without statistics nothing changes. Tests: optimizer tests/it/late_payload.rs: q21 and q18 move and nothing else in TPC-H does; repeating dimension keys (nation on n_regionkey, five rows a key) under semi, anti and projection consumers and NULL keys; two dimensions under one consumer; a left outer path; refusals (payload or dimension key read by the semi join, a nondeterministic dimension, an 8 B payload, the larger dimension, a broadcast semi side, an existence join, a right outer join with the path on the right, an all-inner path, default row counts); the switch by option and config; each checked against the reference interpreter and for the fixed point; TPC-DS with row counts is a fixed point. exec tests/planner/late_payload.rs: q21, q18 and two repeating-key queries at SF0.01 (broadcast off, so the rule fires) return the rows of the same plan with the rule off. common: the key's default and parsing. Review: krishiv-internal ops/reviews/2026-11-09-opus-1-late-payload-diff.md (Reviewer: Opus; request changes, fixed: the orient-matching build gate, the executor test against the rule off, TPC-DS with statistics, the missing refusals and NULL keys, the D cost bound, shared path nodes, the prune/tidy loop, module order). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-common/src/config/registry.rs | 19 + crates/krishiv-common/src/config/tests.rs | 15 + .../tests/planner/late_payload.rs | 60 ++ crates/krishiv-exec/tests/planner/main.rs | 1 + crates/krishiv-exec/tests/planner/tpch.rs | 24 + crates/krishiv-optimizer/CLAUDE.md | 2 + crates/krishiv-optimizer/src/late_payload.rs | 785 ++++++++++++++++++ crates/krishiv-optimizer/src/lib.rs | 138 ++- crates/krishiv-optimizer/src/plan_stats.rs | 2 +- crates/krishiv-optimizer/src/restrict.rs | 6 +- .../tests/it/late_payload.rs | 350 ++++++++ crates/krishiv-optimizer/tests/it/main.rs | 1 + .../it/snapshots/it__join_order__q18.snap | 30 +- .../it/snapshots/it__join_order__q21.snap | 59 +- ...pcds_with_row_counts_is_a_fixed_point.snap | 22 + docs/reference/config/index.md | 1 + 16 files changed, 1444 insertions(+), 71 deletions(-) create mode 100644 crates/krishiv-exec/tests/planner/late_payload.rs create mode 100644 crates/krishiv-optimizer/src/late_payload.rs create mode 100644 crates/krishiv-optimizer/tests/it/late_payload.rs create mode 100644 crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap diff --git a/crates/krishiv-common/src/config/registry.rs b/crates/krishiv-common/src/config/registry.rs index 7bf0a0114..68c1f736b 100644 --- a/crates/krishiv-common/src/config/registry.rs +++ b/crates/krishiv-common/src/config/registry.rs @@ -112,6 +112,7 @@ pub static REGISTRY: &[&ConfigEntry] = &[ &AGENT_GATE_BYTES, &SHARE_CTE_INSTANCES, &EAGER_AGGREGATION, + &LATE_PAYLOAD_JOIN, &JOIN_ENUMERATOR, &JOIN_DP_LIMIT, &NDV_SAMPLING_ENABLED, @@ -228,6 +229,7 @@ pub static INDEX: &[(&str, &ConfigEntry)] = &[ "krishiv.memory.reservationQuantum", &MEMORY_RESERVATION_QUANTUM, ), + ("krishiv.optimizer.latePayloadJoin", &LATE_PAYLOAD_JOIN), ( "krishiv.sql.files.maxPartitionNum", &FILES_MAX_PARTITION_NUM, @@ -1908,6 +1910,23 @@ pub static NDV_SAMPLING_TIMEOUT: ConfigEntry = ConfigEntry { spark_alias: None, }; +/// A dimension's payload joins above the semi and anti joins that never read +/// it (`krishiv.optimizer.latePayloadJoin`). +pub static LATE_PAYLOAD_JOIN: ConfigEntry = ConfigEntry { + key: "krishiv.optimizer.latePayloadJoin", + value_type: ValueType::Bool, + default: Some(ConfigValue::Bool(true)), + doc: "Join a dimension's payload columns (such as a name read only by the aggregate on top) \ + after the semi, anti and outer joins that never read them, and leave a semi join on the \ + dimension's keys where the join was, so those joins build and probe narrower rows. Applies \ + when statistics show the payload is wide and the dimension is the smaller side. Results \ + are the same either way.", + scope: Scope::Session, + settable_by: SettableBy::User, + version_added: "0.1.0", + spark_alias: None, +}; + /// Eager aggregation below joins (`krishiv.optimizer.eagerAggregation`). pub static EAGER_AGGREGATION: ConfigEntry = ConfigEntry { key: "krishiv.optimizer.eagerAggregation", diff --git a/crates/krishiv-common/src/config/tests.rs b/crates/krishiv-common/src/config/tests.rs index d124ecc72..db6475405 100644 --- a/crates/krishiv-common/src/config/tests.rs +++ b/crates/krishiv-common/src/config/tests.rs @@ -1094,3 +1094,18 @@ fn ui_enabled_defaults_on_and_parses() { assert!(c.set("krishiv.ui.enabled", "maybe").is_err()); assert_eq!(UI_ENABLED.scope, Scope::Cluster); } + +#[test] +fn late_payload_join_defaults_on_and_parses() { + let mut c = Config::new(); + assert_eq!( + c.get(&LATE_PAYLOAD_JOIN).and_then(ConfigValue::as_bool), + Some(true) + ); + c.set("krishiv.optimizer.latePayloadJoin", "false").unwrap(); + assert_eq!( + c.get(&LATE_PAYLOAD_JOIN).and_then(ConfigValue::as_bool), + Some(false) + ); + assert!(c.set("krishiv.optimizer.latePayloadJoin", "maybe").is_err()); +} diff --git a/crates/krishiv-exec/tests/planner/late_payload.rs b/crates/krishiv-exec/tests/planner/late_payload.rs new file mode 100644 index 000000000..c0272a834 --- /dev/null +++ b/crates/krishiv-exec/tests/planner/late_payload.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The late payload join rule (P2b S5 lever (2), optimizer `late_payload`): +//! where it moves a dimension's payload above the semi and anti joins, the +//! executed plan returns the rows of the same plan with the rule off. +//! +//! The data set is the committed TPC-H SF0.01 Parquet fixture (the design note +//! asked for SF0.1, which is not committed). + +use std::path::Path; + +use super::partial_aggregate::{physical, runtime, sorted}; +use super::run; +use super::tpch::{parquet_tables, sql}; + +/// TPC-H q21 and q18, and queries whose dimension key repeats (five nations a +/// region), with the rule on and off over the same statistics. +#[test] +fn moved_payload_joins_return_the_rows_of_the_rule_off() { + let rt = runtime(); + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tpch-sf0.01-parquet"); + let t = parquet_tables(&dir, &rt); + let queries = [ + sql(21), + sql(18), + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name" + .to_owned(), + "SELECT n_name, count(*), sum(s_acctbal) FROM supplier, nation \ + WHERE s_nationkey = n_regionkey \ + AND NOT EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey \ + AND l_comment LIKE '%xyzzy%') GROUP BY n_name" + .to_owned(), + ]; + // A large query's semi and anti joins are not broadcast; at SF0.01 every + // one would be, and then none builds the path. + let on = [("spark.sql.autoBroadcastJoinThreshold", "-1")]; + let off = [ + ("spark.sql.autoBroadcastJoinThreshold", "-1"), + ("krishiv.optimizer.latePayloadJoin", "false"), + ]; + for q in &queries { + let (with, steps) = t.optimized_in(q, &on).unwrap(); + let (without, _) = t.optimized_in(q, &off).unwrap(); + let notes: Vec<&String> = steps + .iter() + .filter(|s| s.batch == "LatePayloadJoin") + .map(|s| &s.note) + .collect(); + // The rule fired: it is the only difference between the two plans. + assert!( + notes.iter().any(|n| n.starts_with("moved")) && !with.structurally_eq(&without), + "{q}\n{notes:?}\n{}", + with.root() + ); + let got = sorted(&run(&physical(&t, &with).unwrap()).unwrap()); + let want = sorted(&run(&physical(&t, &without).unwrap()).unwrap()); + assert!(!want.is_empty(), "{q}"); + assert_eq!(got, want, "{q}\n{}", with.root()); + } +} diff --git a/crates/krishiv-exec/tests/planner/main.rs b/crates/krishiv-exec/tests/planner/main.rs index c94a8dc3c..869d58a6a 100644 --- a/crates/krishiv-exec/tests/planner/main.rs +++ b/crates/krishiv-exec/tests/planner/main.rs @@ -22,6 +22,7 @@ mod folding; mod group_limit; mod hints; mod late_fetch; +mod late_payload; mod listing; mod orientation; mod partial_aggregate; diff --git a/crates/krishiv-exec/tests/planner/tpch.rs b/crates/krishiv-exec/tests/planner/tpch.rs index f8d418ca1..d475caedd 100644 --- a/crates/krishiv-exec/tests/planner/tpch.rs +++ b/crates/krishiv-exec/tests/planner/tpch.rs @@ -221,6 +221,30 @@ impl Tpch { }) } + /// `text` parsed, analyzed and optimized by the tables' statistics under + /// the session `settings`, with the optimizer's trace. + pub(crate) fn optimized_in( + &self, + text: &str, + settings: &[(&str, &str)], + ) -> Result<(Plan, Vec), KrishivError> { + let plan = + krishiv_sql::parse_and_lower(text, &krishiv_sql::ParserOptions::default(), SourceId(0)) + .map_err(KrishivError::from)?; + let plan: Plan = block_on(DefaultAnalyzer.analyze(&plan, &self.ctx))?.plan; + let stats = ProviderStatistics(Arc::clone(&self.tables) as _); + let mut config = Config::new(); + for (k, v) in settings { + config.set(k, v)?; + } + krishiv_optimizer::optimize_traced_with_udfs( + &plan, + &stats, + &krishiv_udf_sdk::UdfLibrary::new(), + &config, + ) + } + /// As [`Self::plan_sql_with`], its operators spilling into `spill`. pub(crate) fn plan_sql_in( &self, diff --git a/crates/krishiv-optimizer/CLAUDE.md b/crates/krishiv-optimizer/CLAUDE.md index 5a4e8efb7..35db2f89c 100644 --- a/crates/krishiv-optimizer/CLAUDE.md +++ b/crates/krishiv-optimizer/CLAUDE.md @@ -16,6 +16,7 @@ Paths are relative to this crate. - `src/restrict.rs` subquery restriction rules - `src/inline_cte.rs` CTE inlining (a nondeterministic definition read twice stays a `CteRef`); `src/share_cte.rs` `ShareCte` (P2-T198): equal deterministic instances with a pipeline breaker become one `WITH` definition, only when a session configuration is given and `krishiv.sql.optimizer.shareCteInstances` is on - `src/eager.rs` `EagerAggregate` (P2-T167): an aggregate's calls run below a join and merge above it, gated on statistics; `krishiv.optimizer.eagerAggregation`, `Options::eager_aggregation` +- `src/late_payload.rs` `LatePayloadJoin` (P2b S5 lever (2), design `design/P2b-S5-dimension-payload-late.md`): a dimension's payload columns join above the semi/anti/outer joins that never read them, a key-only semi join left in place; runs after pruning, statistics only; `krishiv.optimizer.latePayloadJoin`, `Options::late_payload_join` - `src/join_order.rs` inner-join ordering (enumerator choice `krishiv.optimizer.joinEnumerator` / `joinDpLimit`, a `TreeStats` note per tree in `EXPLAIN VERBOSE`); `src/join_order/dp.rs` the exact enumerator DPsub over a hypergraph of input masks (P2-T113 kept it over DPhyp: `exp/t113-dphyp`); `src/stats.rs` `Statistics` it reads and the estimate types (`RelStats`, `AttrStats`, `StatSource`) - `src/estimate.rs` the `Estimator`: what each operator leaves known of its columns (a column a filter or join does not read keeps the distinct values the kept rows are expected to hold, `kept_distinct`, Cardenas); `src/narrow.rs` predicates over estimates (selectivity and narrowed ranges); `src/plan_stats.rs` `PlanStats`, one pass over an optimized plan (physical planner, `EXPLAIN COST`) - `tests/it/plan_stats.rs` one test per propagation rule; `tests/it/qerror.rs` the q-error report (`cargo xtask qerror`) @@ -35,6 +36,7 @@ Paths are relative to this crate. decimals are never rewritten. Conditions and the one residual assumption (fewer than 2^46 rows per aggregate) are in the module docs; the executor differential test is `krishiv-exec` `tests/planner/sum_rewrite.rs`. - Eager aggregation moves only calls whose result cannot depend on row order and whose arguments cannot raise (`eager.rs` module docs); it reads statistics only, runs after join ordering, and states each decision in `EXPLAIN VERBOSE`. The executor differential is `krishiv-exec` `tests/planner/eager_aggregate.rs`. +- Late payload joins move an inner equi-join only above filters, projections and joins that do not read the dimension's columns and are deterministic, with at least one non-inner join on the path (join ordering never moves an inner join back across one: fixed point); exact for any key multiplicity (join commutation, no uniqueness assumed); the dimension must be `restrict::repeatable` (computed twice). Tests: `tests/it/late_payload.rs`. - Join ordering re-plans each inner-join tree (through deterministic filters and column selections) from `Statistics`; it never adds a cross product, places each conjunct at the lowest join that has its inputs (subquery conjuncts as filters), and restores column order only for positional consumers. Inputs are diff --git a/crates/krishiv-optimizer/src/late_payload.rs b/crates/krishiv-optimizer/src/late_payload.rs new file mode 100644 index 000000000..da4a94610 --- /dev/null +++ b/crates/krishiv-optimizer/src/late_payload.rs @@ -0,0 +1,785 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `LatePayloadJoin`: a dimension's payload columns join above the semi and +//! anti joins that never read them (P2b S5 lever (2); design +//! `design/P2b-S5-dimension-payload-late.md` in krishiv-internal). +//! +//! ```text +//! Consumer(Path(X ⋈[x = d] D)) +//! => Consumer(Project[out(Path)](Path'(X ⋉[x = d'] Shift(π_d D)) ⋈[x = d] D)) +//! ``` +//! +//! `D` is one side of an inner equi-join whose other columns (the payload, +//! TPC-H q21's `s_name`) ride through every build of the path above it and are +//! first read by the consumer. Each operator on the path commutes with an +//! inner join on columns it does not read: `(X ⋈ D) ⋉ L = (X ⋉ L) ⋈ D` when +//! `L`'s condition reads only `X`, the same for an anti join, an existence +//! join and the preserved side of an outer join, and filters and projections +//! trivially. So the join moves to the top of the path with the same rows, +//! each `X` row joined to every matching `D` row whatever the multiplicity; +//! no key needs to be unique. The semi join left in its place drops only the +//! rows the top join drops (no match, NULL keys included), so the path keeps +//! the reduction the dimension gave it. `D` is computed twice, the semi +//! join's instance under a [`PlanNode::Shift`] as in `restrict`, so it must +//! be repeatable (`restrict::repeatable`). +//! +//! # When it applies +//! +//! - The join is inner, its condition only plain `=` of a column of each side, +//! of one type whose equality and grouping agree; +//! - the path from the join to its consumer holds only filters, projections +//! and joins that do not read the payload or `D`'s key columns, with the path +//! on the side whose rows they keep (inner either side; left semi, anti, +//! existence and outer on the left; right outer on the right), and every +//! expression on it is deterministic (a filter evaluated once per `X` row +//! instead of once per joined copy would sample differently); +//! - at least one join on the path is not inner: join ordering never moves an +//! inner join across one, so it cannot undo the rule, and once applied the +//! rule finds no such join between the moved join and its consumer: the +//! plan stays a fixed point. An all-inner path is join ordering's business; +//! - the join and the path's nodes are read by no other parent (rebuilding +//! them would copy a shared subtree); +//! - gates, from statistics only (both sides' estimates `from_statistics`, +//! never default row counts): the payload the path carries holds a +//! variable-width column or at least [`MIN_PAYLOAD_BYTES`] a row; `D` +//! estimates no more rows than `X`, and its scans read no more rows than +//! `X` holds (it is computed twice); and some join on the path builds the +//! path's side as the physical planner orients it (`builds_path`: an inner +//! join's right input, or the left input of a semi, anti or left outer join +//! whose right input costs more than twice as much and is not broadcast). A +//! payload on a probe side only streams, and moving it would only add work. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use krishiv_common::error::KrishivError; +use krishiv_plan::{ + AttrId, BoundColumn, Builtin, Expr, JoinCondition, JoinKind, LogicalPlan, PlanNode, +}; +use krishiv_types::DataType; + +use crate::dag; +use crate::estimate::Estimator; +use crate::expr::{ + Attrs, builtin, column, conjuncts, is_deterministic, output_of, recall, references, +}; +use crate::restrict::{MAX_ID, key_type, max_id, repeatable, shift}; +use crate::stats::Statistics; + +/// The least fixed width, in bytes a row, of a payload worth moving. +pub(crate) const MIN_PAYLOAD_BYTES: usize = 16; + +/// The most joins moved below one consumer. +const MAX_MOVES: usize = 16; + +/// The most plan nodes one walk down from a consumer visits. +const MAX_WALK: usize = 256; + +/// Moves dimension payload joins above the non-inner joins of their path, +/// entering subquery plans. Also returns one line per move, and per refused +/// candidate the gate that refused it (`EXPLAIN VERBOSE`). +/// +/// `broadcast_max` is the physical planner's broadcast threshold +/// (`spark.sql.autoBroadcastJoinThreshold`, `None` when off): a join that +/// broadcasts its other side never builds the path. +pub(crate) fn rewrite( + root: Arc, + stats: &dyn Statistics, + propagate: bool, + broadcast_max: Option, +) -> Result<(Arc, Vec), KrishivError> { + let mut ctx = Context { + next: max_id(&root, &mut HashMap::new()).saturating_add(1), + notes: Vec::new(), + shared: shared_nodes(&root), + broadcast_max, + }; + let mut est = Estimator::with_propagation(stats, propagate); + let out = dag::rewrite_up(root, &mut |p| { + // Several dimensions may carry payloads to one consumer: move each, + // so one pass reaches the fixed point. + let mut at = p; + for _ in 0..MAX_MOVES { + match at_consumer(&at, &mut est, &mut ctx) { + Some(moved) => at = moved, + None => break, + } + } + Ok(at) + })?; + Ok((out, ctx.notes)) +} + +/// What one pass carries between consumers. +struct Context { + /// The next free attribute id. + next: u64, + /// `EXPLAIN VERBOSE` lines, each once. + notes: Vec, + /// Nodes read by more than one parent in the input plan. + shared: HashSet<*const LogicalPlan>, + broadcast_max: Option, +} + +/// Appends `line` unless it is there already (the moves below one consumer +/// walk the same refused candidates again). +fn note(notes: &mut Vec, line: String) { + if !notes.contains(&line) { + notes.push(line); + } +} + +/// The nodes of `root` that more than one parent reads (subquery plans not +/// entered: the rule rewrites inside them as their own roots). +fn shared_nodes(root: &Arc) -> HashSet<*const LogicalPlan> { + let mut parents: HashMap<*const LogicalPlan, usize> = HashMap::new(); + let mut seen = HashSet::new(); + let mut stack = vec![Arc::clone(root)]; + while let Some(p) = stack.pop() { + if !seen.insert(Arc::as_ptr(&p)) { + continue; + } + p.for_each_child(&mut |c| { + *parents.entry(Arc::as_ptr(c)).or_default() += 1; + stack.push(Arc::clone(c)); + }); + } + parents + .into_iter() + .filter(|(_, n)| *n > 1) + .map(|(p, _)| p) + .collect() +} + +/// Whether the rule can put a new child under `p` (`with_child`). +fn replaceable(p: &LogicalPlan) -> bool { + matches!( + p.node, + PlanNode::Join { .. } + | PlanNode::Project { .. } + | PlanNode::Filter { .. } + | PlanNode::Aggregate { .. } + | PlanNode::Sort { .. } + | PlanNode::Limit { .. } + | PlanNode::Offset { .. } + | PlanNode::SubqueryAlias { .. } + | PlanNode::WithCte { .. } + | PlanNode::Shift { .. } + | PlanNode::JoinHint { .. } + | PlanNode::Repartition { .. } + | PlanNode::Window { .. } + | PlanNode::Generate { .. } + | PlanNode::ReplicateRows { .. } + ) +} + +/// How much larger the other side's table must be before the physical +/// planner builds a semi, anti or left outer join's left input +/// (`krishiv-exec` `planner/orient.rs` `STATS_FACTOR`, both sides from +/// statistics; `DEFAULT_FACTOR` otherwise). +const STATS_FACTOR: f64 = 2.0; +const DEFAULT_FACTOR: f64 = 8.0; + +/// Whether the physical planner builds the hash table of `s`'s join from the +/// path's input (`krishiv-exec` `planner/join.rs` and `planner/orient.rs`): +/// an inner join's right input, or the left input of a semi, anti (not +/// null-aware) or left outer join whose right input costs more than the +/// factor times as much and is not broadcast. Existence, null-aware anti and +/// right outer joins never build their left input. +fn builds_path(s: &Step, est: &mut Estimator<'_>, broadcast_max: Option) -> bool { + let PlanNode::Join { + kind, + condition: JoinCondition::On(on), + .. + } = &s.node.node + else { + return false; + }; + let kids = children(&s.node); + let (Some(path), Some(other)) = (kids.get(s.child), kids.get(usize::from(s.child == 0))) else { + return false; + }; + match kind { + JoinKind::Inner => s.child == 1, + JoinKind::LeftSemi | JoinKind::LeftOuter => { + s.child == 0 && flips(path, other, est, broadcast_max) + } + JoinKind::LeftAnti => { + s.child == 0 && !null_aware(on) && flips(path, other, est, broadcast_max) + } + _ => false, + } +} + +/// `orient.rs`'s decision to build the left input over `right`. +fn flips( + left: &Arc, + right: &Arc, + est: &mut Estimator<'_>, + broadcast_max: Option, +) -> bool { + let (l, r) = (est.plan(left), est.plan(right)); + let (lb, rb) = (bytes(left, l.rows), bytes(right, r.rows)); + if broadcast_max.is_some_and(|max| rb <= max) { + return false; + } + let factor = if l.from_statistics && r.from_statistics { + STATS_FACTOR + } else { + DEFAULT_FACTOR + }; + rb > factor * lb +} + +/// Spark's size estimate of `rows` rows of `p`'s output. +fn bytes(p: &LogicalPlan, rows: f64) -> f64 { + let width = p + .resolved_output() + .unwrap_or_default() + .iter() + .map(|c| crate::plan_stats::default_size(&c.data_type)) + .fold(8, u64::saturating_add); + #[expect(clippy::cast_precision_loss, reason = "an estimate")] + let width = width as f64; + rows.max(0.0) * width +} + +/// Whether `on` is Spark's null-aware anti-join key `(a = b) OR isnull(a = b)`. +fn null_aware(on: &Arc) -> bool { + let conj = conjuncts(on); + let [only] = conj.as_slice() else { + return false; + }; + matches!(builtin(only), Some((Builtin::Or, [_, is_null], _)) + if matches!(builtin(is_null), Some((Builtin::IsNull, _, _)))) +} + +/// The rows the scans of `p` read, as estimated. +fn scanned_rows(p: &Arc, est: &mut Estimator<'_>) -> f64 { + let mut total = 0.0; + let mut seen = HashSet::new(); + let mut stack = vec![Arc::clone(p)]; + while let Some(x) = stack.pop() { + if !seen.insert(Arc::as_ptr(&x)) { + continue; + } + if matches!(x.node, PlanNode::Scan(_)) { + total += est.plan(&x).rows; + } + x.for_each_child(&mut |c| stack.push(Arc::clone(c))); + } + total +} + +/// One step of a path: a node and the index of the child the path came from. +#[derive(Clone)] +struct Step { + node: Arc, + child: usize, +} + +/// A join that may move: the path from just above it to the consumer's child +/// (bottom first), the join, and which side is the dimension. +struct Candidate { + path: Vec, + join: Arc, + dim_left: bool, +} + +/// `consumer` with one payload join below it moved up, if any qualifies. +fn at_consumer( + consumer: &Arc, + est: &mut Estimator<'_>, + ctx: &mut Context, +) -> Option> { + let kids = children(consumer); + if !replaceable(consumer) { + return None; + } + let (next, notes) = (&mut ctx.next, &mut ctx.notes); + for (i, kid) in kids.iter().enumerate() { + let mut found = Vec::new(); + let mut budget = MAX_WALK; + walk(kid, &mut Vec::new(), &mut found, &mut budget); + for cand in found { + if cand + .path + .iter() + .any(|s| ctx.shared.contains(&Arc::as_ptr(&s.node))) + || ctx.shared.contains(&Arc::as_ptr(&cand.join)) + { + // Rebuilding a node other parents read would copy it. + continue; + } + let Some(shape) = Shape::of(&cand, ctx.broadcast_max) else { + continue; + }; + // The consumer must be where the path ends for this payload: a + // node the payload could pass is not the consumer (its parent is). + if passes(consumer, i, &shape.blocked) { + continue; + } + if !cand.path.iter().any(|s| is_non_inner_join(&s.node)) { + continue; + } + // Nothing of the dimension above the join: nothing to move. + if shape.carried(&cand).is_empty() { + continue; + } + if let Err(why) = shape.gates(&cand, est) { + note( + notes, + format!("not applied to the join on [{}]: {why}", shape.keys_text()), + ); + continue; + } + let Some(moved) = shape.moved(&cand, kid, next) else { + note( + notes, + "not applied: the plan has no attribute ids left".to_owned(), + ); + continue; + }; + let out = with_child(consumer, i, moved); + note( + notes, + format!( + "moved the join on [{}] above {} non-inner join(s), payload [{}]", + shape.keys_text(), + cand.path + .iter() + .filter(|s| is_non_inner_join(&s.node)) + .count(), + shape.payload_text(), + ), + ); + return Some(out); + } + } + None +} + +/// Every inner join reachable from `at` through pass-through nodes, with the +/// path above it. `path` holds the nodes from the consumer's child down to +/// `at`'s parent (top first while walking). +fn walk( + at: &Arc, + path: &mut Vec, + found: &mut Vec, + budget: &mut usize, +) { + if *budget == 0 { + return; + } + *budget -= 1; + if let PlanNode::Join { + kind: JoinKind::Inner, + .. + } = &at.node + { + for dim_left in [true, false] { + found.push(Candidate { + path: path.iter().rev().cloned().collect(), + join: Arc::clone(at), + dim_left, + }); + } + } + // Continue down through the children a path may come from (the payload is + // not known yet, so `passes` is checked once it is). + let kids = children(at); + for (i, kid) in kids.iter().enumerate() { + if may_pass(at, i) { + path.push(Step { + node: Arc::clone(at), + child: i, + }); + walk(kid, path, found, budget); + path.pop(); + } + } +} + +/// Whether a path may come up through `node` from child `i`, whatever it +/// carries: the node kinds and join sides of the module docs. +fn may_pass(node: &LogicalPlan, i: usize) -> bool { + match &node.node { + PlanNode::Project { .. } | PlanNode::Filter { .. } => i == 0, + PlanNode::Join { + kind, condition, .. + } => { + let det = match condition { + JoinCondition::On(on) => is_deterministic(on), + _ => false, + }; + det && match kind { + JoinKind::Inner => true, + JoinKind::LeftSemi + | JoinKind::LeftAnti + | JoinKind::LeftOuter + | JoinKind::Existence(_) => i == 0, + JoinKind::RightOuter => i == 1, + _ => false, + } + } + _ => false, + } +} + +/// Whether a path carrying the columns `blocked` (the payload and the +/// dimension's keys) may come up through `node` from child `i`: the node does +/// not read them and its expressions are deterministic. A projection may pass +/// a blocked column through as a bare column. +fn passes(node: &LogicalPlan, i: usize, blocked: &Attrs) -> bool { + if !may_pass(node, i) { + return false; + } + match &node.node { + PlanNode::Project { exprs, .. } => exprs.iter().all(|e| { + is_deterministic(e) && (plain_column(e) || references(e).is_disjoint(blocked)) + }), + PlanNode::Filter { predicate, .. } => { + is_deterministic(predicate) && references(predicate).is_disjoint(blocked) + } + PlanNode::Join { + condition: JoinCondition::On(on), + .. + } => references(on).is_disjoint(blocked), + _ => false, + } +} + +fn is_non_inner_join(p: &LogicalPlan) -> bool { + matches!(&p.node, PlanNode::Join { kind, .. } if *kind != JoinKind::Inner) +} + +/// A candidate's join, read: the equalities, the dimension's columns. +struct Shape { + /// `(x, d)` per equality: the other side's column, the dimension's. + eqs: Vec<(BoundColumn, BoundColumn)>, + /// The conjuncts, as written. + conj: Vec>, + /// The dimension's output columns other than its keys. + payload: Vec, + /// The payload and the dimension's keys. + blocked: Attrs, + /// The broadcast threshold the physical planner uses, `None` when off. + broadcast_max: Option, +} + +impl Shape { + fn of(c: &Candidate, broadcast_max: Option) -> Option { + let PlanNode::Join { + left, + right, + condition: JoinCondition::On(on), + .. + } = &c.join.node + else { + return None; + }; + let (dim, other) = if c.dim_left { + (left, right) + } else { + (right, left) + }; + let dim_cols = dim.resolved_output()?; + let other_cols = other.resolved_output()?; + let in_dim = |id: AttrId| dim_cols.iter().any(|c| c.id == id); + let in_other = |id: AttrId| other_cols.iter().any(|c| c.id == id); + let conj = conjuncts(on); + let mut eqs = Vec::new(); + for e in &conj { + let (op, args, _) = builtin(e)?; + if op != Builtin::Eq || args.len() != 2 { + return None; + } + let (a, b) = (bare_column(&args[0])?, bare_column(&args[1])?); + if a.data_type != b.data_type || !key_type(&a.data_type) { + return None; + } + let pair = if in_other(a.id) && in_dim(b.id) { + (a.clone(), b.clone()) + } else if in_dim(a.id) && in_other(b.id) { + (b.clone(), a.clone()) + } else { + return None; + }; + eqs.push(pair); + } + if eqs.is_empty() || !repeatable(dim) { + return None; + } + let keys: Attrs = eqs.iter().map(|(_, d)| d.id).collect(); + let payload: Vec = dim_cols + .iter() + .filter(|c| !keys.contains(&c.id)) + .cloned() + .collect(); + let mut blocked = keys; + blocked.extend(payload.iter().map(|c| c.id)); + let shape = Self { + eqs, + conj, + payload, + blocked, + broadcast_max, + }; + // Every node on the path must pass what the shape blocks. + c.path + .iter() + .all(|s| passes(&s.node, s.child, &shape.blocked)) + .then_some(shape) + } + + /// The statistics gates, or why the move is not worth it. + fn gates(&self, c: &Candidate, est: &mut Estimator<'_>) -> Result<(), String> { + let carried = self.carried(c); + let width: Option = carried.iter().map(|c| fixed_width(&c.data_type)).sum(); + if width.is_some_and(|w| w < MIN_PAYLOAD_BYTES) { + return Err(format!( + "a payload of {} B a row is under {MIN_PAYLOAD_BYTES} B", + width.unwrap_or(0) + )); + } + let PlanNode::Join { left, right, .. } = &c.join.node else { + return Err("not a join".to_owned()); + }; + let (dim, other) = if c.dim_left { + (left, right) + } else { + (right, left) + }; + let (de, xe) = (est.plan(dim), est.plan(other)); + // Never blind: a default row count says nothing of which side builds. + if !de.from_statistics || !xe.from_statistics { + return Err("the estimates rest on default row counts".to_owned()); + } + let (d, x) = (de.rows, xe.rows); + if d > x { + return Err(format!( + "the dimension estimates more rows ({d:.0}) than the other side ({x:.0})" + )); + } + // Computing D a second time must cost less than the rows it narrows. + let scanned = scanned_rows(dim, est); + if scanned > x { + return Err(format!( + "the dimension scans more rows ({scanned:.0}) than the other side holds ({x:.0})" + )); + } + // A payload on a join's probe side only streams through it; one on + // its build side is held in the hash table. Some join on the path must + // build the path's side, as the physical planner orients it. + if !c + .path + .iter() + .any(|s| builds_path(s, est, self.broadcast_max)) + { + return Err("no join on the path builds it: the payload only streams".to_owned()); + } + Ok(()) + } + + /// The payload columns the path's top still outputs. + fn carried(&self, c: &Candidate) -> Vec { + let top = c.path.last().map_or(&c.join, |s| &s.node); + let out = top.resolved_output().unwrap_or_default(); + self.payload + .iter() + .filter(|p| out.iter().any(|o| o.id == p.id)) + .cloned() + .collect() + } + + /// The path rebuilt over the semi join, the moved join over it, and a + /// projection giving the path's original output; `None` when attribute + /// ids run out or a plan has no resolved output. + fn moved( + &self, + c: &Candidate, + top: &Arc, + next: &mut u64, + ) -> Option> { + let PlanNode::Join { + left, + right, + kind, + condition, + .. + } = &c.join.node + else { + return None; + }; + let (dim, other) = if c.dim_left { + (left, right) + } else { + (right, left) + }; + + // X ⋉ Shift(π_d D) on x = shifted d. + let offset = *next; + let high = max_id(dim, &mut HashMap::new()); + *next = offset + .checked_add(high)? + .checked_add(1) + .filter(|n| *n <= MAX_ID)?; + let mut key_cols: Vec = Vec::new(); + for (_, d) in &self.eqs { + if !key_cols.iter().any(|k| k.id == d.id) { + key_cols.push(d.clone()); + } + } + let keys_only = Arc::new(LogicalPlan::new(PlanNode::Project { + input: Arc::clone(dim), + exprs: key_cols.iter().map(column).collect(), + })); + let shifted = Arc::new(LogicalPlan::new(PlanNode::Shift { + input: keys_only, + offset, + })); + let semi_cond = self + .conj + .iter() + .map(|e| { + let (op, args, f) = builtin(e)?; + let args = args + .iter() + .map(|a| { + if bare_column(a).is_some_and(|b| self.blocked.contains(&b.id)) { + shift(a, offset) + } else { + Arc::clone(a) + } + }) + .collect(); + Some(recall(f, op, args)) + }) + .collect::>>()?; + let semi_cond = crate::expr::and_all(semi_cond)?; + let mut below = Arc::new( + LogicalPlan::new(PlanNode::Join { + left: Arc::clone(other), + right: shifted, + kind: JoinKind::LeftSemi, + condition: JoinCondition::On(semi_cond), + }) + .with_origin(c.join.origin), + ); + + // The path, bottom up, without the dimension's columns and with the + // other side's key columns kept for the moved join. + let xs: Vec = self.eqs.iter().map(|(x, _)| x.clone()).collect(); + for s in &c.path { + below = match &s.node.node { + PlanNode::Project { exprs, .. } => { + let mut kept: Vec> = exprs + .iter() + .filter(|e| output_of(e).is_none_or(|o| !self.blocked.contains(&o.id))) + .cloned() + .collect(); + for x in &xs { + if !kept + .iter() + .any(|e| output_of(e).is_some_and(|o| o.id == x.id)) + { + kept.push(column(x)); + } + } + let mut p = LogicalPlan::clone(&s.node); + if let PlanNode::Project { input, exprs } = &mut p.node { + *input = below; + *exprs = kept; + } + Arc::new(p) + } + _ => with_child(&s.node, s.child, below), + }; + } + + // The moved join, then the path's original output. + let joined = Arc::new( + LogicalPlan::new(PlanNode::Join { + left: below, + right: Arc::clone(dim), + kind: *kind, + condition: condition.clone(), + }) + .with_origin(c.join.origin), + ); + let out = top.resolved_output()?; + Some(Arc::new( + LogicalPlan::new(PlanNode::Project { + input: joined, + exprs: out.iter().map(column).collect(), + }) + .with_origin(top.origin), + )) + } + + fn keys_text(&self) -> String { + let v: Vec = self + .eqs + .iter() + .map(|(x, d)| format!("{} = {}", x.name, d.name)) + .collect(); + v.join(", ") + } + + fn payload_text(&self) -> String { + let v: Vec<&str> = self.payload.iter().map(|c| c.name.as_str()).collect(); + v.join(", ") + } +} + +/// Whether `e` is a column, not renamed. +fn plain_column(e: &Expr) -> bool { + matches!( + &e.node, + krishiv_plan::ExprNode::Resolved(krishiv_plan::ResolvedExpr::Column(_)) + ) +} + +/// The column `e` is, under aliases only. +fn bare_column(e: &Arc) -> Option<&BoundColumn> { + match &crate::expr::strip_alias(e).node { + krishiv_plan::ExprNode::Resolved(krishiv_plan::ResolvedExpr::Column(c)) => Some(c), + _ => None, + } +} + +/// Bytes a row of a fixed-width type; `None` for a variable-width one. +fn fixed_width(t: &DataType) -> Option { + Some(match t { + DataType::Boolean | DataType::Byte => 1, + DataType::Short => 2, + DataType::Integer | DataType::Float | DataType::Date => 4, + DataType::Long | DataType::Double | DataType::Timestamp | DataType::TimestampNtz => 8, + DataType::Decimal(d) if d.precision() <= 18 => 8, + DataType::Decimal(_) => 16, + _ => return None, + }) +} + +fn children(p: &LogicalPlan) -> Vec> { + let mut v = Vec::new(); + p.for_each_child(&mut |c| v.push(Arc::clone(c))); + v +} + +/// `p` with its child `i` replaced (a relation with one input, or a join). +fn with_child(p: &Arc, i: usize, child: Arc) -> Arc { + match &p.node { + PlanNode::Join { .. } => { + let mut j = LogicalPlan::clone(p); + if let PlanNode::Join { left, right, .. } = &mut j.node { + if i == 0 { + *left = child; + } else { + *right = child; + } + } + Arc::new(j) + } + _ => dag::with_input(p, child), + } +} diff --git a/crates/krishiv-optimizer/src/lib.rs b/crates/krishiv-optimizer/src/lib.rs index f9979856e..6ee54389d 100644 --- a/crates/krishiv-optimizer/src/lib.rs +++ b/crates/krishiv-optimizer/src/lib.rs @@ -33,6 +33,10 @@ //! 5. **Tidy**: the rewrite batch again over the pruned plan; prune and //! tidy repeat until neither changes the plan (folding can leave columns //! unread). +//! **Late payload join**, once, with statistics (`late_payload`): a +//! dimension's payload columns join above the semi, anti and outer joins +//! that never read them, a key-only semi join left in place; then join +//! order, eager aggregation, prune and tidy again when it moved a join. //! 6. **CSE** (`cse`, once): common-subexpression elimination within one //! operator; disabled by [`Options::cse`]. //! @@ -56,6 +60,7 @@ mod fold; mod group_limit; mod inline_cte; mod join_order; +mod late_payload; mod limit; mod narrow; mod plan_stats; @@ -71,8 +76,8 @@ mod sum_rewrite; use std::sync::Arc; use krishiv_common::config::{ - ANSI_ENABLED, Config, ConfigValue, EAGER_AGGREGATION, JOIN_DP_LIMIT, JOIN_ENUMERATOR, - SESSION_TIME_ZONE, SHARE_CTE_INSTANCES, + ANSI_ENABLED, AUTO_BROADCAST_JOIN_THRESHOLD, Config, ConfigValue, EAGER_AGGREGATION, + JOIN_DP_LIMIT, JOIN_ENUMERATOR, LATE_PAYLOAD_JOIN, SESSION_TIME_ZONE, SHARE_CTE_INSTANCES, }; use krishiv_common::error::{InternalError, KrishivError}; use krishiv_plan::{ @@ -145,6 +150,10 @@ pub struct Options { /// statistics say it pays. Needs statistics; a session configuration, /// when given, can switch it off. pub eager_aggregation: bool, + /// Late payload joins (`late_payload`): a dimension's payload columns + /// join above the semi and anti joins that never read them. Needs + /// statistics; a session configuration, when given, can switch it off. + pub late_payload_join: bool, } impl Default for Options { @@ -153,6 +162,7 @@ impl Default for Options { cse: true, stats_propagation: true, eager_aggregation: true, + late_payload_join: true, } } } @@ -402,40 +412,48 @@ fn run( }); let root = match stats { Some(stats) if eager => { - let (after, notes) = - eager::rewrite(Arc::clone(&root), stats, options.stats_propagation)?; - t.notes("EagerAggregate", &after, notes); - // The moved aggregate changes what its join's inputs estimate, - // and so their canonical order. + eager_pass(&root, stats, options.stats_propagation, joins, &mut t)? + } + _ => root, + }; + let root = prune_and_tidy(&root, &mut t)?; + // After pruning, so a projection on the path holds only what is read + // above it and the payload is what the path really carries; the moved + // join is then ordered with the dimension's own joins and its path + // pruned and tidied again (the rule finds nothing more to move there). + let late = options.late_payload_join + && udfs.is_none_or(|(_, config)| { + config + .get(&LATE_PAYLOAD_JOIN) + .and_then(ConfigValue::as_bool) + .unwrap_or(true) + }); + let root = match stats { + Some(stats) if late => { + let (after, notes) = late_payload::rewrite( + Arc::clone(&root), + stats, + options.stats_propagation, + broadcast_max(udfs.map(|(_, config)| config)), + )?; + t.notes("LatePayloadJoin", &after, notes); if Arc::ptr_eq(&after, &root) { after } else { - order_and_restrict(after, stats, options.stats_propagation, joins, &mut t)? + let mut ordered = + order_and_restrict(after, stats, options.stats_propagation, joins, &mut t)?; + // The moved join may now sit directly under its aggregate, + // the shape eager aggregation reads (where its gates allow, + // the aggregate then runs by the dimension's key below it). + if eager { + ordered = + eager_pass(&ordered, stats, options.stats_propagation, joins, &mut t)?; + } + prune_and_tidy(&ordered, &mut t)? } } _ => root, }; - let pruned = t.once("Prune", &root, |r| Ok(prune::prune(r)))?; - let mut root = batch(Arc::clone(&pruned), "Tidy", &mut t)?; - // Tidying can fold a filter to a constant `false` and leave columns - // unread (a `false` that reached a filter on `b` frees `b`; constants - // pass grouped aggregates), so after a tidy that changed the plan and - // left such a filter, prune and tidy again until nothing changes: - // then the plan is a fixed point. Other plans (no `Filter false`, or - // a tidy that changed nothing) take no extra pass. - let mut changed = !Arc::ptr_eq(&root, &pruned) && has_false_filter(&root); - let mut round = 2; - while changed && round <= MAX_ITERATIONS { - let pruned = prune::prune(&root); - t.step("Prune", round, "Prune", &root, &pruned); - if Arc::ptr_eq(&pruned, &root) { - break; - } - let tidied = batch(Arc::clone(&pruned), "Tidy", &mut t)?; - changed = !Arc::ptr_eq(&tidied, &pruned); - root = tidied; - round += 1; - } // Common-subexpression elimination runs once after the final // rewrite batch: the conjunct order of filters above is then the // order that executes, so "first operand" is execution order. @@ -491,6 +509,19 @@ fn run( /// The join-ordering settings of a session configuration (defaults /// without one), describing each tree when `trace`. +/// The physical planner's broadcast threshold (`krishiv-exec` +/// `planner/join.rs` `broadcast_max_bytes`): the session's +/// `spark.sql.autoBroadcastJoinThreshold`, 10 MiB without a session, `None` +/// when negative (off). +fn broadcast_max(config: Option<&Config>) -> Option { + let max = config + .and_then(|c| c.get(&AUTO_BROADCAST_JOIN_THRESHOLD)) + .and_then(ConfigValue::as_int) + .unwrap_or(10 << 20); + #[expect(clippy::cast_precision_loss, reason = "compared with an estimate")] + (max >= 0).then_some(max as f64) +} + fn join_settings(config: Option<&Config>, trace: bool) -> join_order::Settings { let mut s = join_order::Settings { trace, @@ -509,6 +540,55 @@ fn join_settings(config: Option<&Config>, trace: bool) -> join_order::Settings { s } +/// Column pruning, then the rewrite batch over the pruned plan, repeated +/// while tidying leaves a constant `false` filter and changes the plan. +fn prune_and_tidy( + root: &Arc, + t: &mut Trace, +) -> Result, KrishivError> { + let pruned = t.once("Prune", root, |r| Ok(prune::prune(r)))?; + let mut root = batch(Arc::clone(&pruned), "Tidy", t)?; + // Tidying can fold a filter to a constant `false` and leave columns + // unread (a `false` that reached a filter on `b` frees `b`; constants + // pass grouped aggregates), so after a tidy that changed the plan and + // left such a filter, prune and tidy again until nothing changes: + // then the plan is a fixed point. Other plans (no `Filter false`, or + // a tidy that changed nothing) take no extra pass. + let mut changed = !Arc::ptr_eq(&root, &pruned) && has_false_filter(&root); + let mut round = 2; + while changed && round <= MAX_ITERATIONS { + let pruned = prune::prune(&root); + t.step("Prune", round, "Prune", &root, &pruned); + if Arc::ptr_eq(&pruned, &root) { + break; + } + let tidied = batch(Arc::clone(&pruned), "Tidy", t)?; + changed = !Arc::ptr_eq(&tidied, &pruned); + root = tidied; + round += 1; + } + Ok(root) +} + +/// Eager aggregation, then join ordering again when it moved an aggregate. +fn eager_pass( + root: &Arc, + stats: &dyn Statistics, + propagate: bool, + joins: join_order::Settings, + t: &mut Trace, +) -> Result, KrishivError> { + let (after, notes) = eager::rewrite(Arc::clone(root), stats, propagate)?; + t.notes("EagerAggregate", &after, notes); + // The moved aggregate changes what its join's inputs estimate, and so + // their canonical order. + if Arc::ptr_eq(&after, root) { + Ok(after) + } else { + order_and_restrict(after, stats, propagate, joins, t) + } +} + /// Join ordering alternating with restriction, until neither changes the /// plan. The trees of the last ordering, the plan's own, are described in /// the trace. diff --git a/crates/krishiv-optimizer/src/plan_stats.rs b/crates/krishiv-optimizer/src/plan_stats.rs index e2dc45745..502fc1037 100644 --- a/crates/krishiv-optimizer/src/plan_stats.rs +++ b/crates/krishiv-optimizer/src/plan_stats.rs @@ -170,7 +170,7 @@ fn subquery_plans<'a>(p: &'a LogicalPlan, f: &mut impl FnMut(&'a Arc u64 { +pub(crate) fn default_size(t: &DataType) -> u64 { match t { DataType::Null | DataType::Boolean | DataType::Byte => 1, DataType::Short => 2, diff --git a/crates/krishiv-optimizer/src/restrict.rs b/crates/krishiv-optimizer/src/restrict.rs index a060cd338..ce8954df6 100644 --- a/crates/krishiv-optimizer/src/restrict.rs +++ b/crates/krishiv-optimizer/src/restrict.rs @@ -639,7 +639,7 @@ fn cannot_raise(e: &Arc) -> bool { } /// Types whose grouping and `=` agree value for value. -fn key_type(t: &DataType) -> bool { +pub(crate) fn key_type(t: &DataType) -> bool { matches!( t, DataType::Boolean @@ -676,7 +676,7 @@ fn node_exprs_deterministic(p: &LogicalPlan) -> bool { /// function of their inputs (no limit or sample; no CTE reference, which a /// shift does not renumber), deterministic expressions (subquery plans /// included) and no outer references. -fn repeatable(p: &Arc) -> bool { +pub(crate) fn repeatable(p: &Arc) -> bool { let mut seen = HashSet::new(); let mut stack = vec![Arc::clone(p)]; while let Some(x) = stack.pop() { @@ -703,7 +703,7 @@ fn repeatable(p: &Arc) -> bool { /// `e` with every column id raised by `offset` (no subqueries inside: the /// restricting equalities never hold one). -fn shift(e: &Arc, offset: u64) -> Arc { +pub(crate) fn shift(e: &Arc, offset: u64) -> Arc { let out = Arc::clone(e).transform_up(&mut |x| { Ok::<_, std::convert::Infallible>(match &x.node { ExprNode::Resolved(ResolvedExpr::Column(c)) => { diff --git a/crates/krishiv-optimizer/tests/it/late_payload.rs b/crates/krishiv-optimizer/tests/it/late_payload.rs new file mode 100644 index 000000000..2476cd9a2 --- /dev/null +++ b/crates/krishiv-optimizer/tests/it/late_payload.rs @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `LatePayloadJoin` (P2b S5 lever (2)): where a dimension's payload joins +//! above the semi and anti joins that never read it, where it stays, and that +//! the rows never change (any key multiplicity, NULL keys) and the plan is a +//! fixed point. + +use std::collections::HashMap; + +use krishiv_optimizer::{ + NoStatistics, Options, Statistics, TableStats, optimize_traced_with_options, + optimize_with_options, +}; +use krishiv_plan::{ReadSource, Scan}; + +use crate::join_order::fixture; +use crate::support::{self, data, eval}; +use crate::tpcds; + +/// The optimized plan, its text and the rule's notes, with the rule on or off. +fn run_with( + sql: &str, + stats: &dyn Statistics, + on: bool, +) -> (krishiv_plan::Plan, String, Vec) { + let plan = support::resolve(sql, &support::ctx()); + let options = Options { + late_payload_join: on, + ..Options::default() + }; + let (optimized, steps) = optimize_traced_with_options(&plan, stats, &options).unwrap(); + let notes = steps + .into_iter() + .filter(|s| s.batch == "LatePayloadJoin") + .map(|s| s.note) + .collect(); + let text = optimized.root().to_string(); + (optimized, text, notes) +} + +fn run(sql: &str, on: bool) -> (krishiv_plan::Plan, String, Vec) { + run_with(sql, &fixture(), on) +} + +/// The rule's result returns the unoptimised plan's rows, keeps its output and +/// is a fixed point. +fn check(sql: &str) -> (String, Vec) { + let plan = support::resolve(sql, &support::ctx()); + let (optimized, text, notes) = run(sql, true); + assert_eq!( + optimized.root().resolved_output(), + plan.root().resolved_output(), + "{text}" + ); + let again = optimize_with_options(&optimized, &fixture(), &Options::default()).unwrap(); + assert!( + again.structurally_eq(&optimized), + "not a fixed point:\n{}", + again.root() + ); + let db = data::tpch(7, 150); + assert_eq!( + eval::rows(&db, optimized.root()), + eval::rows(&db, plan.root()), + "{text}" + ); + (text, notes) +} + +fn moved(notes: &[String]) -> Vec<&String> { + notes.iter().filter(|n| n.starts_with("moved")).collect() +} + +/// TPC-H q21's `s_name` joins above the `EXISTS` and `NOT EXISTS` joins, and +/// `lineitem` keeps the Saudi suppliers' reduction through a key-only semi +/// join; q18's `c_name` leaves the `orders` build of the join with +/// `lineitem`. No other TPC-H query changes. +#[test] +fn q21_and_q18_payloads_join_above_their_semi_and_anti_joins() { + for q in 1..=22 { + let (text, notes) = check(&support::tpch_sql(q)); + match q { + 21 => { + assert_eq!( + moved(¬es), + [ + "moved the join on [l_suppkey = s_suppkey] above 2 non-inner join(s), payload [s_name]" + ], + "{notes:?}" + ); + let anti = text.find("Join LeftAnti").unwrap(); + let semi_on_supplier = text.find("Join LeftSemi, (s_suppkey#").unwrap(); + assert!(anti < semi_on_supplier, "{text}"); + // No build below the anti join carries the name. + assert!(!text[anti..].contains("s_name#1,"), "{text}"); + } + 18 => assert_eq!( + moved(¬es), + [ + "moved the join on [o_custkey = c_custkey] above 1 non-inner join(s), payload [c_name]" + ], + "{notes:?}" + ), + _ => assert!(moved(¬es).is_empty(), "q{q:02}: {notes:?}"), + } + } +} + +/// A dimension key that repeats: each supplier meets the five nations of its +/// region, and every one of them still counts it. NULL keys meet nothing, +/// before and after. +#[test] +fn repeating_and_null_dimension_keys_keep_every_joined_row() { + for sql in [ + // EXISTS over lineitem: a semi join building the suppliers. + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name", + // NOT EXISTS: an anti join. + "SELECT n_name, count(*), sum(s_acctbal) FROM supplier, nation \ + WHERE s_nationkey = n_regionkey \ + AND NOT EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey AND l_quantity > 49) \ + GROUP BY n_name", + // A payload read by a projection, not an aggregate. + "SELECT s_suppkey, upper(n_name) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey)", + // A key that is NULL for a third of the suppliers. + "SELECT n_name, count(*) FROM (SELECT s_suppkey, \ + CASE WHEN s_suppkey % 3 = 0 THEN NULL ELSE s_nationkey END AS nk FROM supplier) s, nation \ + WHERE nk = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name", + ] { + let (text, notes) = check(sql); + assert_eq!(moved(¬es).len(), 1, "{sql}\n{notes:?}\n{text}"); + } +} + +/// Two dimensions under one consumer both move in one pass. +#[test] +fn two_dimensions_under_one_consumer_both_move() { + let sql = "SELECT p_name, s_name, count(*) FROM partsupp, part, supplier \ + WHERE ps_partkey = p_partkey AND ps_suppkey = s_suppkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_partkey = ps_partkey \ + AND l_suppkey = ps_suppkey) GROUP BY p_name, s_name"; + let (text, notes) = check(sql); + assert_eq!(moved(¬es).len(), 2, "{notes:?}\n{text}"); +} + +/// A left outer join with the dimension's side preserved passes it too. +#[test] +fn an_outer_join_on_the_preserved_side_passes_the_payload() { + let sql = "SELECT n_name, count(l_orderkey) FROM (SELECT * FROM supplier, nation \ + WHERE s_nationkey = n_regionkey) sn \ + LEFT JOIN lineitem ON l_suppkey = s_suppkey AND l_quantity > 49 GROUP BY n_name"; + let (text, notes) = check(sql); + assert_eq!(moved(¬es).len(), 1, "{notes:?}\n{text}"); +} + +/// Where the move cannot help or is not exact, the plan stays as it was, with +/// the gate that said so where one did. +#[test] +fn the_rule_stays_out_where_it_does_not_apply() { + for (sql, why) in [ + // The semi join reads the payload. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey AND l_comment <> n_name) \ + GROUP BY n_name", + None, + ), + // The semi join reads the dimension's key. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey \ + AND l_linenumber <> n_regionkey) GROUP BY n_name", + None, + ), + // A dimension that is not the same rows twice. + ( + "SELECT n_name, count(*) FROM supplier, (SELECT * FROM nation WHERE rand() < 0.5) n \ + WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_name", + None, + ), + // Eight bytes of payload. + ( + "SELECT n_nationkey, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) GROUP BY n_nationkey", + Some("a payload of 8 B a row is under 16 B"), + ), + // The dimension is the larger side. + ( + "SELECT s_name, count(*) FROM nation, supplier WHERE s_nationkey = n_nationkey \ + AND EXISTS (SELECT * FROM lineitem WHERE l_suppkey = n_regionkey) GROUP BY s_name", + Some("the dimension estimates more rows"), + ), + // The semi join's other side is small: it is broadcast and the path + // only streams through it. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND EXISTS (SELECT * FROM region WHERE r_regionkey = s_nationkey) GROUP BY n_name", + Some("no join on the path builds it"), + ), + // An existence join (EXISTS under OR) never builds its left input. + ( + "SELECT n_name, count(*) FROM supplier, nation WHERE s_nationkey = n_regionkey \ + AND (EXISTS (SELECT * FROM lineitem WHERE l_suppkey = s_suppkey) OR s_acctbal > 0) \ + GROUP BY n_name", + Some("no join on the path builds it"), + ), + // A right outer join with the path on its right builds its left input. + ( + "SELECT n_name, count(l_orderkey) FROM lineitem RIGHT JOIN (SELECT * FROM supplier, \ + nation WHERE s_nationkey = n_regionkey) sn ON l_suppkey = s_suppkey \ + AND l_quantity > 49 GROUP BY n_name", + Some("no join on the path builds it"), + ), + // Only inner joins: join ordering's business. + ( + "SELECT n_name, count(*) FROM supplier, nation, lineitem \ + WHERE s_nationkey = n_nationkey AND l_suppkey = s_suppkey GROUP BY n_name", + None, + ), + ] { + let (_, text, notes) = run(sql, true); + let (_, off, _) = run(sql, false); + assert_eq!(text, off, "{sql}\n{notes:?}"); + assert!(moved(¬es).is_empty(), "{sql}\n{notes:?}"); + if let Some(why) = why { + assert!(notes.iter().any(|n| n.contains(why)), "{sql}\n{notes:?}"); + } + if !sql.contains("rand()") { + check(sql); + } + } +} + +/// Default row counts never move a join. +#[test] +fn default_row_counts_move_nothing() { + let (_, _, notes) = run_with(&support::tpch_sql(21), &NoStatistics, true); + assert!(moved(¬es).is_empty(), "{notes:?}"); + assert!( + notes + .iter() + .any(|n| n.contains("rest on default row counts")), + "{notes:?}" + ); +} + +/// Switched off, by option or by the session key, nothing moves. +#[test] +fn the_rule_can_be_switched_off() { + let sql = support::tpch_sql(21); + let (_, on, _) = run(&sql, true); + let (_, off, notes) = run(&sql, false); + assert_ne!(on, off); + assert!(notes.is_empty(), "{notes:?}"); + + let plan = support::resolve(&sql, &support::ctx()); + let library = krishiv_udf_sdk::UdfLibrary::new(); + let mut config = krishiv_common::config::Config::new(); + let with = krishiv_optimizer::optimize_with_udfs(&plan, &fixture(), &library, &config).unwrap(); + assert_eq!(with.root().to_string(), on); + config + .set("krishiv.optimizer.latePayloadJoin", "false") + .unwrap(); + let without = + krishiv_optimizer::optimize_with_udfs(&plan, &fixture(), &library, &config).unwrap(); + assert_eq!(without.root().to_string(), off); +} + +/// TPC-DS SF1 row counts (no column statistics): enough for the rule's +/// `from_statistics` gate. +#[derive(Debug)] +struct TpcdsRows; + +const TPCDS_SF1_ROWS: [(&str, u64); 24] = [ + ("call_center", 6), + ("catalog_page", 11_718), + ("catalog_returns", 144_067), + ("catalog_sales", 1_441_548), + ("customer", 100_000), + ("customer_address", 50_000), + ("customer_demographics", 1_920_800), + ("date_dim", 73_049), + ("household_demographics", 7_200), + ("income_band", 20), + ("inventory", 11_745_000), + ("item", 18_000), + ("promotion", 300), + ("reason", 35), + ("ship_mode", 20), + ("store", 12), + ("store_returns", 287_514), + ("store_sales", 2_880_404), + ("time_dim", 86_400), + ("warehouse", 5), + ("web_page", 60), + ("web_returns", 71_763), + ("web_sales", 719_384), + ("web_site", 30), +]; + +impl Statistics for TpcdsRows { + fn table(&self, scan: &Scan) -> Option { + let ReadSource::Table { name, .. } = &scan.source else { + return None; + }; + let table = name.last()?; + let (_, rows) = TPCDS_SF1_ROWS.iter().find(|t| t.0 == table)?; + Some(TableStats { + rows: *rows, + columns: HashMap::new(), + }) + } +} + +/// Over TPC-DS with row counts, every query keeps its output and is a fixed +/// point (q35 moves two dimensions under one consumer); the queries the rule +/// changes are listed, so a change to the list is a reviewed decision. +#[test] +fn tpcds_with_row_counts_is_a_fixed_point() { + let ctx = support::ctx_over(tpcds::catalog()); + let mut changed = Vec::new(); + for q in 1..=99 { + let Some(plan) = support::try_resolve(&tpcds::sql(q), &ctx) else { + continue; + }; + let (optimized, steps) = + optimize_traced_with_options(&plan, &TpcdsRows, &Options::default()).unwrap(); + assert_eq!( + optimized.root().resolved_output(), + plan.root().resolved_output(), + "q{q:02}" + ); + let again = optimize_with_options(&optimized, &TpcdsRows, &Options::default()).unwrap(); + assert!( + again.structurally_eq(&optimized), + "q{q:02} is not a fixed point:\n{}", + again.root() + ); + let moves = steps + .iter() + .filter(|s| s.batch == "LatePayloadJoin" && s.note.starts_with("moved")) + .count(); + if moves > 0 { + changed.push((q, moves)); + } + } + insta::assert_debug_snapshot!(changed); +} diff --git a/crates/krishiv-optimizer/tests/it/main.rs b/crates/krishiv-optimizer/tests/it/main.rs index 63494d02b..f8667ff8e 100644 --- a/crates/krishiv-optimizer/tests/it/main.rs +++ b/crates/krishiv-optimizer/tests/it/main.rs @@ -11,6 +11,7 @@ mod cse; mod join_enumerator; mod join_order; +mod late_payload; mod plan_stats; mod props; mod qerror; diff --git a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap index 4126b23e4..b1121d050 100644 --- a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap +++ b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q18.snap @@ -5,16 +5,20 @@ expression: optimized.root().to_string() Limit 100 +- Sort [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST], true +- Aggregate [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21) AS sum(l_quantity)#50] - +- Join LeftSemi, (o_orderkey#8 == l_orderkey#33) - :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- Join Inner, (o_orderkey#8 == l_orderkey#17) - : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (shared #1) - : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- Join Inner, (c_custkey#0 == o_custkey#9) - : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] - : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] - +- Project [l_orderkey#33] - +- Filter (sum(l_quantity)#49 > 300.00) - +- Aggregate [l_orderkey#33], [l_orderkey#33, sum(l_quantity#37) AS sum(l_quantity)#49] - +- Shift +16 - +- Shared #1, printed above + +- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] + +- Join Inner, (c_custkey#0 == o_custkey#9) + :- Join LeftSemi, (o_orderkey#8 == l_orderkey#33) + : :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, o_custkey#9] + : : +- Join Inner, (o_orderkey#8 == l_orderkey#17) + : : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (shared #1) + : : +- Join LeftSemi, (c_custkey#51 == o_custkey#9) + : : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] + : : +- Shift +51 + : : +- Project [c_custkey#0] + : : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] (shared #2) + : +- Project [l_orderkey#33] + : +- Filter (sum(l_quantity)#49 > 300.00) + : +- Aggregate [l_orderkey#33], [l_orderkey#33, sum(l_quantity#37) AS sum(l_quantity)#49] + : +- Shift +16 + : +- Shared #1, printed above + +- Shared #2, printed above diff --git a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap index 6a8398abd..a07dd0f51 100644 --- a/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap +++ b/crates/krishiv-optimizer/tests/it/snapshots/it__join_order__q21.snap @@ -6,28 +6,37 @@ Limit 100 +- Sort [numwait#68 DESC NULLS LAST, s_name#1 ASC NULLS FIRST], true +- Aggregate [s_name#1], [s_name#1, count(1) AS numwait#68] +- Project [s_name#1] - +- Join LeftAnti, ((l_orderkey#52 == l_orderkey#7) and (NOT (l_suppkey#54 == l_suppkey#9))) - :- Join LeftSemi, ((l_orderkey#36 == l_orderkey#7) and (NOT (l_suppkey#38 == l_suppkey#9))) - : :- Project [l_orderkey#7, l_suppkey#9, s_name#1] - : : +- Join Inner, (o_orderkey#23 == l_orderkey#7) - : : :- Project [o_orderkey#23] - : : : +- Filter (o_orderstatus#25 == 'F') - : : : +- Scan spark_catalog.default.orders [o_orderkey#23, o_orderstatus#25] - : : +- Project [l_orderkey#7, l_suppkey#9, s_name#1] - : : +- Join Inner, (s_suppkey#0 == l_suppkey#9) - : : :- Project [l_orderkey#7, l_suppkey#9] - : : : +- Filter (l_receiptdate#19 > l_commitdate#18) - : : : +- Scan spark_catalog.default.lineitem [l_orderkey#7, l_suppkey#9, l_commitdate#18, l_receiptdate#19] (shared #1) - : : +- Project [s_suppkey#0, s_name#1] - : : +- Join Inner, (s_nationkey#3 == n_nationkey#32) - : : :- Scan spark_catalog.default.supplier [s_suppkey#0, s_name#1, s_nationkey#3] - : : +- Project [n_nationkey#32] - : : +- Filter (n_name#33 == 'SAUDI ARABIA') - : : +- Scan spark_catalog.default.nation [n_nationkey#32, n_name#33] - : +- Project [l_orderkey#36, l_suppkey#38] - : +- Shift +29 - : +- Shared #1, printed above - +- Project [l_orderkey#52, l_suppkey#54] - +- Shift +45 - +- Filter (l_receiptdate#19 > l_commitdate#18) - +- Shared #1, printed above + +- Join Inner, (s_nationkey#3 == n_nationkey#32) + :- Project [s_name#1, s_nationkey#3] + : +- Join Inner, (s_suppkey#0 == l_suppkey#9) + : :- Scan spark_catalog.default.supplier [s_suppkey#0, s_name#1, s_nationkey#3] (shared #1) + : +- Project [l_suppkey#9] + : +- Join LeftAnti, ((l_orderkey#52 == l_orderkey#7) and (NOT (l_suppkey#54 == l_suppkey#9))) + : :- Join LeftSemi, ((l_orderkey#36 == l_orderkey#7) and (NOT (l_suppkey#38 == l_suppkey#9))) + : : :- Project [l_orderkey#7, l_suppkey#9] + : : : +- Join Inner, (o_orderkey#23 == l_orderkey#7) + : : : :- Project [o_orderkey#23] + : : : : +- Filter (o_orderstatus#25 == 'F') + : : : : +- Scan spark_catalog.default.orders [o_orderkey#23, o_orderstatus#25] + : : : +- Join LeftSemi, (s_suppkey#69 == l_suppkey#9) + : : : :- Project [l_orderkey#7, l_suppkey#9] + : : : : +- Filter (l_receiptdate#19 > l_commitdate#18) + : : : : +- Scan spark_catalog.default.lineitem [l_orderkey#7, l_suppkey#9, l_commitdate#18, l_receiptdate#19] (shared #2) + : : : +- Shift +69 + : : : +- Project [s_suppkey#0] + : : : +- Join Inner, (s_nationkey#3 == n_nationkey#32) + : : : :- Project [s_suppkey#0, s_nationkey#3] + : : : : +- Shared #1, printed above + : : : +- Project [n_nationkey#32] + : : : +- Filter (n_name#33 == 'SAUDI ARABIA') + : : : +- Scan spark_catalog.default.nation [n_nationkey#32, n_name#33] (shared #3) + : : +- Project [l_orderkey#36, l_suppkey#38] + : : +- Shift +29 + : : +- Shared #2, printed above + : +- Project [l_orderkey#52, l_suppkey#54] + : +- Shift +45 + : +- Filter (l_receiptdate#19 > l_commitdate#18) + : +- Shared #2, printed above + +- Project [n_nationkey#32] + +- Filter (n_name#33 == 'SAUDI ARABIA') + +- Shared #3, printed above diff --git a/crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap b/crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap new file mode 100644 index 000000000..bb783622b --- /dev/null +++ b/crates/krishiv-optimizer/tests/it/snapshots/it__late_payload__tpcds_with_row_counts_is_a_fixed_point.snap @@ -0,0 +1,22 @@ +--- +source: crates/krishiv-optimizer/tests/it/late_payload.rs +expression: changed +--- +[ + ( + 35, + 1, + ), + ( + 58, + 3, + ), + ( + 72, + 1, + ), + ( + 83, + 3, + ), +] diff --git a/docs/reference/config/index.md b/docs/reference/config/index.md index b4de07a75..66940f589 100644 --- a/docs/reference/config/index.md +++ b/docs/reference/config/index.md @@ -58,6 +58,7 @@ Keys under `krishiv.experimental.` are experimental and off by default. Spark al | `krishiv.optimizer.eagerAggregation` | bool | `true` | session | user | 0.1.0 | | Run an aggregate's calls below a join and merge their states above it, when statistics show the join keeps most rows and the join keys group them well. Only calls whose result cannot depend on row order or raise on a row the join would drop are moved. Results are the same either way. | | `krishiv.optimizer.joinDpLimit` | positive int | `10` | session | user | 0.1.0 | | Most inputs of an inner-join tree the `dpsub` join enumerator plans exactly; larger trees are planned greedily. Values above 14 count as 14: the work grows as 3 to the power of the inputs. Results are the same either way. | | `krishiv.optimizer.joinEnumerator` | enum (`dpsub`, `greedy`) | `dpsub` | session | user | 0.1.0 | | How the join optimizer orders each tree of inner joins. `dpsub`: dynamic programming over every split of every subset of the inputs, for trees of at most `krishiv.optimizer.joinDpLimit` inputs, greedy above. `greedy`: always join the connected pair with the fewest result rows. Results are the same either way; for comparing plans and planning time. | +| `krishiv.optimizer.latePayloadJoin` | bool | `true` | session | user | 0.1.0 | | Join a dimension's payload columns (such as a name read only by the aggregate on top) after the semi, anti and outer joins that never read them, and leave a semi join on the dimension's keys where the join was, so those joins build and probe narrower rows. Applies when statistics show the payload is wide and the dimension is the smaller side. Results are the same either way. | | `krishiv.optimizer.ndvSampling.enabled` | bool | `true` | session | user | 0.1.0 | | Before a query is optimized, count the distinct values of the file columns an equality or IN compares with values, when the files store no distinct count, from the first rows of the table's files. The count steers join ordering and build-side choice; results are the same either way. Samples are kept per file version. | | `krishiv.optimizer.ndvSampling.rows` | positive int | `131072` | session | user | 0.1.0 | | The most rows a distinct-value sample of a table reads, from its first files in order. A table no larger than this is counted exactly. | | `krishiv.optimizer.ndvSampling.timeout` | duration | `2s` | session | user | 0.1.0 | | The time a distinct-value sample of one table may take; past it the table keeps its default estimates. The row limit is what bounds a sample; this only guards against slow storage, so that how fast the machine is does not decide the plan. | From a336ce5cb796f4ec2005682c029f78b02b8d0c72 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:52:09 +0000 Subject: [PATCH 17/25] optimizer, session: LatePayloadJoin re-check nits: every move gets its note, join_settings keeps its doc; q18's EXPLAIN ANALYZE snapshot Refusal notes are still written once; two moves with the same keys each get their line. The q18 EXPLAIN ANALYZE snapshot (session explain_analyze_tpch) takes the plan the rule gives it (c_name joins above the IN semi join). Review: krishiv-internal ops/reviews/2026-11-09-opus-1-late-payload-diff.md (re-check, Reviewer: Opus, approve with nits). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-optimizer/src/late_payload.rs | 6 +- crates/krishiv-optimizer/src/lib.rs | 4 +- .../explain_analyze_tpch__analyze_q18.snap | 105 ++++++++++-------- 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/crates/krishiv-optimizer/src/late_payload.rs b/crates/krishiv-optimizer/src/late_payload.rs index da4a94610..0489ebfa8 100644 --- a/crates/krishiv-optimizer/src/late_payload.rs +++ b/crates/krishiv-optimizer/src/late_payload.rs @@ -120,10 +120,10 @@ struct Context { broadcast_max: Option, } -/// Appends `line` unless it is there already (the moves below one consumer -/// walk the same refused candidates again). +/// Appends `line`; a refusal only once (the moves below one consumer walk +/// the same refused candidates again), every move. fn note(notes: &mut Vec, line: String) { - if !notes.contains(&line) { + if line.starts_with("moved") || !notes.contains(&line) { notes.push(line); } } diff --git a/crates/krishiv-optimizer/src/lib.rs b/crates/krishiv-optimizer/src/lib.rs index 6ee54389d..b2fbf1312 100644 --- a/crates/krishiv-optimizer/src/lib.rs +++ b/crates/krishiv-optimizer/src/lib.rs @@ -507,8 +507,6 @@ fn run( } } -/// The join-ordering settings of a session configuration (defaults -/// without one), describing each tree when `trace`. /// The physical planner's broadcast threshold (`krishiv-exec` /// `planner/join.rs` `broadcast_max_bytes`): the session's /// `spark.sql.autoBroadcastJoinThreshold`, 10 MiB without a session, `None` @@ -522,6 +520,8 @@ fn broadcast_max(config: Option<&Config>) -> Option { (max >= 0).then_some(max as f64) } +/// The join-ordering settings of a session configuration (defaults +/// without one), describing each tree when `trace`. fn join_settings(config: Option<&Config>, trace: bool) -> join_order::Settings { let mut s = join_order::Settings { trace, diff --git a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap index c891dbd34..0792b1117 100644 --- a/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap +++ b/crates/krishiv-session/tests/snapshots/explain_analyze_tpch__analyze_q18.snap @@ -3,66 +3,75 @@ source: crates/krishiv-session/tests/explain_analyze_tpch.rs expression: text --- == Physical Plan (analyzed) == -wall , 8 pipelines, drivers, threads (most in one pipeline), 1 worker +wall , 9 pipelines, drivers, threads (most in one pipeline), 1 worker memory: peak reserved , limit hash probe: % of driver busy time TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] : · sink #1 TopN p1: in 5 rows (1 batch); compute (max , drivers); peak mem -: · source #24 TopN p0: out 5 rows (1 batch); compute (max , drivers); io (max ) +: · source #28 TopN p0: out 5 rows (1 batch); compute (max , drivers); io (max ) +- Exchange SinglePartition +- LocalTopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] : · sink #2 TopN p2: in 5 rows (1 batch); compute (max , drivers); peak mem - : · source #23 TopN p1: out 5 rows (1 batch); compute (max , drivers); io (max ) + : · source #27 TopN p1: out 5 rows (1 batch); compute (max , drivers); io (max ) +- Project [c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11, sum(l_quantity#21)#4611686018427387907 AS sum(l_quantity)#50] - : · operator #22 Project p2: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) + : · operator #26 Project p2: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) +- HashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] - : · source #21 HashAggregate p2: out 5 rows (1 batch); compute (max , drivers); io (max ) + : · source #25 HashAggregate p2: out 5 rows (1 batch); compute (max , drivers); io (max ) +- Exchange hashpartitioning(c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11) +- PartialHashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] : · sink #3 HashAggregate p3: in 20 rows (1 batch); compute (max , drivers); peak mem ; groups=5 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=5 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= - +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey - : · sink #4 HashJoinBuild p4: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain - : · operator #20 HashJoinProbe p3: in 160 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= - :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : : · operator #19 Project p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers) - : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey - : : · sink #11 HashJoinBuild p6: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain - : : · operator #18 HashJoinProbe p3: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= - : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] - : : · source #17 MemoryScan p3: out 160 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [l_orderkey]: range, tested 160, dropped 0 - : +- BroadcastExchange - : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : : · operator #16 Project p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) - : +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey - : : · sink #12 HashJoinBuild p7: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain - : : · operator #15 HashJoinProbe p6: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= - : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] - : : · source #14 MemoryScan p6: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [o_custkey]: range, tested 40, dropped 0, #3 [o_orderkey]: small set of 5, tested 0, dropped 0 - : +- BroadcastExchange - : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] - : · source #13 MemoryScan p7: out 20 rows (1 batch); compute (max , drivers); io (max ) - +- BroadcastExchange - +- Project [l_orderkey#33] - : · operator #10 Project p4: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) - +- Filter (sum(l_quantity)#49 > 300.00) - : · operator #9 Filter p4: in 40 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) - +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387904 AS sum(l_quantity)#49] - : · operator #8 Project p4: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) - +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - : · source #7 HashAggregate p4: out 40 rows (1 batch); compute (max , drivers); io (max ) - +- Exchange hashpartitioning(l_orderkey#33) - +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - : · sink #5 HashAggregate p5: in 160 rows (1 batch); compute (max , drivers); peak mem ; groups=40 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=40 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= - +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (ids +16) - · source #6 MemoryScan p5: out 160 rows (1 batch); compute (max , drivers); io (max ) + +- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] + : · operator #24 Project p3: in 20 rows (1 batch); out 20 rows (1 batch); compute (max , drivers) + +- HashJoin Inner, [c_custkey#0] = [o_custkey#9], RuntimeFilter #4: c_custkey#0 -> spark_catalog.default.customer.c_custkey + : · sink #4 HashJoinBuild p4: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · operator #23 HashJoinProbe p3: in 5 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=5 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + :- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] + : · source #22 MemoryScan p3: out 5 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=20 runtime_filter_rows_dropped=15 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#4 [c_custkey]: small set of 5, tested 20, dropped 15 + +- BroadcastExchange + +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey + : · sink #5 HashJoinBuild p5: in 5 rows (1 batch); compute (max , drivers); peak mem ; build_rows=5 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=5 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : · operator #21 HashJoinProbe p4: in 160 rows (1 batch); out 20 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, o_custkey#9] + : : · operator #20 Project p4: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers) + : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey + : : · sink #12 HashJoinBuild p7: in 40 rows (1 batch); compute (max , drivers); peak mem ; build_rows=40 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=40 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · operator #19 HashJoinProbe p4: in 160 rows (1 batch); out 160 rows (1 batch); compute (max , drivers); probe_rows=160 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] + : : · source #18 MemoryScan p4: out 160 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=160 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#2 [l_orderkey]: range, tested 160, dropped 0 + : +- BroadcastExchange + : +- HashJoin LeftSemi, [o_custkey#9] = [c_custkey#51], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey + : : · sink #13 HashJoinBuild p8: in 20 rows (1 batch); compute (max , drivers); peak mem ; build_rows=20 steady_reserved_bytes= arena_bytes= arena_hash_bytes= arena_key_bytes= index_bytes= max_finalize_copy_bytes= max_coalesce_copy_bytes= hash_entries=20 hash_buckets= spilled_partitions=0 spill_file_bytes= finalize_ns= finalize_concat_ns= finalize_index_ns= finalize_max_partition_ns= hash_index=row-flat+chain + : : · operator #17 HashJoinProbe p7: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers); probe_rows=40 probe_spilled_rows=0 spill_file_bytes= grace_partitions=0 repartitions=0 recursion_depth=0 grace_hash_entries=0 grace_hash_buckets= reversed_partitions=0 probe_table_ns= probe_lookups= probe_key_compares= max_probe_length= + : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] + : : · source #16 MemoryScan p7: out 40 rows (1 batch); compute (max , drivers); io (max ); runtime_filter_rows_tested=40 runtime_filter_rows_dropped=0 runtime_filter_units_pruned=0 runtime_filter_bytes_pruned=0 runtime_filter_splits_skipped=0 runtime_filters=#1 [o_custkey]: range, tested 40, dropped 0, #3 [o_orderkey]: small set of 5, tested 0, dropped 0 + : +- BroadcastExchange + : +- Project [c_custkey#0] (ids +51) + : : · operator #15 Project p8: in 20 rows (1 batch); out 20 rows (1 batch); compute (max , drivers) + : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] + : · source #14 MemoryScan p8: out 20 rows (1 batch); compute (max , drivers); io (max ) + +- BroadcastExchange + +- Project [l_orderkey#33] + : · operator #11 Project p5: in 5 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) + +- Filter (sum(l_quantity)#49 > 300.00) + : · operator #10 Filter p5: in 40 rows (1 batch); out 5 rows (1 batch); compute (max , drivers) + +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387904 AS sum(l_quantity)#49] + : · operator #9 Project p5: in 40 rows (1 batch); out 40 rows (1 batch); compute (max , drivers) + +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : · source #8 HashAggregate p5: out 40 rows (1 batch); compute (max , drivers); io (max ) + +- Exchange hashpartitioning(l_orderkey#33) + +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : · sink #6 HashAggregate p6: in 160 rows (1 batch); compute (max , drivers); peak mem ; groups=40 spill_runs=0 spill_files=0 spill_bytes= repartitions=0 hash_entries=40 hash_buckets= hash_resizes= hash_index=swiss row_abandons=0 row_final_tasks=0 row_final_spills=0 row_final_flushes=0 table_layout=rows probe_lookups= probe_key_compares= max_probe_length= exchange_rows= exchange_bytes= exchange_keys= exchange_map_tasks= exchange_partitions= exchange_max_partition_rows= + +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (ids +16) + · source #7 MemoryScan p6: out 160 rows (1 batch); compute (max , drivers); io (max ) Result: sink #0 Result p0: in 5 rows (1 batch); compute (max , drivers) Pipelines: - p0: wall , drivers, threads: #24 TopN -> #0 Result - p1: wall , drivers, threads: #23 TopN -> #1 TopN - p2: wall , drivers, threads: #21 HashAggregate -> #22 Project -> #2 TopN - p3: wall , drivers, threads: #17 MemoryScan -> #18 HashJoinProbe -> #19 Project -> #20 HashJoinProbe -> #3 HashAggregate - p4: wall , drivers, threads: #7 HashAggregate -> #8 Project -> #9 Filter -> #10 Project -> #4 HashJoinBuild - p5: wall , drivers, threads: #6 MemoryScan -> #5 HashAggregate - p6: wall , drivers, threads: #14 MemoryScan -> #15 HashJoinProbe -> #16 Project -> #11 HashJoinBuild - p7: wall , drivers, threads: #13 MemoryScan -> #12 HashJoinBuild + p0: wall , drivers, threads: #28 TopN -> #0 Result + p1: wall , drivers, threads: #27 TopN -> #1 TopN + p2: wall , drivers, threads: #25 HashAggregate -> #26 Project -> #2 TopN + p3: wall , drivers, threads: #22 MemoryScan -> #23 HashJoinProbe -> #24 Project -> #3 HashAggregate + p4: wall , drivers, threads: #18 MemoryScan -> #19 HashJoinProbe -> #20 Project -> #21 HashJoinProbe -> #4 HashJoinBuild + p5: wall , drivers, threads: #8 HashAggregate -> #9 Project -> #10 Filter -> #11 Project -> #5 HashJoinBuild + p6: wall , drivers, threads: #7 MemoryScan -> #6 HashAggregate + p7: wall , drivers, threads: #16 MemoryScan -> #17 HashJoinProbe -> #12 HashJoinBuild + p8: wall , drivers, threads: #14 MemoryScan -> #15 Project -> #13 HashJoinBuild From 624cd92080e3b33ad7c31a4238d2d24a00cbcf30 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:01:13 +0000 Subject: [PATCH 18/25] exec: q18 physical plan snapshot takes LatePayloadJoin's plan (c_name joins above the IN semi join) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .../planner/snapshots/planner__tpch__q18.snap | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap b/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap index fd64fad42..6f84ff95a 100644 --- a/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap +++ b/crates/krishiv-exec/tests/planner/snapshots/planner__tpch__q18.snap @@ -9,21 +9,25 @@ TopN 100, [o_totalprice#11 DESC NULLS LAST, o_orderdate#12 ASC NULLS FIRST] +- HashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] +- Exchange hashpartitioning(c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11) +- PartialHashAggregate keys=[c_name#1, c_custkey#0, o_orderkey#8, o_orderdate#12, o_totalprice#11], functions=[sum(l_quantity#21)] - +- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey - :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey - : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] - : +- BroadcastExchange - : +- Project [o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] - : +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey - : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] - : +- BroadcastExchange - : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] - +- BroadcastExchange - +- Project [l_orderkey#33] - +- Filter (sum(l_quantity)#49 > 300.00) - +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387904 AS sum(l_quantity)#49] - +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - +- Exchange hashpartitioning(l_orderkey#33) - +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] - +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (ids +16) + +- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, c_custkey#0, c_name#1] + +- HashJoin Inner, [o_custkey#9] = [c_custkey#0], RuntimeFilter #4: o_custkey#9 -> spark_catalog.default.orders.o_custkey + :- HashJoin LeftSemi, [o_orderkey#8] = [l_orderkey#33], RuntimeFilter #3: o_orderkey#8 -> spark_catalog.default.orders.o_orderkey + : :- Project [l_quantity#21, o_orderkey#8, o_totalprice#11, o_orderdate#12, o_custkey#9] + : : +- HashJoin Inner, [l_orderkey#17] = [o_orderkey#8], RuntimeFilter #2: l_orderkey#17 -> spark_catalog.default.lineitem.l_orderkey + : : :- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] + : : +- BroadcastExchange + : : +- HashJoin LeftSemi, [o_custkey#9] = [c_custkey#51], RuntimeFilter #1: o_custkey#9 -> spark_catalog.default.orders.o_custkey + : : :- Scan spark_catalog.default.orders [o_orderkey#8, o_custkey#9, o_totalprice#11, o_orderdate#12] + : : +- BroadcastExchange + : : +- Project [c_custkey#0] (ids +51) + : : +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] + : +- BroadcastExchange + : +- Project [l_orderkey#33] + : +- Filter (sum(l_quantity)#49 > 300.00) + : +- Project [l_orderkey#33, sum(l_quantity#37)#4611686018427387904 AS sum(l_quantity)#49] + : +- HashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : +- Exchange hashpartitioning(l_orderkey#33) + : +- PartialHashAggregate keys=[l_orderkey#33], functions=[sum(l_quantity#37)] + : +- Scan spark_catalog.default.lineitem [l_orderkey#17, l_quantity#21] (ids +16) + +- BroadcastExchange + +- Scan spark_catalog.default.customer [c_custkey#0, c_name#1] From 47eb83c895a74bddde21735975d3f88926fd674c Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:59:05 +0100 Subject: [PATCH 19/25] exec: a salted directory takes 5/4 of its keys rounded up, so it never grows past its reservation salted_slots rounded 5/4 of the key count down. Where n + n/4 lands exactly on a power of two and n is not a multiple of 4 (1,639 keys: 1,639 + 409 = 2,048 slots), the last key failed the load-4/5 check and the directory doubled while being built, past the index bytes reserved for it (grace loads and the finalize true-up both assume it never grows). Now n + ceil(n/4): 5n <= 4 slots always holds. A test builds 1,639 and neighbouring key counts and checks the directory keeps its planned size (it fails on the old sizing). Comments that still assumed load 1/2 say which measurements were taken at it. Opus review of the S5 stack: ops/reviews/2026-11-13-perf-2-s5-stack-diff.md (Important finding). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .../krishiv-exec/src/rows/flooding_tests.rs | 35 ++++++++++++++++--- crates/krishiv-exec/src/rows/index.rs | 5 +-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/crates/krishiv-exec/src/rows/flooding_tests.rs b/crates/krishiv-exec/src/rows/flooding_tests.rs index ee816ab8d..17a2171a4 100644 --- a/crates/krishiv-exec/src/rows/flooding_tests.rs +++ b/crates/krishiv-exec/src/rows/flooding_tests.rs @@ -157,10 +157,12 @@ fn crafted_partial_collisions_scatter() { /// `m` distinct keys forged to one full 64-bit `h`, among ordinary keys: /// every key is still found, absent keys with the same `h` are not, the -/// colliding keys' probe run grows linearly in `m` (at load 1/2 the -/// cluster takes in about as many neighbours: at most `2m` plus an -/// ordinary run); only the ordinary keys next to the cluster get longer -/// runs, so their total grows by `O(m²)`, whatever the table size. +/// colliding keys' probe run grows linearly in `m` (at this test's load, +/// about 1/2, the cluster takes in about as many neighbours: at most `2m` +/// plus an ordinary run; a directory near its 4/5 bound takes in more, still +/// linear in `m`, which this test does not measure); only the ordinary keys +/// next to the cluster get longer runs, so their total grows by `O(m²)`, +/// whatever the table size. #[test] fn full_collisions_stay_correct_and_bounded() { const BASE: usize = 3840; @@ -240,3 +242,28 @@ fn a_flat_index_has_no_probe_runs() { assert!(index.is_flat()); assert_eq!(index.probe_run(enc.hashes()[0]), None); } + +/// The directory is sized for its keys at load ≤ 4/5 and never grows: at +/// key counts where 5/4 of them lands exactly on a power of two and they are +/// not a multiple of 4 (1,639: 1,639 + 409 = 2,048), rounding 5/4 down left +/// the last key past the load bound, and the directory doubled with no +/// reservation behind it (review of S5's 5/4 sizing). +#[test] +fn the_directory_holds_its_keys_without_growing() { + let layout = layout(); + for n in [1_639_usize, 1_638, 1_640, 3_277, 6_554] { + let keys: Vec = (0..i64::try_from(n).unwrap_or(0)).collect(); + let (index, _) = salted(&layout, &keys, &[], 0, 7); + assert_eq!(index.len(), n); + assert!( + 5 * n <= 4 * index.buckets(), + "{n} keys in {}", + index.buckets() + ); + assert_eq!( + index.buckets(), + salted_bytes(n) / 8, + "{n} keys grew the directory" + ); + } +} diff --git a/crates/krishiv-exec/src/rows/index.rs b/crates/krishiv-exec/src/rows/index.rs index 73415a64d..9e145863d 100644 --- a/crates/krishiv-exec/src/rows/index.rs +++ b/crates/krishiv-exec/src/rows/index.rs @@ -135,7 +135,7 @@ const SALTED_LOAD_DEN: usize = 5; /// Slots of a salted directory for `n` keys at load ≤ 4/5. fn salted_slots(n: usize) -> usize { - n.saturating_add(n / SALTED_LOAD_NUM) + n.saturating_add(n.div_ceil(SALTED_LOAD_NUM)) .checked_next_power_of_two() .unwrap_or(usize::MAX) .max(MIN_SLOTS) @@ -492,7 +492,8 @@ fn grow_salted(old: &[u64], g: impl Fn(u32) -> u64) -> Result> { /// its salt, so their cache misses overlap before the chunk is resolved in /// order. It reads only the first slot: on a probe that mostly misses the salt /// test is well predicted, where walking the run takes the `s == 0` branch a -/// second time, about even at load 1/2 (TPC-DS q72). An empty slot of salt 0 +/// second time, about even at load 1/2 (TPC-DS q72; measured before the +/// directory moved to load up to 4/5). An empty slot of salt 0 /// prefetches row 0, a harmless hint. fn prefetch_chunk(slots: &[u64], gs: &[u64], src: &impl RowSource) { let mask = slots.len() - 1; From 8739b40c8e628ed4f64cf4c6e0d5576e1b5334a2 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:04:07 +0100 Subject: [PATCH 20/25] exec: a join build merges its tiny chunks before the block cut, so finalize of a small build no longer walks thousands Option C replaced the build side's size-tiered coalesce with the block cut, which merges chunks only once a trailing run holds a block's 2^14 rows. A build fed small batches pushes chunks of a few rows (12 drivers cutting each batch into its partitions): TPC-H q10 at SF10 held 29,359 chunks for 573,157 rows and a 1.1M-row build 115,394. Finalize walked every one twice for its reservation (RowArena::reblock_bytes) and then concatenated them, serially before any partition started: finalize took 72-89 ms where main took 9 (perf-1's 11-09 ABBA line: "every small build slower", the open item before option C's PR). Part::block now first merges runs of 64 chunks shorter than a block, size-tiered among themselves (the probe side's coalesce rule, fan-out 64, chunks of a block or more never merged again; chunks big enough to fill a block before 64 are held are still copied once, by the cut). Finalize also computes each partition's reblock bytes once, not twice. Measured on the laptop, SF10 warm PROFILE, 12 threads, finalize wall per build (instrumented scratch build of this code, not committed), option C before -> after: q10's 573K-row build 72-89 -> 14.5-16.3 ms (chunks 29,359 -> about 5,000; main was 9), the 1.1M-row build 310-339 -> 24-26 ms, q09's 435K-row build 50-77 -> 17-23 ms, its 109K-row build 10-11 -> 8-9 ms; the 3.3M-row build (whole blocks) unchanged at 40-42 ms. A test pushes 5,000 chunks of 20 rows and checks fewer than 128 are held, rows in order, the reservation covering them. Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/block.rs | 169 +++++++++++++++++++------- crates/krishiv-exec/src/join/build.rs | 17 +-- 2 files changed, 137 insertions(+), 49 deletions(-) diff --git a/crates/krishiv-exec/src/join/block.rs b/crates/krishiv-exec/src/join/block.rs index 8a193d59d..65883d022 100644 --- a/crates/krishiv-exec/src/join/block.rs +++ b/crates/krishiv-exec/src/join/block.rs @@ -26,58 +26,84 @@ impl Part { /// Returns the bytes of the largest run merged (each merge reserves the /// run again while it copies: the transient peak), 0 when none was. pub(crate) fn coalesce(&mut self, schema: &SchemaRef) -> Result { - const FANOUT: usize = 16; - let size = |c: &Batch| c.data().get_array_memory_size(); - let tier = |c: &Batch| size(c).max(1).ilog(FANOUT); - let mut largest = 0; - while self.chunks.len() >= FANOUT { - let Some(top) = self.chunks.last().map(tier) else { - return Ok(largest); - }; - let run = self - .chunks - .iter() - .rev() - .take_while(|c| tier(c) <= top) - .count(); - if run < FANOUT { - return Ok(largest); - } - let from = self.chunks.len() - run; - let old: usize = self.chunks[from..].iter().map(size).sum(); - let Some(m) = self.memory.first_mut().filter(|m| m.size() >= old) else { - return Ok(largest); - }; - if m.try_grow(old).is_err() { - return Ok(largest); - } - let merged = - Batch::concat(schema, &self.chunks[from..]).map_err(|e| internal(e.to_string())); - let merged = match merged { - Ok(b) if size(&b) <= old => b, - other => { - m.shrink(old); - return other.map(|_| largest); - } - }; - largest = largest.max(old); - m.shrink(old + old - size(&merged)); - self.chunks.truncate(from); - self.chunks.push(merged); + match self.memory.first_mut() { + Some(m) => merge_tiers(&mut self.chunks, m, schema, usize::MAX, 16), + None => Ok(0), } - Ok(largest) } /// [`block_chunks`] over the part's chunks, the copies reserved in its - /// first reservation (the one its chunks are reserved in). + /// first reservation (the one its chunks are reserved in). Runs of + /// [`BUILD_FANOUT`] chunks shorter than a block are first merged + /// size-tiered among themselves ([`merge_tiers`]): a build fed small + /// batches pushes chunks of a few rows (12 drivers cutting each batch + /// into its partitions: about 20 rows each on TPC-H q10 at SF10, 29,359 + /// chunks for 573,157 rows), and finalize's accounting and tail copy then + /// walked every one of them (finalize 9 -> 72-89 ms). Chunks of a block or + /// more are never merged again, and chunks large enough to fill a block + /// before [`BUILD_FANOUT`] of them are held are copied once, by the cut. pub(crate) fn block(&mut self, schema: &SchemaRef) -> Result { match self.memory.first_mut() { - Some(m) => block_chunks(&mut self.chunks, m, schema), + Some(m) => { + let merged = merge_tiers(&mut self.chunks, m, schema, ARENA_ROWS, BUILD_FANOUT)?; + Ok(merged.max(block_chunks(&mut self.chunks, m, schema)?)) + } None => Ok(0), } } } +/// The build side's merge fan-out: 64 chunks of a block's 2^14 rows hold +/// 256 rows each on average, so only chunks smaller than that are merged +/// before the block cut copies them. +const BUILD_FANOUT: usize = 64; + +/// [`Part::coalesce`]'s size-tiered merge over the trailing chunks shorter +/// than `rows` rows, `fanout` of one tier at a time, each merged copy +/// reserved in `m` first. +fn merge_tiers( + chunks: &mut Vec, + m: &mut Reservation, + schema: &SchemaRef, + rows: usize, + fanout: usize, +) -> Result { + let size = |c: &Batch| c.data().get_array_memory_size(); + let tier = |c: &Batch| size(c).max(1).ilog(16); + let mut largest = 0; + while chunks.len() >= fanout { + let Some(top) = chunks.last().map(tier) else { + return Ok(largest); + }; + let run = chunks + .iter() + .rev() + .take_while(|c| c.num_rows() < rows && tier(c) <= top) + .count(); + if run < fanout { + return Ok(largest); + } + let from = chunks.len() - run; + let old: usize = chunks[from..].iter().map(size).sum(); + if m.size() < old || m.try_grow(old).is_err() { + return Ok(largest); + } + let merged = Batch::concat(schema, &chunks[from..]).map_err(|e| internal(e.to_string())); + let merged = match merged { + Ok(b) if size(&b) <= old => b, + other => { + m.shrink(old); + return other.map(|_| largest); + } + }; + largest = largest.max(old); + m.shrink(old + old - size(&merged)); + chunks.truncate(from); + chunks.push(merged); + } + Ok(largest) +} + /// Cuts build rows into the arena's blocks as they come (X3' step 3 /// option C): once the trailing chunks shorter than a block hold a /// block's rows, they are copied into one block of exactly @@ -339,4 +365,65 @@ mod tests { drop(parts); assert_eq!(pool.reserved(), 0); } + + /// A build fed tiny chunks (TPC-H q10 at SF10: about 20 rows each) holds + /// few of them: runs of `BUILD_FANOUT` merge before the block cut, so + /// finalize does not walk thousands. Rows keep their order, none is + /// lost, and the reservation covers what is held. + #[test] + fn tiny_build_chunks_are_merged_before_the_block_cut() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("tiny", None); + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("s", DataType::Utf8, false), + ])); + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + let mut most = 0; + for _ in 0..5000 { + let k = Int64Array::from_iter_values(next..next + 20); + let s = StringArray::from_iter_values((next..next + 20).map(|j| format!("row-{j}"))); + next += 20; + let c = Batch::new( + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k), Arc::new(s)]).unwrap(), + ); + reserve_chunks(&mut parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].block(&schema).unwrap(); + most = most.max(parts[0].chunks.len()); + } + // Without the merge: up to a block's 819 chunks of 20 rows. + assert!(most < 2 * BUILD_FANOUT, "{most} chunks held"); + let keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + assert!( + keys.into_iter().eq(0..next), + "rows lost, duplicated or reordered" + ); + let held: usize = parts[0] + .chunks + .iter() + .map(|c| c.data().get_array_memory_size()) + .sum(); + assert!( + parts[0].bytes() >= held, + "{} reserved, {held} held", + parts[0].bytes() + ); + drop(parts); + assert_eq!(pool.reserved(), 0); + } } diff --git a/crates/krishiv-exec/src/join/build.rs b/crates/krishiv-exec/src/join/build.rs index 6eaffdcd1..c00425dd4 100644 --- a/crates/krishiv-exec/src/join/build.rs +++ b/crates/krishiv-exec/src/join/build.rs @@ -646,12 +646,13 @@ impl BuildSink { } /// What indexing a partition's in-memory rows reserves, as `(lasting, -/// transient)`: lasting is its [`JoinTable`] index (and match bitmap when +/// transient, kept)`: lasting is its [`JoinTable`] index (and match bitmap when /// `marks`) plus the copied chunk tails that stay beside chunks the arena /// also slices (held through the probe); transient is the rest of the /// tails' copy, which replaces chunks dropped once the arena is built -/// ([`RowArena::reblock_bytes`]). `None` if no rows are in memory. -fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize)> { +/// ([`RowArena::reblock_bytes`]); kept is the tails' part of lasting. +/// `None` if no rows are in memory. +fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usize, usize)> { if part.chunks.is_empty() { return None; } @@ -662,7 +663,7 @@ fn build_need(part: &Part, row_layout: bool, marks: bool) -> Option<(usize, usiz index = index.saturating_add(JoinTable::marks_bytes(rows)); } let (copy, kept) = RowArena::reblock_bytes(&part.chunks, 0); - Some((index.saturating_add(kept), copy.saturating_sub(kept))) + Some((index.saturating_add(kept), copy.saturating_sub(kept), kept)) } /// Returns what a built partition's chunks reserved beyond its arena: the @@ -1015,7 +1016,7 @@ impl Sink for BuildSink { let rows = g.hashing.rows(); let marks = s.join_type.builds_preserved(); let needs = c.parts.iter().filter_map(|p| build_need(p, rows, marks)); - needs.fold((0usize, 0usize), |(i, m), (index, copy)| { + needs.fold((0usize, 0usize), |(i, m), (index, copy, _)| { (i.saturating_add(index), usize::max(m, copy)) }) }; @@ -1039,12 +1040,12 @@ impl Sink for BuildSink { let marks = s.join_type.builds_preserved(); Some((i, build_need(p, g.hashing.rows(), marks)?, p, keys)) }) - .map(|(i, need, p, keys)| Job { + .map(|(i, (lasting, copy, kept), p, keys)| Job { part: i, bytes: p.bytes(), - kept: RowArena::reblock_bytes(&p.chunks, 0).1, + kept, chunks: std::mem::take(&mut p.chunks), - need, + need: (lasting, copy), keys, }) .collect() From 2dd5875e373029b6642e6ab90234e8d9c0eaf6a8 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:37:05 +0100 Subject: [PATCH 21/25] exec, common: a join build cuts blocks before merging its tiny chunks; deterministicOrder names the build's block copy Re-review M6 of the S5 stack: Part::block merged runs of 64 short chunks before the block cut, so a run that already filled a block (63 chunks of 255 rows, then 400) became one 16,465-row chunk: its rows were copied twice and an 81-row tail stayed beside it. The cut now runs first. A test checks that run gives a block and an 81-row tail. M1: the krishiv.deterministicOrder doc says a hash join build's block copy under memory pressure can change the order of what it returns, as a spill can. Review: ops/reviews/2026-11-14-perf-2-s5-stack-rereview.md (APPROVED). Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-common/src/config/registry.rs | 3 +- crates/krishiv-exec/src/join/block.rs | 37 +++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/krishiv-common/src/config/registry.rs b/crates/krishiv-common/src/config/registry.rs index 014bf551e..4b29d1030 100644 --- a/crates/krishiv-common/src/config/registry.rs +++ b/crates/krishiv-common/src/config/registry.rs @@ -766,7 +766,8 @@ pub static DETERMINISTIC_ORDER: ConfigEntry = ConfigEntry { seed, so an operator that spills splits its rows the same way on every run. Costs \ parallelism where it falls back to one driver: meant for reproducing a result, \ comparing two builds, and tests. Memory pressure from other queries in the process \ - can still change when an operator spills, and so the order of what it returns. The \ + can still change when an operator spills, or whether a hash join build copies its \ + rows into blocks as they arrive, and so the order of what it returns. The \ branches of a `UNION ALL` under an operator that must see all of them (a hash join \ probing a union) take turns in branch order, so they lose their overlap too. Off by \ default.", diff --git a/crates/krishiv-exec/src/join/block.rs b/crates/krishiv-exec/src/join/block.rs index 65883d022..65f942d46 100644 --- a/crates/krishiv-exec/src/join/block.rs +++ b/crates/krishiv-exec/src/join/block.rs @@ -45,8 +45,11 @@ impl Part { pub(crate) fn block(&mut self, schema: &SchemaRef) -> Result { match self.memory.first_mut() { Some(m) => { + // Cut first: a run that already fills a block is copied once, + // into the block, not merged and then cut again. + let cut = block_chunks(&mut self.chunks, m, schema)?; let merged = merge_tiers(&mut self.chunks, m, schema, ARENA_ROWS, BUILD_FANOUT)?; - Ok(merged.max(block_chunks(&mut self.chunks, m, schema)?)) + Ok(cut.max(merged)) } None => Ok(0), } @@ -366,6 +369,38 @@ mod tests { assert_eq!(pool.reserved(), 0); } + /// A run of short chunks that fills a block is cut into the block, not + /// merged into one chunk past the block first: 63 chunks of 255 rows and + /// one of 400 give a block and an 81-row tail (review M6 of the S5 stack). + #[test] + fn a_run_that_fills_a_block_is_cut_not_merged() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("cut-first", None); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + for n in std::iter::repeat_n(255i64, 63).chain([400]) { + let c = Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from_iter_values(next..next + n))], + ) + .unwrap(), + ); + next += n; + reserve_chunks(&mut parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + parts[0].block(&schema).unwrap(); + } + let rows: Vec = parts[0].chunks.iter().map(Batch::num_rows).collect(); + assert_eq!(rows, [ARENA_ROWS, 63 * 255 + 400 - ARENA_ROWS]); + drop(parts); + assert_eq!(pool.reserved(), 0); + } + /// A build fed tiny chunks (TPC-H q10 at SF10: about 20 rows each) holds /// few of them: runs of `BUILD_FANOUT` merge before the block cut, so /// finalize does not walk thousands. Rows keep their order, none is From 5c06617993848f2950d138661653a68f4bc8fec6 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:51:46 +0100 Subject: [PATCH 22/25] docs: config reference follows the krishiv.deterministicOrder doc Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- docs/reference/config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/config/index.md b/docs/reference/config/index.md index 04865e956..b8e9d110b 100644 --- a/docs/reference/config/index.md +++ b/docs/reference/config/index.md @@ -31,7 +31,7 @@ Keys under `krishiv.experimental.` are experimental and off by default. Spark al | `krishiv.connect.session.idleTimeout` | duration | `1h` | cluster | startup | 0.3.0 | | A Spark Connect session no RPC used for this long is released (its id then answers `INVALID_HANDLE.SESSION_CLOSED`); a session running a query is kept. `0s` never expires (Spark's `spark.connect.session.manager.defaultSessionTimeout`, whose `-1` this is). | | `krishiv.connect.session.maintenanceInterval` | duration (bare number: ms) | `30s` | cluster | startup | 0.3.0 | `spark.connect.session.manager.maintenanceInterval` | How often the server looks for idle Spark Connect sessions to release. | | `krishiv.delta.logThreads` | positive int | `4` | cluster | startup | 0.2.0 | | Experimental. Threads of the one process-wide pool on which Delta log work runs (log replay, snapshots, commits, checkpoints): at most this many Delta kernel calls run at once across all queries, and a query waits for a free thread (a cancellable wait, measured by the `delta.log_pool.wait_ms` metric). Read when the pool first starts. | -| `krishiv.deterministicOrder` | bool | `false` | session | user | 0.1.0 | | Make a query's result reproducible, aiming at the same bytes at any thread count (a float sum under a window, a sort-merge join or a very large grouping can still differ, because those size their buckets by the thread count). A scan pipeline whose source can be split deterministically runs one driver per logical partition (up to 16), morsels are dealt to them by ordinal, and their partial states are combined in partition order, so partial states (a decimal `sum` or `avg`, whose overflow can depend on the order of the merges) merge the same way and a query without `ORDER BY` returns its rows in the same order on every run, byte for byte, for the engine's own scans. Any other pipeline (a sink that is not lane-safe, a source that reports no positions, a pushed `LIMIT`, an operator with work after its input, a memory limit under 4 GiB) runs on one driver. The key hashes take a fixed seed, so an operator that spills splits its rows the same way on every run. Costs parallelism where it falls back to one driver: meant for reproducing a result, comparing two builds, and tests. Memory pressure from other queries in the process can still change when an operator spills, and so the order of what it returns. The branches of a `UNION ALL` under an operator that must see all of them (a hash join probing a union) take turns in branch order, so they lose their overlap too. Off by default. | +| `krishiv.deterministicOrder` | bool | `false` | session | user | 0.1.0 | | Make a query's result reproducible, aiming at the same bytes at any thread count (a float sum under a window, a sort-merge join or a very large grouping can still differ, because those size their buckets by the thread count). A scan pipeline whose source can be split deterministically runs one driver per logical partition (up to 16), morsels are dealt to them by ordinal, and their partial states are combined in partition order, so partial states (a decimal `sum` or `avg`, whose overflow can depend on the order of the merges) merge the same way and a query without `ORDER BY` returns its rows in the same order on every run, byte for byte, for the engine's own scans. Any other pipeline (a sink that is not lane-safe, a source that reports no positions, a pushed `LIMIT`, an operator with work after its input, a memory limit under 4 GiB) runs on one driver. The key hashes take a fixed seed, so an operator that spills splits its rows the same way on every run. Costs parallelism where it falls back to one driver: meant for reproducing a result, comparing two builds, and tests. Memory pressure from other queries in the process can still change when an operator spills, or whether a hash join build copies its rows into blocks as they arrive, and so the order of what it returns. The branches of a `UNION ALL` under an operator that must see all of them (a hash join probing a union) take turns in branch order, so they lose their overlap too. Off by default. | | `krishiv.exec.agg.directKeys` | bool | `true` | session | user | 0.2.0 | | Internal: a grouped aggregation whose key columns in a batch are all one-byte strings or small integers without NULLs finds each row's group through a table indexed by the packed key, and hashes only each new key's first row. Results are the same either way; false hashes every row. | | `krishiv.exec.agg.rowLayout` | bool | `true` | session | user | 0.1.0 | | Internal: grouped aggregations keep their keys as fixed-width rows in radix-partitioned tables and merge the partitions in parallel. Results are the same either way; aggregations with nested or dictionary keys keep the columnar path. | | `krishiv.exec.batchSize` | int | `4096` | session | admin | 0.1.0 | | Rows per batch passed between operators. Operators do about a millisecond of work per batch, so the scheduler can switch tasks between batches. | From e04bda8083baf5361e527c0352fc067dbb2f4d92 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:45:20 +0100 Subject: [PATCH 23/25] exec: a join build's block cut keeps the chunks held past the block block_chunks built a block from the trailing run's chunks up to the one that fills it, then truncated the list at the run's start: every chunk past that one was dropped with it. A cut skipped while its copy could not be reserved, then more chunks pushed (a spilling build, a grace reload under pressure) left such chunks, and their rows were lost: the bench-runner spill_equivalence and q21-shaped memory_bound tests returned fewer rows (option C, ab45aa524, found by bisecting #749's CI failure). The chunks past the block now stay, after its tail. A test cuts three chunks of 10,000 rows at once and checks every row is kept. Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-exec/src/join/block.rs | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/crates/krishiv-exec/src/join/block.rs b/crates/krishiv-exec/src/join/block.rs index 65f942d46..e1e0b8275 100644 --- a/crates/krishiv-exec/src/join/block.rs +++ b/crates/krishiv-exec/src/join/block.rs @@ -193,9 +193,13 @@ pub(crate) fn block_chunks( // Held now: the block, and the last chunk while its tail lives // (`old + excess >= new`). m.shrink(old + excess + freed - new); + // Chunks past `last` (held while an earlier cut could not be + // reserved) are not in the block: they stay, after its tail. + let rest: Vec = chunks.drain(last + 1..).collect(); chunks.truncate(from); chunks.push(block); chunks.extend(tail); + chunks.extend(rest); } } @@ -461,4 +465,53 @@ mod tests { drop(parts); assert_eq!(pool.reserved(), 0); } + + /// Chunks held past the one that fills a block (a cut skipped while its + /// copy could not be reserved, then more chunks pushed; a grace reload + /// under pressure) stay: the cut drops only the chunks it copied. Three + /// chunks of 10,000 rows give a block, the second's tail and the third. + #[test] + fn chunks_past_the_block_are_kept() { + let pool: Arc = TreePool::new(&MemoryConfig::default()); + let id = pool.register("past-the-block", None); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); + let mut parts = vec![Part::with_memory(krishiv_memory::Reservation::new( + Arc::clone(&pool), + id, + ))]; + let mut next = 0i64; + for _ in 0..3 { + let c = Batch::new( + RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from_iter_values(next..next + 10_000))], + ) + .unwrap(), + ); + next += 10_000; + reserve_chunks(&mut parts, &[(0, c.clone())]).unwrap(); + parts[0].chunks.push(c); + } + let part = &mut parts[0]; + let Some(m) = part.memory.first_mut() else { + panic!("no reservation"); + }; + block_chunks(&mut part.chunks, m, &schema).unwrap(); + let mut keys: Vec = parts[0] + .chunks + .iter() + .flat_map(|c| { + let k = c + .data() + .column(0) + .as_primitive::(); + k.values().to_vec() + }) + .collect(); + keys.sort_unstable(); + assert!(keys.into_iter().eq(0..next), "rows lost or duplicated"); + assert_eq!(parts[0].chunks[0].num_rows(), ARENA_ROWS); + drop(parts); + assert_eq!(pool.reserved(), 0); + } } From 17a690ae2dd80354b5a617e3b7b531e002a8e7d2 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:47:27 +0100 Subject: [PATCH 24/25] optimizer: LatePayloadJoin collects a candidate only below a non-inner join The rule read every inner join under every consumer as a candidate (output resolution, a repeatable walk, the path checks) and only then refused those with no non-inner join on their path: a plan of inner joins paid for all of it and moved nothing (CodSpeed on #749: join_chain_12 +23-30%, optimize_tpch_22 +11%). walk now carries whether its path holds a non-inner join and collects candidates only then; what moves is the same. Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-optimizer/src/late_payload.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/krishiv-optimizer/src/late_payload.rs b/crates/krishiv-optimizer/src/late_payload.rs index 03ebdd495..db0332c80 100644 --- a/crates/krishiv-optimizer/src/late_payload.rs +++ b/crates/krishiv-optimizer/src/late_payload.rs @@ -299,7 +299,7 @@ fn at_consumer( for (i, kid) in kids.iter().enumerate() { let mut found = Vec::new(); let mut budget = MAX_WALK; - walk(kid, &mut Vec::new(), &mut found, &mut budget); + walk(kid, &mut Vec::new(), false, &mut found, &mut budget); for cand in found { if cand .path @@ -318,9 +318,6 @@ fn at_consumer( if passes(consumer, i, &shape.blocked) { continue; } - if !cand.path.iter().any(|s| is_non_inner_join(&s.node)) { - continue; - } // Nothing of the dimension above the join: nothing to move. if shape.carried(&cand).is_empty() { continue; @@ -358,12 +355,15 @@ fn at_consumer( None } -/// Every inner join reachable from `at` through pass-through nodes, with the -/// path above it. `path` holds the nodes from the consumer's child down to -/// `at`'s parent (top first while walking). +/// Every inner join reachable from `at` through pass-through nodes with a +/// non-inner join on the path above it (nothing else can move), with that +/// path. `path` holds the nodes from the consumer's child down to `at`'s +/// parent (top first while walking); `non_inner` says it holds a non-inner +/// join, so a plan of inner joins only yields no candidate to read. fn walk( at: &Arc, path: &mut Vec, + non_inner: bool, found: &mut Vec, budget: &mut usize, ) { @@ -375,6 +375,7 @@ fn walk( kind: JoinKind::Inner, .. } = &at.node + && non_inner { for dim_left in [true, false] { found.push(Candidate { @@ -393,7 +394,7 @@ fn walk( node: Arc::clone(at), child: i, }); - walk(kid, path, found, budget); + walk(kid, path, non_inner || is_non_inner_join(at), found, budget); path.pop(); } } From 32a3ff476d19ae1824ee76c3f6fe949516aa7bb8 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:10:05 +0100 Subject: [PATCH 25/25] changelog: fragment for #749 Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- changelog.d/749.performance.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/749.performance.md diff --git a/changelog.d/749.performance.md b/changelog.d/749.performance.md new file mode 100644 index 000000000..777e41ecd --- /dev/null +++ b/changelog.d/749.performance.md @@ -0,0 +1,3 @@ +Hash joins keep their build side in fixed-size row blocks with a smaller, denser hash directory, and a join's +unused columns are joined in only after the filtering joins above it: TPC-H q21 at scale factor 1000 runs in +277 s instead of 474 s with no spill (laptop, cold run).