From f9bfbddc2147bd4117c53e6981f1e6ac573d5a6b Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 24 Oct 2026 09:55:04 +0800 Subject: [PATCH 1/4] docs: clarify estimated_size accounting --- datasketches/src/bloom/sketch.rs | 5 ++++- datasketches/src/countmin/sketch.rs | 5 ++++- datasketches/src/cpc/pair_table.rs | 2 +- datasketches/src/cpc/sketch.rs | 5 ++++- datasketches/src/cpc/union.rs | 5 ++++- .../reverse_purge_item_hash_map.rs | 2 +- datasketches/src/frequencies/sketch.rs | 7 +++++- datasketches/src/hll/array4.rs | 2 +- datasketches/src/hll/array6.rs | 2 +- datasketches/src/hll/array8.rs | 2 +- datasketches/src/hll/aux_map.rs | 2 +- datasketches/src/hll/container.rs | 2 +- datasketches/src/hll/sketch.rs | 5 ++++- datasketches/src/hll/union.rs | 5 ++++- datasketches/src/tdigest/sketch.rs | 11 ++++++++-- .../src/thetafamily/common/hash_table.rs | 2 +- .../src/thetafamily/common/intersection.rs | 2 +- datasketches/src/thetafamily/common/union.rs | 2 +- .../src/thetafamily/theta/intersection.rs | 5 ++++- datasketches/src/thetafamily/theta/sketch.rs | 10 +++++++-- datasketches/src/thetafamily/theta/union.rs | 5 ++++- .../src/thetafamily/tuple/intersection.rs | 7 +++++- datasketches/src/thetafamily/tuple/sketch.rs | 14 ++++++++++-- datasketches/src/thetafamily/tuple/union.rs | 7 +++++- tests-integration/tests/bloom_test/sketch.rs | 10 +++++++++ .../tests/countmin_test/sketch.rs | 7 ++++-- tests-integration/tests/cpc_test/update.rs | 14 ++++++++++++ .../tests/frequencies_test/update.rs | 16 ++++++++++++++ tests-integration/tests/hll_test/update.rs | 17 ++++++++++++++ tests-integration/tests/theta_test/sketch.rs | 2 ++ tests-integration/tests/theta_test/union.rs | 14 ++++++++++++ tests-integration/tests/tuple_test/sketch.rs | 22 +++++++++++++++++++ tests-integration/tests/tuple_test/union.rs | 15 +++++++++++++ 33 files changed, 204 insertions(+), 29 deletions(-) diff --git a/datasketches/src/bloom/sketch.rs b/datasketches/src/bloom/sketch.rs index 0d399d58..6879e140 100644 --- a/datasketches/src/bloom/sketch.rs +++ b/datasketches/src/bloom/sketch.rs @@ -639,7 +639,10 @@ impl BloomFilter { was_set } - /// Returns the estimated size of the filter in bytes. + /// Returns the estimated in-memory size of the filter in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.bit_array.len() * size_of::() } diff --git a/datasketches/src/countmin/sketch.rs b/datasketches/src/countmin/sketch.rs index e69150c7..5e24eb6e 100644 --- a/datasketches/src/countmin/sketch.rs +++ b/datasketches/src/countmin/sketch.rs @@ -495,7 +495,10 @@ impl CountMinSketch { Ok(sketch) } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.counts.capacity() * size_of::() diff --git a/datasketches/src/cpc/pair_table.rs b/datasketches/src/cpc/pair_table.rs index a476ec1f..72782494 100644 --- a/datasketches/src/cpc/pair_table.rs +++ b/datasketches/src/cpc/pair_table.rs @@ -258,7 +258,7 @@ impl PairTable { } } - /// Returns the estimated size of the heap allocations in bytes + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.slots.capacity() * size_of::() } diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs index c42b3672..1c59e7de 100644 --- a/datasketches/src/cpc/sketch.rs +++ b/datasketches/src/cpc/sketch.rs @@ -459,7 +459,10 @@ impl CpcSketch { matrix } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { let heap_size = self.sliding_window.capacity() + self diff --git a/datasketches/src/cpc/union.rs b/datasketches/src/cpc/union.rs index 67870f59..ca542bbf 100644 --- a/datasketches/src/cpc/union.rs +++ b/datasketches/src/cpc/union.rs @@ -347,7 +347,10 @@ impl CpcUnion { } } - /// Returns the estimated size of the union in bytes. + /// Returns the estimated in-memory size of the union in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { // The state's inline size is already covered by size_of::(). let heap_size = match &self.state { diff --git a/datasketches/src/frequencies/reverse_purge_item_hash_map.rs b/datasketches/src/frequencies/reverse_purge_item_hash_map.rs index ce3d62dc..f724d3f2 100644 --- a/datasketches/src/frequencies/reverse_purge_item_hash_map.rs +++ b/datasketches/src/frequencies/reverse_purge_item_hash_map.rs @@ -197,7 +197,7 @@ impl ReversePurgeItemHashMap { self.num_active } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.keys.capacity() * size_of::>() + self.values.capacity() * size_of::() diff --git a/datasketches/src/frequencies/sketch.rs b/datasketches/src/frequencies/sketch.rs index 4808ef58..946519f6 100644 --- a/datasketches/src/frequencies/sketch.rs +++ b/datasketches/src/frequencies/sketch.rs @@ -323,7 +323,12 @@ impl FrequentItemsSketch { self.hash_map.lg_length() } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Inline representations of items are included, but heap allocations owned by those items are + /// not recursively inspected. pub fn estimated_size(&self) -> usize { size_of::() + self.hash_map.estimated_size() } diff --git a/datasketches/src/hll/array4.rs b/datasketches/src/hll/array4.rs index 19c96320..a008f163 100644 --- a/datasketches/src/hll/array4.rs +++ b/datasketches/src/hll/array4.rs @@ -419,7 +419,7 @@ impl Array4 { bytes.into_bytes() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() + self diff --git a/datasketches/src/hll/array6.rs b/datasketches/src/hll/array6.rs index c29a9015..7c05b144 100644 --- a/datasketches/src/hll/array6.rs +++ b/datasketches/src/hll/array6.rs @@ -233,7 +233,7 @@ impl Array6 { bytes.into_bytes() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() } diff --git a/datasketches/src/hll/array8.rs b/datasketches/src/hll/array8.rs index 355bcc18..4030a5b0 100644 --- a/datasketches/src/hll/array8.rs +++ b/datasketches/src/hll/array8.rs @@ -278,7 +278,7 @@ impl Array8 { bytes.into_bytes() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() } diff --git a/datasketches/src/hll/aux_map.rs b/datasketches/src/hll/aux_map.rs index e14ef31f..09541974 100644 --- a/datasketches/src/hll/aux_map.rs +++ b/datasketches/src/hll/aux_map.rs @@ -192,7 +192,7 @@ impl AuxMap { }) } - /// Returns the estimated size of the heap allocations in bytes + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.entries.len() * size_of::() } diff --git a/datasketches/src/hll/container.rs b/datasketches/src/hll/container.rs index 8f0313c7..f55ee8f1 100644 --- a/datasketches/src/hll/container.rs +++ b/datasketches/src/hll/container.rs @@ -121,7 +121,7 @@ impl Container { self.coupons.iter().filter(|&&c| !c.is_empty()).copied() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.coupons.len() * size_of::() } diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 92d4d5b3..464efae0 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -470,7 +470,10 @@ impl HllSketch { } } - /// Returns the estimated memory usage in bytes, including owned heap allocations. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { let heap_size = match &self.mode { Mode::List { list, .. } => list.container().estimated_size(), diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index 0f6e20ec..7f262e43 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -365,7 +365,10 @@ impl HllUnion { self.gadget.lower_bound(num_std_dev) } - /// Returns the estimated memory usage in bytes, including owned heap allocations. + /// Returns the estimated in-memory size of the union in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { // The gadget's inline size is already covered by size_of::(). size_of::() - size_of::() + self.gadget.estimated_size() diff --git a/datasketches/src/tdigest/sketch.rs b/datasketches/src/tdigest/sketch.rs index 1fb4b16f..6f826829 100644 --- a/datasketches/src/tdigest/sketch.rs +++ b/datasketches/src/tdigest/sketch.rs @@ -156,6 +156,7 @@ impl TDigestBuffer { self.centroids } + /// Returns the estimated size of retained heap allocations in bytes. fn estimated_size(&self) -> usize { self.centroids.capacity() * size_of::() } @@ -1012,7 +1013,10 @@ impl TDigestMut { centroids.shrink_to(target_capacity); } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.buffer.estimated_size() } @@ -1630,7 +1634,10 @@ impl TDigest { ) } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.centroids.capacity() * size_of::() } diff --git a/datasketches/src/thetafamily/common/hash_table.rs b/datasketches/src/thetafamily/common/hash_table.rs index d2138c93..1d86bd66 100644 --- a/datasketches/src/thetafamily/common/hash_table.rs +++ b/datasketches/src/thetafamily/common/hash_table.rs @@ -344,7 +344,7 @@ where }) } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.entries.capacity() * size_of::>() } diff --git a/datasketches/src/thetafamily/common/intersection.rs b/datasketches/src/thetafamily/common/intersection.rs index 486b2cb7..ca5e7f20 100644 --- a/datasketches/src/thetafamily/common/intersection.rs +++ b/datasketches/src/thetafamily/common/intersection.rs @@ -234,7 +234,7 @@ where self.result_state != IntersectionResultState::Uninitialized } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.table.estimated_size() } diff --git a/datasketches/src/thetafamily/common/union.rs b/datasketches/src/thetafamily/common/union.rs index 0131b55a..4c585bd2 100644 --- a/datasketches/src/thetafamily/common/union.rs +++ b/datasketches/src/thetafamily/common/union.rs @@ -150,7 +150,7 @@ where self.result_theta = None; } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns the estimated size of retained heap allocations in bytes. pub fn estimated_size(&self) -> usize { self.table.estimated_size() } diff --git a/datasketches/src/thetafamily/theta/intersection.rs b/datasketches/src/thetafamily/theta/intersection.rs index ace0d41b..994e8ed8 100644 --- a/datasketches/src/thetafamily/theta/intersection.rs +++ b/datasketches/src/thetafamily/theta/intersection.rs @@ -78,7 +78,10 @@ impl ThetaIntersection { self.state.has_result() } - /// Returns the estimated size of the intersection in bytes. + /// Returns the estimated in-memory size of the intersection in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.state.estimated_size() } diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index b75d1aff..7be5b456 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -427,7 +427,10 @@ impl ThetaSketch { .expect("theta should always be valid") } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.table.estimated_size() } @@ -1027,7 +1030,10 @@ impl CompactThetaSketch { Ok(Self::from_compact_state(compact_state)) } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.compact_state.retained_entries_capacity() * size_of::() } diff --git a/datasketches/src/thetafamily/theta/union.rs b/datasketches/src/thetafamily/theta/union.rs index c315c4f2..31f64466 100644 --- a/datasketches/src/thetafamily/theta/union.rs +++ b/datasketches/src/thetafamily/theta/union.rs @@ -64,7 +64,10 @@ impl ThetaUnion { self.state.reset(); } - /// Returns the estimated size of the union in bytes. + /// Returns the estimated in-memory size of the union in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { size_of::() + self.state.estimated_size() } diff --git a/datasketches/src/thetafamily/tuple/intersection.rs b/datasketches/src/thetafamily/tuple/intersection.rs index 38dfe9ed..ce1bea95 100644 --- a/datasketches/src/thetafamily/tuple/intersection.rs +++ b/datasketches/src/thetafamily/tuple/intersection.rs @@ -138,7 +138,12 @@ where self.state.has_result() } - /// Returns the estimated size of the intersection in bytes. + /// Returns the estimated in-memory size of the intersection in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Inline representations of summaries and the policy are included, but heap allocations owned + /// by those values are not recursively inspected. pub fn estimated_size(&self) -> usize { size_of::() + self.state.estimated_size() } diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index e3a690fe..2ce99ddc 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -404,7 +404,12 @@ where .expect("theta should always be valid") } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Inline representations of summaries and the policy are included, but heap allocations owned + /// by those values are not recursively inspected. pub fn estimated_size(&self) -> usize { size_of::() + self.table.estimated_size() } @@ -541,7 +546,12 @@ impl CompactTupleSketch { .expect("compact theta should always be valid") } - /// Returns the estimated size of the sketch in bytes. + /// Returns the estimated in-memory size of the sketch in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Inline representations of summaries are included, but heap allocations owned by those + /// summaries are not recursively inspected. pub fn estimated_size(&self) -> usize { size_of::() + self.compact_state.retained_entries_capacity() * size_of::>() diff --git a/datasketches/src/thetafamily/tuple/union.rs b/datasketches/src/thetafamily/tuple/union.rs index df7786b7..39418e62 100644 --- a/datasketches/src/thetafamily/tuple/union.rs +++ b/datasketches/src/thetafamily/tuple/union.rs @@ -112,7 +112,12 @@ where self.state.reset(); } - /// Returns the estimated size of the union in bytes. + /// Returns the estimated in-memory size of the union in bytes, including its inline + /// representation and the retained heap allocations it owns. + /// + /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Inline representations of summaries and the policy are included, but heap allocations owned + /// by those values are not recursively inspected. pub fn estimated_size(&self) -> usize { size_of::() + self.state.estimated_size() } diff --git a/tests-integration/tests/bloom_test/sketch.rs b/tests-integration/tests/bloom_test/sketch.rs index db0a0db1..4bc3a3d2 100644 --- a/tests-integration/tests/bloom_test/sketch.rs +++ b/tests-integration/tests/bloom_test/sketch.rs @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of; +use std::mem::size_of_val; + use datasketches::bloom::BloomFilter; use datasketches::bloom::BloomFilterBuilder; use datasketches::error::ErrorKind; @@ -211,6 +214,13 @@ fn test_requested_size_rounds_to_word_boundary() { assert_eq!(filter.capacity(), 128); } +#[test] +fn test_estimated_size() { + let filter = BloomFilterBuilder::with_size(65, 3).build().unwrap(); + let expected_size = size_of_val(&filter) + filter.capacity() / 64 * size_of::(); + assert_eq!(filter.estimated_size(), expected_size); +} + #[test] fn test_accuracy_builder_rejects_zero_items_at_build() { let error = BloomFilterBuilder::with_accuracy(0, 0.01) diff --git a/tests-integration/tests/countmin_test/sketch.rs b/tests-integration/tests/countmin_test/sketch.rs index a5d39189..15cb645f 100644 --- a/tests-integration/tests/countmin_test/sketch.rs +++ b/tests-integration/tests/countmin_test/sketch.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of; +use std::mem::size_of_val; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -445,11 +447,12 @@ fn test_increment_single_key_like_rust_count_min_sketch() { #[test] fn test_estimated_size() { let mut sketch = CountMinSketch::::new(4, 128).unwrap(); - assert_eq!(sketch.estimated_size(), 4224); + let expected_size = size_of_val(&sketch) + 4 * 128 * size_of::() + 4 * size_of::(); + assert_eq!(sketch.estimated_size(), expected_size); // The backing tables are allocated up front; updates do not grow the sketch. sketch.update("apple"); - assert_eq!(sketch.estimated_size(), 4224); + assert_eq!(sketch.estimated_size(), expected_size); } #[test] diff --git a/tests-integration/tests/cpc_test/update.rs b/tests-integration/tests/cpc_test/update.rs index 99a1cb53..680f50e8 100644 --- a/tests-integration/tests/cpc_test/update.rs +++ b/tests-integration/tests/cpc_test/update.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of_val; + use datasketches::common::NumStdDev; use datasketches::cpc::CpcSketch; use datasketches::error::ErrorKind; @@ -67,3 +69,15 @@ fn test_max_serialized_bytes_validates_lg_k() { assert_eq!(error.kind(), ErrorKind::InvalidArgument); } } + +#[test] +fn test_estimated_size() { + let mut sketch = CpcSketch::new(11).unwrap(); + let empty_size = sketch.estimated_size(); + assert_eq!(empty_size, size_of_val(&sketch)); + + for i in 0..10_000 { + sketch.update(i); + } + assert!(sketch.estimated_size() > empty_size); +} diff --git a/tests-integration/tests/frequencies_test/update.rs b/tests-integration/tests/frequencies_test/update.rs index 06489d36..2c9ed0fa 100644 --- a/tests-integration/tests/frequencies_test/update.rs +++ b/tests-integration/tests/frequencies_test/update.rs @@ -642,3 +642,19 @@ fn test_estimated_size() { } assert_eq!(sketch.estimated_size(), 1800); } + +#[test] +fn estimated_size_is_shallow_for_generic_items() { + let mut small_item_sketch = FrequentItemsSketch::new(8).unwrap(); + small_item_sketch.update("key".to_string()); + + let mut large_item = String::with_capacity(4096); + large_item.push_str("key"); + let mut large_item_sketch = FrequentItemsSketch::new(8).unwrap(); + large_item_sketch.update(large_item); + + assert_eq!( + small_item_sketch.estimated_size(), + large_item_sketch.estimated_size() + ); +} diff --git a/tests-integration/tests/hll_test/update.rs b/tests-integration/tests/hll_test/update.rs index 50fd0b45..56ea21ad 100644 --- a/tests-integration/tests/hll_test/update.rs +++ b/tests-integration/tests/hll_test/update.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of_val; + use datasketches::error::ErrorKind; use datasketches::hll::HllSketch; use datasketches::hll::HllType; @@ -201,3 +203,18 @@ fn test_invalid_lg_k_high_returns_error() { let error = HllSketch::new(22, HllType::Hll8).unwrap_err(); assert_eq!(error.kind(), ErrorKind::InvalidArgument); } + +#[test] +fn estimated_size_tracks_retained_storage() { + let mut sketch = HllSketch::new(10, HllType::Hll8).unwrap(); + let initial_size = sketch.estimated_size(); + assert!(initial_size > size_of_val(&sketch)); + + sketch.update(0); + assert_eq!(sketch.estimated_size(), initial_size); + + for i in 1..1000 { + sketch.update(i); + } + assert!(sketch.estimated_size() > initial_size); +} diff --git a/tests-integration/tests/theta_test/sketch.rs b/tests-integration/tests/theta_test/sketch.rs index b900ab1b..f79c74c3 100644 --- a/tests-integration/tests/theta_test/sketch.rs +++ b/tests-integration/tests/theta_test/sketch.rs @@ -157,6 +157,7 @@ fn test_reset() { assert_that!(sketch.num_retained(), gt(32)); assert_that!(sketch.theta(), lt(1.0)); + let size_before_reset = sketch.estimated_size(); sketch.reset(); assert!(sketch.is_empty()); assert_eq!(sketch.estimate(), 0.0); @@ -165,6 +166,7 @@ fn test_reset() { assert!(!sketch.is_estimation_mode()); assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); + assert_eq!(sketch.estimated_size(), size_before_reset); } #[test] diff --git a/tests-integration/tests/theta_test/union.rs b/tests-integration/tests/theta_test/union.rs index 14df5bfe..1e297945 100644 --- a/tests-integration/tests/theta_test/union.rs +++ b/tests-integration/tests/theta_test/union.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of_val; + use datasketches::error::ErrorKind; use datasketches::theta::CompactThetaSketch; use datasketches::theta::ThetaSketch; @@ -740,3 +742,15 @@ fn test_union_estimated_size() { union.update(&sketch).unwrap(); assert_eq!(union.estimated_size(), 65616); } + +#[test] +fn compact_estimated_size_includes_retained_capacity() { + let sketch = sketch_with_range(5, 0, 1000); + let mut union = ThetaUnionBuilder::default().lg_k(5).build().unwrap(); + union.update(&sketch).unwrap(); + let result = union.to_sketch(true); + + assert_eq!(result.num_retained(), 32); + let logical_size = size_of_val(&result) + result.num_retained() * std::mem::size_of::(); + assert!(result.estimated_size() > logical_size); +} diff --git a/tests-integration/tests/tuple_test/sketch.rs b/tests-integration/tests/tuple_test/sketch.rs index 194060d2..5ca06c8d 100644 --- a/tests-integration/tests/tuple_test/sketch.rs +++ b/tests-integration/tests/tuple_test/sketch.rs @@ -131,6 +131,28 @@ fn custom_update_policy_accepts_multiple_value_representations() { ); } +#[test] +fn estimated_size_is_shallow_for_generic_summaries() { + let mut small_summary_sketch = TupleSketchBuilder::new(ArraySumPolicy { num_values: 1 }) + .build() + .unwrap(); + small_summary_sketch.update("key", [1.0]); + + let mut large_summary_sketch = TupleSketchBuilder::new(ArraySumPolicy { num_values: 1024 }) + .build() + .unwrap(); + large_summary_sketch.update("key", vec![1.0; 1024]); + + assert_eq!( + small_summary_sketch.estimated_size(), + large_summary_sketch.estimated_size() + ); + assert_eq!( + small_summary_sketch.compact(false).estimated_size(), + large_summary_sketch.compact(false).estimated_size() + ); +} + #[test] fn trim_and_reset_update_public_state() { let mut sketch = default_tuple_sketch_builder().lg_k(5).build().unwrap(); diff --git a/tests-integration/tests/tuple_test/union.rs b/tests-integration/tests/tuple_test/union.rs index 90b68b01..8daa2493 100644 --- a/tests-integration/tests/tuple_test/union.rs +++ b/tests-integration/tests/tuple_test/union.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of_val; + use datasketches::common::NumStdDev; use datasketches::error::ErrorKind; use datasketches::tuple::DefaultUnionPolicy; @@ -192,3 +194,16 @@ fn union_estimated_size_grows_with_updates() { union.update(&sketch).unwrap(); assert_eq!(union.estimated_size(), 131152); } + +#[test] +fn compact_estimated_size_includes_retained_capacity() { + let sketch = tuple_sketch_with_range(0, 1000); + let mut union = default_union_builder().lg_k(5).build().unwrap(); + union.update(&sketch).unwrap(); + let result = union.to_sketch(true); + + assert_eq!(result.num_retained(), 32); + let entry_size = size_of_val(result.iter().next().unwrap()); + let logical_size = size_of_val(&result) + result.num_retained() * entry_size; + assert!(result.estimated_size() > logical_size); +} From 06bf53191dcd168150c5b9bb86ce22953c317d2b Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 24 Oct 2026 10:12:11 +0800 Subject: [PATCH 2/4] refactor: base estimated_size on heap storage --- CHANGELOG.md | 1 + datasketches/src/bloom/sketch.rs | 6 ++--- datasketches/src/countmin/sketch.rs | 8 +++---- datasketches/src/cpc/sketch.rs | 10 ++++---- datasketches/src/cpc/union.rs | 12 ++++------ datasketches/src/frequencies/sketch.rs | 10 ++++---- datasketches/src/hll/sketch.rs | 10 ++++---- datasketches/src/hll/union.rs | 7 +++--- datasketches/src/tdigest/sketch.rs | 12 +++++----- .../src/thetafamily/theta/intersection.rs | 6 ++--- datasketches/src/thetafamily/theta/sketch.rs | 12 +++++----- datasketches/src/thetafamily/theta/union.rs | 6 ++--- .../src/thetafamily/tuple/intersection.rs | 12 ++++++---- datasketches/src/thetafamily/tuple/sketch.rs | 23 ++++++++++--------- datasketches/src/thetafamily/tuple/union.rs | 12 ++++++---- tests-integration/tests/bloom_test/sketch.rs | 3 +-- .../tests/countmin_test/sketch.rs | 3 +-- tests-integration/tests/cpc_test/union.rs | 4 ++-- tests-integration/tests/cpc_test/update.rs | 4 +--- .../tests/frequencies_test/update.rs | 12 ++++++++-- tests-integration/tests/hll_test/union.rs | 4 ++-- tests-integration/tests/hll_test/update.rs | 6 ++--- .../tests/tdigest_test/sketch.rs | 6 ++--- .../tests/theta_test/intersection.rs | 4 ++-- tests-integration/tests/theta_test/union.rs | 8 +++---- .../tests/tuple_test/intersection.rs | 4 ++-- tests-integration/tests/tuple_test/union.rs | 8 +++---- 27 files changed, 103 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aeb2a9a..ff491102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All significant changes to this project will be documented in this file. * `ReqSketch` queries now use `datasketches::common::SearchCriteria` instead of `datasketches::req::SearchCriteria`. `SearchCriteria` no longer implements `Default`; explicitly choose `Inclusive` or `Exclusive`. * `ReqSketch` and `req::SortedView` queries now distinguish empty sketches from invalid arguments. `rank` returns `Option`; `quantile`, `quantiles`, `cdf`, and `pmf`, where available, return `Result, Error>`. Handle `None` for valid queries on an empty sketch, and propagate or handle `InvalidArgument` for invalid ranks or split points, which are checked before emptiness. * `TDigest` and `TDigestMut` rank, quantile, CDF, and PMF queries now return `Result, Error>`. Invalid arguments return `InvalidArgument` instead of panicking, including on empty digests. Propagate or handle the `Result` before handling `None` for an empty digest; `min_value` and `max_value` still return `Option`. +* `estimated_size` methods now estimate retained heap storage without adding the sketch value's inline size. To reproduce the previous inline-plus-heap result, add `size_of_val(&sketch)` to the new estimate. Heap allocations owned by generic items, summaries, or policies remain excluded. ### New features diff --git a/datasketches/src/bloom/sketch.rs b/datasketches/src/bloom/sketch.rs index 6879e140..45da60ff 100644 --- a/datasketches/src/bloom/sketch.rs +++ b/datasketches/src/bloom/sketch.rs @@ -639,12 +639,12 @@ impl BloomFilter { was_set } - /// Returns the estimated in-memory size of the filter in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the filter's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.bit_array.len() * size_of::() + self.bit_array.len() * size_of::() } } diff --git a/datasketches/src/countmin/sketch.rs b/datasketches/src/countmin/sketch.rs index 5e24eb6e..e04c01e6 100644 --- a/datasketches/src/countmin/sketch.rs +++ b/datasketches/src/countmin/sketch.rs @@ -495,14 +495,12 @@ impl CountMinSketch { Ok(sketch) } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() - + self.counts.capacity() * size_of::() - + self.hash_seeds.capacity() * size_of::() + self.counts.capacity() * size_of::() + self.hash_seeds.capacity() * size_of::() } fn make(num_hashes: u8, num_buckets: u32, seed: u64, seed_hash: u16, entries: usize) -> Self { diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs index 1c59e7de..1531e5b3 100644 --- a/datasketches/src/cpc/sketch.rs +++ b/datasketches/src/cpc/sketch.rs @@ -459,19 +459,17 @@ impl CpcSketch { matrix } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - let heap_size = self.sliding_window.capacity() + self.sliding_window.capacity() + self .surprising_value_table .as_ref() .map(|t| t.estimated_size()) - .unwrap_or(0); - - size_of::() + heap_size + .unwrap_or(0) } } diff --git a/datasketches/src/cpc/union.rs b/datasketches/src/cpc/union.rs index ca542bbf..98a5c18d 100644 --- a/datasketches/src/cpc/union.rs +++ b/datasketches/src/cpc/union.rs @@ -347,17 +347,15 @@ impl CpcUnion { } } - /// Returns the estimated in-memory size of the union in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the union's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - // The state's inline size is already covered by size_of::(). - let heap_size = match &self.state { - UnionState::Accumulator(sketch) => sketch.estimated_size() - size_of::(), + match &self.state { + UnionState::Accumulator(sketch) => sketch.estimated_size(), UnionState::BitMatrix(matrix) => matrix.capacity() * size_of::(), - }; - size_of::() + heap_size + } } } diff --git a/datasketches/src/frequencies/sketch.rs b/datasketches/src/frequencies/sketch.rs index 946519f6..5baa429c 100644 --- a/datasketches/src/frequencies/sketch.rs +++ b/datasketches/src/frequencies/sketch.rs @@ -323,14 +323,14 @@ impl FrequentItemsSketch { self.hash_map.lg_length() } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Inline representations of items are included, but heap allocations owned by those items are - /// not recursively inspected. + /// Storage for items held inside owned containers is included, but heap allocations owned by + /// those items are not recursively inspected. pub fn estimated_size(&self) -> usize { - size_of::() + self.hash_map.estimated_size() + self.hash_map.estimated_size() } /// Updates the sketch with a count of one. diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 464efae0..1261a733 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -470,20 +470,18 @@ impl HllSketch { } } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - let heap_size = match &self.mode { + match &self.mode { Mode::List { list, .. } => list.container().estimated_size(), Mode::Set { set, .. } => set.container().estimated_size(), Mode::Array4(arr) => arr.estimated_size(), Mode::Array6(arr) => arr.estimated_size(), Mode::Array8(arr) => arr.estimated_size(), - }; - - size_of::() + heap_size + } } } diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index 7f262e43..b87ae0bc 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -365,13 +365,12 @@ impl HllUnion { self.gadget.lower_bound(num_std_dev) } - /// Returns the estimated in-memory size of the union in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the union's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - // The gadget's inline size is already covered by size_of::(). - size_of::() - size_of::() + self.gadget.estimated_size() + self.gadget.estimated_size() } } diff --git a/datasketches/src/tdigest/sketch.rs b/datasketches/src/tdigest/sketch.rs index 6f826829..121c8eb4 100644 --- a/datasketches/src/tdigest/sketch.rs +++ b/datasketches/src/tdigest/sketch.rs @@ -1013,12 +1013,12 @@ impl TDigestMut { centroids.shrink_to(target_capacity); } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.buffer.estimated_size() + self.buffer.estimated_size() } } @@ -1634,12 +1634,12 @@ impl TDigest { ) } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.centroids.capacity() * size_of::() + self.centroids.capacity() * size_of::() } } diff --git a/datasketches/src/thetafamily/theta/intersection.rs b/datasketches/src/thetafamily/theta/intersection.rs index 994e8ed8..45e5e22b 100644 --- a/datasketches/src/thetafamily/theta/intersection.rs +++ b/datasketches/src/thetafamily/theta/intersection.rs @@ -78,12 +78,12 @@ impl ThetaIntersection { self.state.has_result() } - /// Returns the estimated in-memory size of the intersection in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the intersection's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } /// Returns the current intersection as a compact theta sketch. diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index 7be5b456..9476acc8 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -427,12 +427,12 @@ impl ThetaSketch { .expect("theta should always be valid") } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.table.estimated_size() + self.table.estimated_size() } } @@ -1030,12 +1030,12 @@ impl CompactThetaSketch { Ok(Self::from_compact_state(compact_state)) } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.compact_state.retained_entries_capacity() * size_of::() + self.compact_state.retained_entries_capacity() * size_of::() } } diff --git a/datasketches/src/thetafamily/theta/union.rs b/datasketches/src/thetafamily/theta/union.rs index 31f64466..7f5ea314 100644 --- a/datasketches/src/thetafamily/theta/union.rs +++ b/datasketches/src/thetafamily/theta/union.rs @@ -64,12 +64,12 @@ impl ThetaUnion { self.state.reset(); } - /// Returns the estimated in-memory size of the union in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the union's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } } diff --git a/datasketches/src/thetafamily/tuple/intersection.rs b/datasketches/src/thetafamily/tuple/intersection.rs index ce1bea95..d8e57661 100644 --- a/datasketches/src/thetafamily/tuple/intersection.rs +++ b/datasketches/src/thetafamily/tuple/intersection.rs @@ -138,14 +138,16 @@ where self.state.has_result() } - /// Returns the estimated in-memory size of the intersection in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the intersection's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Inline representations of summaries and the policy are included, but heap allocations owned - /// by those values are not recursively inspected. + /// Storage for summaries held inside owned containers is included, but heap allocations owned + /// by those summaries or the policy are not recursively inspected. + /// The policy's inline representation is excluded because a custom policy can have an + /// arbitrarily large inline payload. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } /// Returns the current intersection as a compact Tuple sketch. diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index 2ce99ddc..bbbb9471 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -404,14 +404,16 @@ where .expect("theta should always be valid") } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Inline representations of summaries and the policy are included, but heap allocations owned - /// by those values are not recursively inspected. + /// Storage for summaries held inside owned containers is included, but heap allocations owned + /// by those summaries or the policy are not recursively inspected. + /// The policy's inline representation is excluded because a custom policy can have an + /// arbitrarily large inline payload. pub fn estimated_size(&self) -> usize { - size_of::() + self.table.estimated_size() + self.table.estimated_size() } } @@ -546,15 +548,14 @@ impl CompactTupleSketch { .expect("compact theta should always be valid") } - /// Returns the estimated in-memory size of the sketch in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Inline representations of summaries are included, but heap allocations owned by those - /// summaries are not recursively inspected. + /// Storage for summaries held inside owned containers is included, but heap allocations owned + /// by those summaries are not recursively inspected. pub fn estimated_size(&self) -> usize { - size_of::() - + self.compact_state.retained_entries_capacity() * size_of::>() + self.compact_state.retained_entries_capacity() * size_of::>() } fn preamble_longs(&self) -> u8 { diff --git a/datasketches/src/thetafamily/tuple/union.rs b/datasketches/src/thetafamily/tuple/union.rs index 39418e62..54c00a0b 100644 --- a/datasketches/src/thetafamily/tuple/union.rs +++ b/datasketches/src/thetafamily/tuple/union.rs @@ -112,14 +112,16 @@ where self.state.reset(); } - /// Returns the estimated in-memory size of the union in bytes, including its inline - /// representation and the retained heap allocations it owns. + /// Returns an estimate of the union's memory usage in bytes, based on the retained heap + /// allocations it owns. /// /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Inline representations of summaries and the policy are included, but heap allocations owned - /// by those values are not recursively inspected. + /// Storage for summaries held inside owned containers is included, but heap allocations owned + /// by those summaries or the policy are not recursively inspected. + /// The policy's inline representation is excluded because a custom policy can have an + /// arbitrarily large inline payload. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } } diff --git a/tests-integration/tests/bloom_test/sketch.rs b/tests-integration/tests/bloom_test/sketch.rs index 4bc3a3d2..06b461e4 100644 --- a/tests-integration/tests/bloom_test/sketch.rs +++ b/tests-integration/tests/bloom_test/sketch.rs @@ -16,7 +16,6 @@ // under the License. use std::mem::size_of; -use std::mem::size_of_val; use datasketches::bloom::BloomFilter; use datasketches::bloom::BloomFilterBuilder; @@ -217,7 +216,7 @@ fn test_requested_size_rounds_to_word_boundary() { #[test] fn test_estimated_size() { let filter = BloomFilterBuilder::with_size(65, 3).build().unwrap(); - let expected_size = size_of_val(&filter) + filter.capacity() / 64 * size_of::(); + let expected_size = filter.capacity() / 64 * size_of::(); assert_eq!(filter.estimated_size(), expected_size); } diff --git a/tests-integration/tests/countmin_test/sketch.rs b/tests-integration/tests/countmin_test/sketch.rs index 15cb645f..68a5f143 100644 --- a/tests-integration/tests/countmin_test/sketch.rs +++ b/tests-integration/tests/countmin_test/sketch.rs @@ -16,7 +16,6 @@ // under the License. use std::mem::size_of; -use std::mem::size_of_val; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -447,7 +446,7 @@ fn test_increment_single_key_like_rust_count_min_sketch() { #[test] fn test_estimated_size() { let mut sketch = CountMinSketch::::new(4, 128).unwrap(); - let expected_size = size_of_val(&sketch) + 4 * 128 * size_of::() + 4 * size_of::(); + let expected_size = 4 * 128 * size_of::() + 4 * size_of::(); assert_eq!(sketch.estimated_size(), expected_size); // The backing tables are allocated up front; updates do not grow the sketch. diff --git a/tests-integration/tests/cpc_test/union.rs b/tests-integration/tests/cpc_test/union.rs index 7b280343..d72a7d4d 100644 --- a/tests-integration/tests/cpc_test/union.rs +++ b/tests-integration/tests/cpc_test/union.rs @@ -183,12 +183,12 @@ fn test_lg_k_too_large_returns_error() { #[test] fn test_union_estimated_size() { let mut union = CpcUnion::new(11).unwrap(); - assert_eq!(union.estimated_size(), 112); + assert_eq!(union.estimated_size(), 0); let mut sketch = CpcSketch::new(11).unwrap(); for i in 0..1000 { sketch.update(i); } union.update(&sketch).unwrap(); - assert_eq!(union.estimated_size(), 16496); + assert_eq!(union.estimated_size(), 16384); } diff --git a/tests-integration/tests/cpc_test/update.rs b/tests-integration/tests/cpc_test/update.rs index 680f50e8..34ef5ea7 100644 --- a/tests-integration/tests/cpc_test/update.rs +++ b/tests-integration/tests/cpc_test/update.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of_val; - use datasketches::common::NumStdDev; use datasketches::cpc::CpcSketch; use datasketches::error::ErrorKind; @@ -74,7 +72,7 @@ fn test_max_serialized_bytes_validates_lg_k() { fn test_estimated_size() { let mut sketch = CpcSketch::new(11).unwrap(); let empty_size = sketch.estimated_size(); - assert_eq!(empty_size, size_of_val(&sketch)); + assert_eq!(empty_size, 0); for i in 0..10_000 { sketch.update(i); diff --git a/tests-integration/tests/frequencies_test/update.rs b/tests-integration/tests/frequencies_test/update.rs index 2c9ed0fa..86e8322e 100644 --- a/tests-integration/tests/frequencies_test/update.rs +++ b/tests-integration/tests/frequencies_test/update.rs @@ -633,14 +633,22 @@ fn test_map_size_above_cross_language_limit_returns_error() { #[test] fn test_estimated_size() { + let expected_size = |sketch: &FrequentItemsSketch| { + let slots = 1usize << sketch.lg_cur_map_size(); + slots + * (std::mem::size_of::>() + + std::mem::size_of::() + + std::mem::size_of::()) + }; + let mut sketch: FrequentItemsSketch = FrequentItemsSketch::new(64).unwrap(); - assert_eq!(sketch.estimated_size(), 344); + assert_eq!(sketch.estimated_size(), expected_size(&sketch)); // The internal map grows from its starting size up to the maximum size. for i in 0..100 { sketch.update(i); } - assert_eq!(sketch.estimated_size(), 1800); + assert_eq!(sketch.estimated_size(), expected_size(&sketch)); } #[test] diff --git a/tests-integration/tests/hll_test/union.rs b/tests-integration/tests/hll_test/union.rs index 7a6409a0..c61a1716 100644 --- a/tests-integration/tests/hll_test/union.rs +++ b/tests-integration/tests/hll_test/union.rs @@ -710,14 +710,14 @@ fn test_union_validation() { #[test] fn test_union_estimated_size() { let mut union = HllUnion::new(10).unwrap(); - assert_eq!(union.estimated_size(), 128); + assert_eq!(union.estimated_size(), 32); let mut sketch = HllSketch::new(10, HllType::Hll8).unwrap(); for i in 0..1000 { sketch.update(i); } union.update(&sketch); - assert_eq!(union.estimated_size(), 1120); + assert_eq!(union.estimated_size(), 1024); } #[test] diff --git a/tests-integration/tests/hll_test/update.rs b/tests-integration/tests/hll_test/update.rs index 56ea21ad..5ec6ac89 100644 --- a/tests-integration/tests/hll_test/update.rs +++ b/tests-integration/tests/hll_test/update.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of_val; - use datasketches::error::ErrorKind; use datasketches::hll::HllSketch; use datasketches::hll::HllType; @@ -208,7 +206,7 @@ fn test_invalid_lg_k_high_returns_error() { fn estimated_size_tracks_retained_storage() { let mut sketch = HllSketch::new(10, HllType::Hll8).unwrap(); let initial_size = sketch.estimated_size(); - assert!(initial_size > size_of_val(&sketch)); + assert_eq!(initial_size, 32); sketch.update(0); assert_eq!(sketch.estimated_size(), initial_size); @@ -216,5 +214,5 @@ fn estimated_size_tracks_retained_storage() { for i in 1..1000 { sketch.update(i); } - assert!(sketch.estimated_size() > initial_size); + assert_eq!(sketch.estimated_size(), 1024); } diff --git a/tests-integration/tests/tdigest_test/sketch.rs b/tests-integration/tests/tdigest_test/sketch.rs index 86bc45fb..4eba82d6 100644 --- a/tests-integration/tests/tdigest_test/sketch.rs +++ b/tests-integration/tests/tdigest_test/sketch.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -119,15 +118,14 @@ fn test_estimated_size_reuses_buffer_after_compression() { const TARGET_CENTROIDS: usize = 410; const MAX_UNMERGED: usize = TARGET_CENTROIDS * 4; - let inline_size = size_of::(); let mut tdigest = TDigestMut::new(K).unwrap(); - assert_eq!(tdigest.estimated_size(), inline_size); + assert_eq!(tdigest.estimated_size(), 0); for value in 0..MAX_UNMERGED { tdigest.update(value as f64); } let size_before_compression = tdigest.estimated_size(); - assert!(size_before_compression > inline_size); + assert!(size_before_compression > 0); tdigest.rank(0.5).unwrap(); assert!(tdigest.estimated_size() <= size_before_compression); diff --git a/tests-integration/tests/theta_test/intersection.rs b/tests-integration/tests/theta_test/intersection.rs index 87f545fe..f6068a13 100644 --- a/tests-integration/tests/theta_test/intersection.rs +++ b/tests-integration/tests/theta_test/intersection.rs @@ -326,9 +326,9 @@ fn test_seed_mismatch_non_empty_returns_error() { #[test] fn test_intersection_estimated_size() { let mut intersection = ThetaIntersection::default(); - assert_eq!(intersection.estimated_size(), 72); + assert_eq!(intersection.estimated_size(), 0); let sketch = sketch_with_range(0, 1000); intersection.update(&sketch).unwrap(); - assert_eq!(intersection.estimated_size(), 16456); + assert_eq!(intersection.estimated_size(), 16384); } diff --git a/tests-integration/tests/theta_test/union.rs b/tests-integration/tests/theta_test/union.rs index 1e297945..0c02df99 100644 --- a/tests-integration/tests/theta_test/union.rs +++ b/tests-integration/tests/theta_test/union.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of_val; - use datasketches::error::ErrorKind; use datasketches::theta::CompactThetaSketch; use datasketches::theta::ThetaSketch; @@ -736,11 +734,11 @@ fn test_corner_case_union_states() { #[test] fn test_union_estimated_size() { let mut union = ThetaUnionBuilder::default().build().unwrap(); - assert_eq!(union.estimated_size(), 1104); + assert_eq!(union.estimated_size(), 1024); let sketch = sketch_with_range(12, 0, 1000); union.update(&sketch).unwrap(); - assert_eq!(union.estimated_size(), 65616); + assert_eq!(union.estimated_size(), 65536); } #[test] @@ -751,6 +749,6 @@ fn compact_estimated_size_includes_retained_capacity() { let result = union.to_sketch(true); assert_eq!(result.num_retained(), 32); - let logical_size = size_of_val(&result) + result.num_retained() * std::mem::size_of::(); + let logical_size = result.num_retained() * std::mem::size_of::(); assert!(result.estimated_size() > logical_size); } diff --git a/tests-integration/tests/tuple_test/intersection.rs b/tests-integration/tests/tuple_test/intersection.rs index 9caa0eb8..d01381ab 100644 --- a/tests-integration/tests/tuple_test/intersection.rs +++ b/tests-integration/tests/tuple_test/intersection.rs @@ -185,9 +185,9 @@ fn estimation_bounds_cover_the_true_intersection() { #[test] fn intersection_estimated_size_grows_with_updates() { let mut intersection = TupleIntersection::new(SumPolicy); - assert_eq!(intersection.estimated_size(), 72); + assert_eq!(intersection.estimated_size(), 0); let sketch = tuple_sketch_with_range(0, 1000); intersection.update(&sketch).unwrap(); - assert_eq!(intersection.estimated_size(), 32840); + assert_eq!(intersection.estimated_size(), 32768); } diff --git a/tests-integration/tests/tuple_test/union.rs b/tests-integration/tests/tuple_test/union.rs index 8daa2493..14af4f91 100644 --- a/tests-integration/tests/tuple_test/union.rs +++ b/tests-integration/tests/tuple_test/union.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of_val; - use datasketches::common::NumStdDev; use datasketches::error::ErrorKind; use datasketches::tuple::DefaultUnionPolicy; @@ -188,11 +186,11 @@ fn estimation_bounds_cover_the_true_union() { #[test] fn union_estimated_size_grows_with_updates() { let mut union = default_union_builder().build().unwrap(); - assert_eq!(union.estimated_size(), 2128); + assert_eq!(union.estimated_size(), 2048); let sketch = tuple_sketch_with_range(0, 1000); union.update(&sketch).unwrap(); - assert_eq!(union.estimated_size(), 131152); + assert_eq!(union.estimated_size(), 131072); } #[test] @@ -204,6 +202,6 @@ fn compact_estimated_size_includes_retained_capacity() { assert_eq!(result.num_retained(), 32); let entry_size = size_of_val(result.iter().next().unwrap()); - let logical_size = size_of_val(&result) + result.num_retained() * entry_size; + let logical_size = result.num_retained() * entry_size; assert!(result.estimated_size() > logical_size); } From b2b044fa84e618a7fafc672d4abd01a3760798a4 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 24 Oct 2026 12:11:28 +0800 Subject: [PATCH 3/4] docs: keep estimated_size contract implementation-neutral --- datasketches/src/bloom/sketch.rs | 5 +---- datasketches/src/countmin/sketch.rs | 5 +---- datasketches/src/cpc/sketch.rs | 5 +---- datasketches/src/cpc/union.rs | 5 +---- datasketches/src/frequencies/sketch.rs | 7 +------ datasketches/src/hll/sketch.rs | 5 +---- datasketches/src/hll/union.rs | 5 +---- datasketches/src/tdigest/sketch.rs | 10 ++-------- .../src/thetafamily/theta/intersection.rs | 5 +---- datasketches/src/thetafamily/theta/sketch.rs | 10 ++-------- datasketches/src/thetafamily/theta/union.rs | 5 +---- .../src/thetafamily/tuple/intersection.rs | 9 +-------- datasketches/src/thetafamily/tuple/sketch.rs | 16 ++-------------- datasketches/src/thetafamily/tuple/union.rs | 9 +-------- 14 files changed, 17 insertions(+), 84 deletions(-) diff --git a/datasketches/src/bloom/sketch.rs b/datasketches/src/bloom/sketch.rs index 45da60ff..245a7262 100644 --- a/datasketches/src/bloom/sketch.rs +++ b/datasketches/src/bloom/sketch.rs @@ -639,10 +639,7 @@ impl BloomFilter { was_set } - /// Returns an estimate of the filter's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the filter's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bit_array.len() * size_of::() } diff --git a/datasketches/src/countmin/sketch.rs b/datasketches/src/countmin/sketch.rs index e04c01e6..fe346ccb 100644 --- a/datasketches/src/countmin/sketch.rs +++ b/datasketches/src/countmin/sketch.rs @@ -495,10 +495,7 @@ impl CountMinSketch { Ok(sketch) } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.counts.capacity() * size_of::() + self.hash_seeds.capacity() * size_of::() } diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs index 1531e5b3..3222cff3 100644 --- a/datasketches/src/cpc/sketch.rs +++ b/datasketches/src/cpc/sketch.rs @@ -459,10 +459,7 @@ impl CpcSketch { matrix } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.sliding_window.capacity() + self diff --git a/datasketches/src/cpc/union.rs b/datasketches/src/cpc/union.rs index 98a5c18d..abd473dd 100644 --- a/datasketches/src/cpc/union.rs +++ b/datasketches/src/cpc/union.rs @@ -347,10 +347,7 @@ impl CpcUnion { } } - /// Returns an estimate of the union's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { match &self.state { UnionState::Accumulator(sketch) => sketch.estimated_size(), diff --git a/datasketches/src/frequencies/sketch.rs b/datasketches/src/frequencies/sketch.rs index 5baa429c..617b98ec 100644 --- a/datasketches/src/frequencies/sketch.rs +++ b/datasketches/src/frequencies/sketch.rs @@ -323,12 +323,7 @@ impl FrequentItemsSketch { self.hash_map.lg_length() } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Storage for items held inside owned containers is included, but heap allocations owned by - /// those items are not recursively inspected. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.hash_map.estimated_size() } diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 1261a733..910a730c 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -470,10 +470,7 @@ impl HllSketch { } } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { match &self.mode { Mode::List { list, .. } => list.container().estimated_size(), diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index b87ae0bc..b245fa71 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -365,10 +365,7 @@ impl HllUnion { self.gadget.lower_bound(num_std_dev) } - /// Returns an estimate of the union's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.gadget.estimated_size() } diff --git a/datasketches/src/tdigest/sketch.rs b/datasketches/src/tdigest/sketch.rs index 121c8eb4..5609bfbf 100644 --- a/datasketches/src/tdigest/sketch.rs +++ b/datasketches/src/tdigest/sketch.rs @@ -1013,10 +1013,7 @@ impl TDigestMut { centroids.shrink_to(target_capacity); } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.buffer.estimated_size() } @@ -1634,10 +1631,7 @@ impl TDigest { ) } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.centroids.capacity() * size_of::() } diff --git a/datasketches/src/thetafamily/theta/intersection.rs b/datasketches/src/thetafamily/theta/intersection.rs index 45e5e22b..137891d8 100644 --- a/datasketches/src/thetafamily/theta/intersection.rs +++ b/datasketches/src/thetafamily/theta/intersection.rs @@ -78,10 +78,7 @@ impl ThetaIntersection { self.state.has_result() } - /// Returns an estimate of the intersection's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the intersection's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.state.estimated_size() } diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index 9476acc8..4ec71b79 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -427,10 +427,7 @@ impl ThetaSketch { .expect("theta should always be valid") } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.table.estimated_size() } @@ -1030,10 +1027,7 @@ impl CompactThetaSketch { Ok(Self::from_compact_state(compact_state)) } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.compact_state.retained_entries_capacity() * size_of::() } diff --git a/datasketches/src/thetafamily/theta/union.rs b/datasketches/src/thetafamily/theta/union.rs index 7f5ea314..1b26d7a2 100644 --- a/datasketches/src/thetafamily/theta/union.rs +++ b/datasketches/src/thetafamily/theta/union.rs @@ -64,10 +64,7 @@ impl ThetaUnion { self.state.reset(); } - /// Returns an estimate of the union's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.state.estimated_size() } diff --git a/datasketches/src/thetafamily/tuple/intersection.rs b/datasketches/src/thetafamily/tuple/intersection.rs index d8e57661..58dce8fd 100644 --- a/datasketches/src/thetafamily/tuple/intersection.rs +++ b/datasketches/src/thetafamily/tuple/intersection.rs @@ -138,14 +138,7 @@ where self.state.has_result() } - /// Returns an estimate of the intersection's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Storage for summaries held inside owned containers is included, but heap allocations owned - /// by those summaries or the policy are not recursively inspected. - /// The policy's inline representation is excluded because a custom policy can have an - /// arbitrarily large inline payload. + /// Returns an estimate of the intersection's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.state.estimated_size() } diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index bbbb9471..d3aa9b01 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -404,14 +404,7 @@ where .expect("theta should always be valid") } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Storage for summaries held inside owned containers is included, but heap allocations owned - /// by those summaries or the policy are not recursively inspected. - /// The policy's inline representation is excluded because a custom policy can have an - /// arbitrarily large inline payload. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.table.estimated_size() } @@ -548,12 +541,7 @@ impl CompactTupleSketch { .expect("compact theta should always be valid") } - /// Returns an estimate of the sketch's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Storage for summaries held inside owned containers is included, but heap allocations owned - /// by those summaries are not recursively inspected. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.compact_state.retained_entries_capacity() * size_of::>() } diff --git a/datasketches/src/thetafamily/tuple/union.rs b/datasketches/src/thetafamily/tuple/union.rs index 54c00a0b..029aeba7 100644 --- a/datasketches/src/thetafamily/tuple/union.rs +++ b/datasketches/src/thetafamily/tuple/union.rs @@ -112,14 +112,7 @@ where self.state.reset(); } - /// Returns an estimate of the union's memory usage in bytes, based on the retained heap - /// allocations it owns. - /// - /// The estimate excludes allocator bookkeeping, temporary allocations, and serialized size. - /// Storage for summaries held inside owned containers is included, but heap allocations owned - /// by those summaries or the policy are not recursively inspected. - /// The policy's inline representation is excluded because a custom policy can have an - /// arbitrarily large inline payload. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { self.state.estimated_size() } From 4811432920aade4339a94c87fa8649c18991d511 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 24 Oct 2026 23:03:56 +0800 Subject: [PATCH 4/4] refactor: simplify estimated_size accounting --- CHANGELOG.md | 2 +- datasketches/src/cpc/pair_table.rs | 2 +- .../reverse_purge_item_hash_map.rs | 2 +- datasketches/src/hll/array4.rs | 2 +- datasketches/src/hll/array6.rs | 2 +- datasketches/src/hll/array8.rs | 2 +- datasketches/src/hll/aux_map.rs | 2 +- datasketches/src/hll/container.rs | 2 +- datasketches/src/tdigest/sketch.rs | 2 +- .../src/thetafamily/common/hash_table.rs | 2 +- .../src/thetafamily/common/intersection.rs | 4 +- datasketches/src/thetafamily/common/union.rs | 4 +- datasketches/src/thetafamily/tuple/sketch.rs | 2 +- tests-integration/tests/bloom_test/sketch.rs | 5 +-- .../tests/countmin_test/sketch.rs | 6 +-- tests-integration/tests/cpc_test/update.rs | 2 +- .../tests/frequencies_test/update.rs | 12 +----- tests-integration/tests/theta_test/sketch.rs | 4 +- tests-integration/tests/theta_test/union.rs | 3 +- tests-integration/tests/tuple_test/sketch.rs | 39 +++++++++++++++++++ tests-integration/tests/tuple_test/union.rs | 4 +- 21 files changed, 64 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff491102..733a7730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ All significant changes to this project will be documented in this file. * `ReqSketch` queries now use `datasketches::common::SearchCriteria` instead of `datasketches::req::SearchCriteria`. `SearchCriteria` no longer implements `Default`; explicitly choose `Inclusive` or `Exclusive`. * `ReqSketch` and `req::SortedView` queries now distinguish empty sketches from invalid arguments. `rank` returns `Option`; `quantile`, `quantiles`, `cdf`, and `pmf`, where available, return `Result, Error>`. Handle `None` for valid queries on an empty sketch, and propagate or handle `InvalidArgument` for invalid ranks or split points, which are checked before emptiness. * `TDigest` and `TDigestMut` rank, quantile, CDF, and PMF queries now return `Result, Error>`. Invalid arguments return `InvalidArgument` instead of panicking, including on empty digests. Propagate or handle the `Result` before handling `None` for an empty digest; `min_value` and `max_value` still return `Option`. -* `estimated_size` methods now estimate retained heap storage without adding the sketch value's inline size. To reproduce the previous inline-plus-heap result, add `size_of_val(&sketch)` to the new estimate. Heap allocations owned by generic items, summaries, or policies remain excluded. +* `estimated_size` now returns an implementation-dependent estimate of memory usage. Returned values may change as implementations evolve and should not be treated as exact totals or a stable accounting contract. ### New features diff --git a/datasketches/src/cpc/pair_table.rs b/datasketches/src/cpc/pair_table.rs index 72782494..d8ab5d81 100644 --- a/datasketches/src/cpc/pair_table.rs +++ b/datasketches/src/cpc/pair_table.rs @@ -258,7 +258,7 @@ impl PairTable { } } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.slots.capacity() * size_of::() } diff --git a/datasketches/src/frequencies/reverse_purge_item_hash_map.rs b/datasketches/src/frequencies/reverse_purge_item_hash_map.rs index f724d3f2..7f430f65 100644 --- a/datasketches/src/frequencies/reverse_purge_item_hash_map.rs +++ b/datasketches/src/frequencies/reverse_purge_item_hash_map.rs @@ -197,7 +197,7 @@ impl ReversePurgeItemHashMap { self.num_active } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.keys.capacity() * size_of::>() + self.values.capacity() * size_of::() diff --git a/datasketches/src/hll/array4.rs b/datasketches/src/hll/array4.rs index a008f163..4190a3f7 100644 --- a/datasketches/src/hll/array4.rs +++ b/datasketches/src/hll/array4.rs @@ -419,7 +419,7 @@ impl Array4 { bytes.into_bytes() } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() + self diff --git a/datasketches/src/hll/array6.rs b/datasketches/src/hll/array6.rs index 7c05b144..636da15d 100644 --- a/datasketches/src/hll/array6.rs +++ b/datasketches/src/hll/array6.rs @@ -233,7 +233,7 @@ impl Array6 { bytes.into_bytes() } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() } diff --git a/datasketches/src/hll/array8.rs b/datasketches/src/hll/array8.rs index 4030a5b0..1269d171 100644 --- a/datasketches/src/hll/array8.rs +++ b/datasketches/src/hll/array8.rs @@ -278,7 +278,7 @@ impl Array8 { bytes.into_bytes() } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() } diff --git a/datasketches/src/hll/aux_map.rs b/datasketches/src/hll/aux_map.rs index 09541974..2ce25a39 100644 --- a/datasketches/src/hll/aux_map.rs +++ b/datasketches/src/hll/aux_map.rs @@ -192,7 +192,7 @@ impl AuxMap { }) } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.entries.len() * size_of::() } diff --git a/datasketches/src/hll/container.rs b/datasketches/src/hll/container.rs index f55ee8f1..bc8cab68 100644 --- a/datasketches/src/hll/container.rs +++ b/datasketches/src/hll/container.rs @@ -121,7 +121,7 @@ impl Container { self.coupons.iter().filter(|&&c| !c.is_empty()).copied() } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.coupons.len() * size_of::() } diff --git a/datasketches/src/tdigest/sketch.rs b/datasketches/src/tdigest/sketch.rs index 5609bfbf..9bc7b032 100644 --- a/datasketches/src/tdigest/sketch.rs +++ b/datasketches/src/tdigest/sketch.rs @@ -156,7 +156,7 @@ impl TDigestBuffer { self.centroids } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. fn estimated_size(&self) -> usize { self.centroids.capacity() * size_of::() } diff --git a/datasketches/src/thetafamily/common/hash_table.rs b/datasketches/src/thetafamily/common/hash_table.rs index 1d86bd66..e0b7523e 100644 --- a/datasketches/src/thetafamily/common/hash_table.rs +++ b/datasketches/src/thetafamily/common/hash_table.rs @@ -344,7 +344,7 @@ where }) } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.entries.capacity() * size_of::>() } diff --git a/datasketches/src/thetafamily/common/intersection.rs b/datasketches/src/thetafamily/common/intersection.rs index ca5e7f20..535bebac 100644 --- a/datasketches/src/thetafamily/common/intersection.rs +++ b/datasketches/src/thetafamily/common/intersection.rs @@ -234,9 +234,9 @@ where self.result_state != IntersectionResultState::Uninitialized } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { - self.table.estimated_size() + size_of::

() + self.table.estimated_size() } /// Returns the current intersection as canonical compact-sketch state. diff --git a/datasketches/src/thetafamily/common/union.rs b/datasketches/src/thetafamily/common/union.rs index 4c585bd2..4ad3b466 100644 --- a/datasketches/src/thetafamily/common/union.rs +++ b/datasketches/src/thetafamily/common/union.rs @@ -150,8 +150,8 @@ where self.result_theta = None; } - /// Returns the estimated size of retained heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { - self.table.estimated_size() + size_of::

() + self.table.estimated_size() } } diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index d3aa9b01..0fee8218 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -406,7 +406,7 @@ where /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - self.table.estimated_size() + size_of::

() + self.table.estimated_size() } } diff --git a/tests-integration/tests/bloom_test/sketch.rs b/tests-integration/tests/bloom_test/sketch.rs index 06b461e4..7d2b7df7 100644 --- a/tests-integration/tests/bloom_test/sketch.rs +++ b/tests-integration/tests/bloom_test/sketch.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of; - use datasketches::bloom::BloomFilter; use datasketches::bloom::BloomFilterBuilder; use datasketches::error::ErrorKind; @@ -216,8 +214,7 @@ fn test_requested_size_rounds_to_word_boundary() { #[test] fn test_estimated_size() { let filter = BloomFilterBuilder::with_size(65, 3).build().unwrap(); - let expected_size = filter.capacity() / 64 * size_of::(); - assert_eq!(filter.estimated_size(), expected_size); + assert_eq!(filter.estimated_size(), 16); } #[test] diff --git a/tests-integration/tests/countmin_test/sketch.rs b/tests-integration/tests/countmin_test/sketch.rs index 68a5f143..b85a8876 100644 --- a/tests-integration/tests/countmin_test/sketch.rs +++ b/tests-integration/tests/countmin_test/sketch.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -446,12 +445,11 @@ fn test_increment_single_key_like_rust_count_min_sketch() { #[test] fn test_estimated_size() { let mut sketch = CountMinSketch::::new(4, 128).unwrap(); - let expected_size = 4 * 128 * size_of::() + 4 * size_of::(); - assert_eq!(sketch.estimated_size(), expected_size); + assert_eq!(sketch.estimated_size(), 4128); // The backing tables are allocated up front; updates do not grow the sketch. sketch.update("apple"); - assert_eq!(sketch.estimated_size(), expected_size); + assert_eq!(sketch.estimated_size(), 4128); } #[test] diff --git a/tests-integration/tests/cpc_test/update.rs b/tests-integration/tests/cpc_test/update.rs index 34ef5ea7..8aec8df7 100644 --- a/tests-integration/tests/cpc_test/update.rs +++ b/tests-integration/tests/cpc_test/update.rs @@ -77,5 +77,5 @@ fn test_estimated_size() { for i in 0..10_000 { sketch.update(i); } - assert!(sketch.estimated_size() > empty_size); + assert_eq!(sketch.estimated_size(), 2304); } diff --git a/tests-integration/tests/frequencies_test/update.rs b/tests-integration/tests/frequencies_test/update.rs index 86e8322e..ca357d64 100644 --- a/tests-integration/tests/frequencies_test/update.rs +++ b/tests-integration/tests/frequencies_test/update.rs @@ -633,22 +633,14 @@ fn test_map_size_above_cross_language_limit_returns_error() { #[test] fn test_estimated_size() { - let expected_size = |sketch: &FrequentItemsSketch| { - let slots = 1usize << sketch.lg_cur_map_size(); - slots - * (std::mem::size_of::>() - + std::mem::size_of::() - + std::mem::size_of::()) - }; - let mut sketch: FrequentItemsSketch = FrequentItemsSketch::new(64).unwrap(); - assert_eq!(sketch.estimated_size(), expected_size(&sketch)); + assert_eq!(sketch.estimated_size(), 208); // The internal map grows from its starting size up to the maximum size. for i in 0..100 { sketch.update(i); } - assert_eq!(sketch.estimated_size(), expected_size(&sketch)); + assert_eq!(sketch.estimated_size(), 1664); } #[test] diff --git a/tests-integration/tests/theta_test/sketch.rs b/tests-integration/tests/theta_test/sketch.rs index f79c74c3..cd43f366 100644 --- a/tests-integration/tests/theta_test/sketch.rs +++ b/tests-integration/tests/theta_test/sketch.rs @@ -157,7 +157,7 @@ fn test_reset() { assert_that!(sketch.num_retained(), gt(32)); assert_that!(sketch.theta(), lt(1.0)); - let size_before_reset = sketch.estimated_size(); + assert_eq!(sketch.estimated_size(), 512); sketch.reset(); assert!(sketch.is_empty()); assert_eq!(sketch.estimate(), 0.0); @@ -166,7 +166,7 @@ fn test_reset() { assert!(!sketch.is_estimation_mode()); assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); - assert_eq!(sketch.estimated_size(), size_before_reset); + assert_eq!(sketch.estimated_size(), 512); } #[test] diff --git a/tests-integration/tests/theta_test/union.rs b/tests-integration/tests/theta_test/union.rs index 0c02df99..3935b827 100644 --- a/tests-integration/tests/theta_test/union.rs +++ b/tests-integration/tests/theta_test/union.rs @@ -749,6 +749,5 @@ fn compact_estimated_size_includes_retained_capacity() { let result = union.to_sketch(true); assert_eq!(result.num_retained(), 32); - let logical_size = result.num_retained() * std::mem::size_of::(); - assert!(result.estimated_size() > logical_size); + assert_eq!(result.estimated_size(), 512); } diff --git a/tests-integration/tests/tuple_test/sketch.rs b/tests-integration/tests/tuple_test/sketch.rs index 5ca06c8d..694644e1 100644 --- a/tests-integration/tests/tuple_test/sketch.rs +++ b/tests-integration/tests/tuple_test/sketch.rs @@ -153,6 +153,45 @@ fn estimated_size_is_shallow_for_generic_summaries() { ); } +struct PaddedArraySumPolicy { + inner: ArraySumPolicy, + _padding: [u8; 1024], +} + +impl SummaryPolicy for PaddedArraySumPolicy { + type Summary = Vec; + + fn create(&self) -> Self::Summary { + self.inner.create() + } +} + +impl SummaryUpdatePolicy for PaddedArraySumPolicy +where + U: AsRef<[f64]>, +{ + fn update(&self, summary: &mut Self::Summary, value: U) { + self.inner.update(summary, value); + } +} + +#[test] +fn estimated_size_includes_policy_inline_storage() { + let mut small_policy_sketch = TupleSketchBuilder::new(ArraySumPolicy { num_values: 1 }) + .build() + .unwrap(); + small_policy_sketch.update("key", [1.0]); + + let policy = PaddedArraySumPolicy { + inner: ArraySumPolicy { num_values: 1 }, + _padding: [0; 1024], + }; + let mut large_policy_sketch = TupleSketchBuilder::new(policy).build().unwrap(); + large_policy_sketch.update("key", [1.0]); + + assert!(large_policy_sketch.estimated_size() > small_policy_sketch.estimated_size()); +} + #[test] fn trim_and_reset_update_public_state() { let mut sketch = default_tuple_sketch_builder().lg_k(5).build().unwrap(); diff --git a/tests-integration/tests/tuple_test/union.rs b/tests-integration/tests/tuple_test/union.rs index 14af4f91..6c23c100 100644 --- a/tests-integration/tests/tuple_test/union.rs +++ b/tests-integration/tests/tuple_test/union.rs @@ -201,7 +201,5 @@ fn compact_estimated_size_includes_retained_capacity() { let result = union.to_sketch(true); assert_eq!(result.num_retained(), 32); - let entry_size = size_of_val(result.iter().next().unwrap()); - let logical_size = result.num_retained() * entry_size; - assert!(result.estimated_size() > logical_size); + assert_eq!(result.estimated_size(), 1024); }