diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aeb2a9a..733a7730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All significant changes to this project will be documented in this file. * `ReqSketch` queries now use `datasketches::common::SearchCriteria` instead of `datasketches::req::SearchCriteria`. `SearchCriteria` no longer implements `Default`; explicitly choose `Inclusive` or `Exclusive`. * `ReqSketch` and `req::SortedView` queries now distinguish empty sketches from invalid arguments. `rank` returns `Option`; `quantile`, `quantiles`, `cdf`, and `pmf`, where available, return `Result, Error>`. Handle `None` for valid queries on an empty sketch, and propagate or handle `InvalidArgument` for invalid ranks or split points, which are checked before emptiness. * `TDigest` and `TDigestMut` rank, quantile, CDF, and PMF queries now return `Result, Error>`. Invalid arguments return `InvalidArgument` instead of panicking, including on empty digests. Propagate or handle the `Result` before handling `None` for an empty digest; `min_value` and `max_value` still return `Option`. +* `estimated_size` now returns an implementation-dependent estimate of memory usage. Returned values may change as implementations evolve and should not be treated as exact totals or a stable accounting contract. ### New features diff --git a/datasketches/src/bloom/sketch.rs b/datasketches/src/bloom/sketch.rs index 0d399d58..245a7262 100644 --- a/datasketches/src/bloom/sketch.rs +++ b/datasketches/src/bloom/sketch.rs @@ -639,9 +639,9 @@ impl BloomFilter { was_set } - /// Returns the estimated size of the filter in bytes. + /// Returns an estimate of the filter's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.bit_array.len() * size_of::() + self.bit_array.len() * size_of::() } } diff --git a/datasketches/src/countmin/sketch.rs b/datasketches/src/countmin/sketch.rs index e69150c7..fe346ccb 100644 --- a/datasketches/src/countmin/sketch.rs +++ b/datasketches/src/countmin/sketch.rs @@ -495,11 +495,9 @@ impl CountMinSketch { Ok(sketch) } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() - + self.counts.capacity() * size_of::() - + self.hash_seeds.capacity() * size_of::() + self.counts.capacity() * size_of::() + self.hash_seeds.capacity() * size_of::() } fn make(num_hashes: u8, num_buckets: u32, seed: u64, seed_hash: u16, entries: usize) -> Self { diff --git a/datasketches/src/cpc/pair_table.rs b/datasketches/src/cpc/pair_table.rs index a476ec1f..d8ab5d81 100644 --- a/datasketches/src/cpc/pair_table.rs +++ b/datasketches/src/cpc/pair_table.rs @@ -258,7 +258,7 @@ impl PairTable { } } - /// Returns the estimated size of the heap allocations in bytes + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.slots.capacity() * size_of::() } diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs index c42b3672..3222cff3 100644 --- a/datasketches/src/cpc/sketch.rs +++ b/datasketches/src/cpc/sketch.rs @@ -459,16 +459,14 @@ impl CpcSketch { matrix } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - let heap_size = self.sliding_window.capacity() + self.sliding_window.capacity() + self .surprising_value_table .as_ref() .map(|t| t.estimated_size()) - .unwrap_or(0); - - size_of::() + heap_size + .unwrap_or(0) } } diff --git a/datasketches/src/cpc/union.rs b/datasketches/src/cpc/union.rs index 67870f59..abd473dd 100644 --- a/datasketches/src/cpc/union.rs +++ b/datasketches/src/cpc/union.rs @@ -347,14 +347,12 @@ impl CpcUnion { } } - /// Returns the estimated size of the union in bytes. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { - // The state's inline size is already covered by size_of::(). - let heap_size = match &self.state { - UnionState::Accumulator(sketch) => sketch.estimated_size() - size_of::(), + match &self.state { + UnionState::Accumulator(sketch) => sketch.estimated_size(), UnionState::BitMatrix(matrix) => matrix.capacity() * size_of::(), - }; - size_of::() + heap_size + } } } diff --git a/datasketches/src/frequencies/reverse_purge_item_hash_map.rs b/datasketches/src/frequencies/reverse_purge_item_hash_map.rs index ce3d62dc..7f430f65 100644 --- a/datasketches/src/frequencies/reverse_purge_item_hash_map.rs +++ b/datasketches/src/frequencies/reverse_purge_item_hash_map.rs @@ -197,7 +197,7 @@ impl ReversePurgeItemHashMap { self.num_active } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.keys.capacity() * size_of::>() + self.values.capacity() * size_of::() diff --git a/datasketches/src/frequencies/sketch.rs b/datasketches/src/frequencies/sketch.rs index 4808ef58..617b98ec 100644 --- a/datasketches/src/frequencies/sketch.rs +++ b/datasketches/src/frequencies/sketch.rs @@ -323,9 +323,9 @@ impl FrequentItemsSketch { self.hash_map.lg_length() } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.hash_map.estimated_size() + self.hash_map.estimated_size() } /// Updates the sketch with a count of one. diff --git a/datasketches/src/hll/array4.rs b/datasketches/src/hll/array4.rs index 19c96320..4190a3f7 100644 --- a/datasketches/src/hll/array4.rs +++ b/datasketches/src/hll/array4.rs @@ -419,7 +419,7 @@ impl Array4 { bytes.into_bytes() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() + self diff --git a/datasketches/src/hll/array6.rs b/datasketches/src/hll/array6.rs index c29a9015..636da15d 100644 --- a/datasketches/src/hll/array6.rs +++ b/datasketches/src/hll/array6.rs @@ -233,7 +233,7 @@ impl Array6 { bytes.into_bytes() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() } diff --git a/datasketches/src/hll/array8.rs b/datasketches/src/hll/array8.rs index 355bcc18..1269d171 100644 --- a/datasketches/src/hll/array8.rs +++ b/datasketches/src/hll/array8.rs @@ -278,7 +278,7 @@ impl Array8 { bytes.into_bytes() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.bytes.len() } diff --git a/datasketches/src/hll/aux_map.rs b/datasketches/src/hll/aux_map.rs index e14ef31f..2ce25a39 100644 --- a/datasketches/src/hll/aux_map.rs +++ b/datasketches/src/hll/aux_map.rs @@ -192,7 +192,7 @@ impl AuxMap { }) } - /// Returns the estimated size of the heap allocations in bytes + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.entries.len() * size_of::() } diff --git a/datasketches/src/hll/container.rs b/datasketches/src/hll/container.rs index 8f0313c7..bc8cab68 100644 --- a/datasketches/src/hll/container.rs +++ b/datasketches/src/hll/container.rs @@ -121,7 +121,7 @@ impl Container { self.coupons.iter().filter(|&&c| !c.is_empty()).copied() } - /// Returns the estimated size of the heap allocations in bytes + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.coupons.len() * size_of::() } diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 92d4d5b3..910a730c 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -470,17 +470,15 @@ impl HllSketch { } } - /// Returns the estimated memory usage in bytes, including owned heap allocations. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - let heap_size = match &self.mode { + match &self.mode { Mode::List { list, .. } => list.container().estimated_size(), Mode::Set { set, .. } => set.container().estimated_size(), Mode::Array4(arr) => arr.estimated_size(), Mode::Array6(arr) => arr.estimated_size(), Mode::Array8(arr) => arr.estimated_size(), - }; - - size_of::() + heap_size + } } } diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index 0f6e20ec..b245fa71 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -365,10 +365,9 @@ impl HllUnion { self.gadget.lower_bound(num_std_dev) } - /// Returns the estimated memory usage in bytes, including owned heap allocations. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { - // The gadget's inline size is already covered by size_of::(). - size_of::() - size_of::() + self.gadget.estimated_size() + self.gadget.estimated_size() } } diff --git a/datasketches/src/tdigest/sketch.rs b/datasketches/src/tdigest/sketch.rs index 1fb4b16f..9bc7b032 100644 --- a/datasketches/src/tdigest/sketch.rs +++ b/datasketches/src/tdigest/sketch.rs @@ -156,6 +156,7 @@ impl TDigestBuffer { self.centroids } + /// Returns an estimate of memory usage in bytes. fn estimated_size(&self) -> usize { self.centroids.capacity() * size_of::() } @@ -1012,9 +1013,9 @@ impl TDigestMut { centroids.shrink_to(target_capacity); } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.buffer.estimated_size() + self.buffer.estimated_size() } } @@ -1630,9 +1631,9 @@ impl TDigest { ) } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.centroids.capacity() * size_of::() + self.centroids.capacity() * size_of::() } } diff --git a/datasketches/src/thetafamily/common/hash_table.rs b/datasketches/src/thetafamily/common/hash_table.rs index d2138c93..e0b7523e 100644 --- a/datasketches/src/thetafamily/common/hash_table.rs +++ b/datasketches/src/thetafamily/common/hash_table.rs @@ -344,7 +344,7 @@ where }) } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { self.entries.capacity() * size_of::>() } diff --git a/datasketches/src/thetafamily/common/intersection.rs b/datasketches/src/thetafamily/common/intersection.rs index 486b2cb7..535bebac 100644 --- a/datasketches/src/thetafamily/common/intersection.rs +++ b/datasketches/src/thetafamily/common/intersection.rs @@ -234,9 +234,9 @@ where self.result_state != IntersectionResultState::Uninitialized } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { - self.table.estimated_size() + size_of::

() + self.table.estimated_size() } /// Returns the current intersection as canonical compact-sketch state. diff --git a/datasketches/src/thetafamily/common/union.rs b/datasketches/src/thetafamily/common/union.rs index 0131b55a..4ad3b466 100644 --- a/datasketches/src/thetafamily/common/union.rs +++ b/datasketches/src/thetafamily/common/union.rs @@ -150,8 +150,8 @@ where self.result_theta = None; } - /// Returns the estimated size of the heap allocations in bytes. + /// Returns an estimate of memory usage in bytes. pub fn estimated_size(&self) -> usize { - self.table.estimated_size() + size_of::

() + self.table.estimated_size() } } diff --git a/datasketches/src/thetafamily/theta/intersection.rs b/datasketches/src/thetafamily/theta/intersection.rs index ace0d41b..137891d8 100644 --- a/datasketches/src/thetafamily/theta/intersection.rs +++ b/datasketches/src/thetafamily/theta/intersection.rs @@ -78,9 +78,9 @@ impl ThetaIntersection { self.state.has_result() } - /// Returns the estimated size of the intersection in bytes. + /// Returns an estimate of the intersection's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } /// Returns the current intersection as a compact theta sketch. diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index b75d1aff..4ec71b79 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -427,9 +427,9 @@ impl ThetaSketch { .expect("theta should always be valid") } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.table.estimated_size() + self.table.estimated_size() } } @@ -1027,9 +1027,9 @@ impl CompactThetaSketch { Ok(Self::from_compact_state(compact_state)) } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.compact_state.retained_entries_capacity() * size_of::() + self.compact_state.retained_entries_capacity() * size_of::() } } diff --git a/datasketches/src/thetafamily/theta/union.rs b/datasketches/src/thetafamily/theta/union.rs index c315c4f2..1b26d7a2 100644 --- a/datasketches/src/thetafamily/theta/union.rs +++ b/datasketches/src/thetafamily/theta/union.rs @@ -64,9 +64,9 @@ impl ThetaUnion { self.state.reset(); } - /// Returns the estimated size of the union in bytes. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } } diff --git a/datasketches/src/thetafamily/tuple/intersection.rs b/datasketches/src/thetafamily/tuple/intersection.rs index 38dfe9ed..58dce8fd 100644 --- a/datasketches/src/thetafamily/tuple/intersection.rs +++ b/datasketches/src/thetafamily/tuple/intersection.rs @@ -138,9 +138,9 @@ where self.state.has_result() } - /// Returns the estimated size of the intersection in bytes. + /// Returns an estimate of the intersection's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } /// Returns the current intersection as a compact Tuple sketch. diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index e3a690fe..0fee8218 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -404,9 +404,9 @@ where .expect("theta should always be valid") } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.table.estimated_size() + size_of::

() + self.table.estimated_size() } } @@ -541,10 +541,9 @@ impl CompactTupleSketch { .expect("compact theta should always be valid") } - /// Returns the estimated size of the sketch in bytes. + /// Returns an estimate of the sketch's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() - + self.compact_state.retained_entries_capacity() * size_of::>() + self.compact_state.retained_entries_capacity() * size_of::>() } fn preamble_longs(&self) -> u8 { diff --git a/datasketches/src/thetafamily/tuple/union.rs b/datasketches/src/thetafamily/tuple/union.rs index df7786b7..029aeba7 100644 --- a/datasketches/src/thetafamily/tuple/union.rs +++ b/datasketches/src/thetafamily/tuple/union.rs @@ -112,9 +112,9 @@ where self.state.reset(); } - /// Returns the estimated size of the union in bytes. + /// Returns an estimate of the union's memory usage in bytes. pub fn estimated_size(&self) -> usize { - size_of::() + self.state.estimated_size() + self.state.estimated_size() } } diff --git a/tests-integration/tests/bloom_test/sketch.rs b/tests-integration/tests/bloom_test/sketch.rs index db0a0db1..7d2b7df7 100644 --- a/tests-integration/tests/bloom_test/sketch.rs +++ b/tests-integration/tests/bloom_test/sketch.rs @@ -211,6 +211,12 @@ fn test_requested_size_rounds_to_word_boundary() { assert_eq!(filter.capacity(), 128); } +#[test] +fn test_estimated_size() { + let filter = BloomFilterBuilder::with_size(65, 3).build().unwrap(); + assert_eq!(filter.estimated_size(), 16); +} + #[test] fn test_accuracy_builder_rejects_zero_items_at_build() { let error = BloomFilterBuilder::with_accuracy(0, 0.01) diff --git a/tests-integration/tests/countmin_test/sketch.rs b/tests-integration/tests/countmin_test/sketch.rs index a5d39189..b85a8876 100644 --- a/tests-integration/tests/countmin_test/sketch.rs +++ b/tests-integration/tests/countmin_test/sketch.rs @@ -445,11 +445,11 @@ fn test_increment_single_key_like_rust_count_min_sketch() { #[test] fn test_estimated_size() { let mut sketch = CountMinSketch::::new(4, 128).unwrap(); - assert_eq!(sketch.estimated_size(), 4224); + assert_eq!(sketch.estimated_size(), 4128); // The backing tables are allocated up front; updates do not grow the sketch. sketch.update("apple"); - assert_eq!(sketch.estimated_size(), 4224); + assert_eq!(sketch.estimated_size(), 4128); } #[test] diff --git a/tests-integration/tests/cpc_test/union.rs b/tests-integration/tests/cpc_test/union.rs index 7b280343..d72a7d4d 100644 --- a/tests-integration/tests/cpc_test/union.rs +++ b/tests-integration/tests/cpc_test/union.rs @@ -183,12 +183,12 @@ fn test_lg_k_too_large_returns_error() { #[test] fn test_union_estimated_size() { let mut union = CpcUnion::new(11).unwrap(); - assert_eq!(union.estimated_size(), 112); + assert_eq!(union.estimated_size(), 0); let mut sketch = CpcSketch::new(11).unwrap(); for i in 0..1000 { sketch.update(i); } union.update(&sketch).unwrap(); - assert_eq!(union.estimated_size(), 16496); + assert_eq!(union.estimated_size(), 16384); } diff --git a/tests-integration/tests/cpc_test/update.rs b/tests-integration/tests/cpc_test/update.rs index 99a1cb53..8aec8df7 100644 --- a/tests-integration/tests/cpc_test/update.rs +++ b/tests-integration/tests/cpc_test/update.rs @@ -67,3 +67,15 @@ fn test_max_serialized_bytes_validates_lg_k() { assert_eq!(error.kind(), ErrorKind::InvalidArgument); } } + +#[test] +fn test_estimated_size() { + let mut sketch = CpcSketch::new(11).unwrap(); + let empty_size = sketch.estimated_size(); + assert_eq!(empty_size, 0); + + for i in 0..10_000 { + sketch.update(i); + } + assert_eq!(sketch.estimated_size(), 2304); +} diff --git a/tests-integration/tests/frequencies_test/update.rs b/tests-integration/tests/frequencies_test/update.rs index 06489d36..ca357d64 100644 --- a/tests-integration/tests/frequencies_test/update.rs +++ b/tests-integration/tests/frequencies_test/update.rs @@ -634,11 +634,27 @@ fn test_map_size_above_cross_language_limit_returns_error() { #[test] fn test_estimated_size() { let mut sketch: FrequentItemsSketch = FrequentItemsSketch::new(64).unwrap(); - assert_eq!(sketch.estimated_size(), 344); + assert_eq!(sketch.estimated_size(), 208); // The internal map grows from its starting size up to the maximum size. for i in 0..100 { sketch.update(i); } - assert_eq!(sketch.estimated_size(), 1800); + assert_eq!(sketch.estimated_size(), 1664); +} + +#[test] +fn estimated_size_is_shallow_for_generic_items() { + let mut small_item_sketch = FrequentItemsSketch::new(8).unwrap(); + small_item_sketch.update("key".to_string()); + + let mut large_item = String::with_capacity(4096); + large_item.push_str("key"); + let mut large_item_sketch = FrequentItemsSketch::new(8).unwrap(); + large_item_sketch.update(large_item); + + assert_eq!( + small_item_sketch.estimated_size(), + large_item_sketch.estimated_size() + ); } diff --git a/tests-integration/tests/hll_test/union.rs b/tests-integration/tests/hll_test/union.rs index 7a6409a0..c61a1716 100644 --- a/tests-integration/tests/hll_test/union.rs +++ b/tests-integration/tests/hll_test/union.rs @@ -710,14 +710,14 @@ fn test_union_validation() { #[test] fn test_union_estimated_size() { let mut union = HllUnion::new(10).unwrap(); - assert_eq!(union.estimated_size(), 128); + assert_eq!(union.estimated_size(), 32); let mut sketch = HllSketch::new(10, HllType::Hll8).unwrap(); for i in 0..1000 { sketch.update(i); } union.update(&sketch); - assert_eq!(union.estimated_size(), 1120); + assert_eq!(union.estimated_size(), 1024); } #[test] diff --git a/tests-integration/tests/hll_test/update.rs b/tests-integration/tests/hll_test/update.rs index 50fd0b45..5ec6ac89 100644 --- a/tests-integration/tests/hll_test/update.rs +++ b/tests-integration/tests/hll_test/update.rs @@ -201,3 +201,18 @@ fn test_invalid_lg_k_high_returns_error() { let error = HllSketch::new(22, HllType::Hll8).unwrap_err(); assert_eq!(error.kind(), ErrorKind::InvalidArgument); } + +#[test] +fn estimated_size_tracks_retained_storage() { + let mut sketch = HllSketch::new(10, HllType::Hll8).unwrap(); + let initial_size = sketch.estimated_size(); + assert_eq!(initial_size, 32); + + sketch.update(0); + assert_eq!(sketch.estimated_size(), initial_size); + + for i in 1..1000 { + sketch.update(i); + } + assert_eq!(sketch.estimated_size(), 1024); +} diff --git a/tests-integration/tests/tdigest_test/sketch.rs b/tests-integration/tests/tdigest_test/sketch.rs index 86bc45fb..4eba82d6 100644 --- a/tests-integration/tests/tdigest_test/sketch.rs +++ b/tests-integration/tests/tdigest_test/sketch.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::mem::size_of; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -119,15 +118,14 @@ fn test_estimated_size_reuses_buffer_after_compression() { const TARGET_CENTROIDS: usize = 410; const MAX_UNMERGED: usize = TARGET_CENTROIDS * 4; - let inline_size = size_of::(); let mut tdigest = TDigestMut::new(K).unwrap(); - assert_eq!(tdigest.estimated_size(), inline_size); + assert_eq!(tdigest.estimated_size(), 0); for value in 0..MAX_UNMERGED { tdigest.update(value as f64); } let size_before_compression = tdigest.estimated_size(); - assert!(size_before_compression > inline_size); + assert!(size_before_compression > 0); tdigest.rank(0.5).unwrap(); assert!(tdigest.estimated_size() <= size_before_compression); diff --git a/tests-integration/tests/theta_test/intersection.rs b/tests-integration/tests/theta_test/intersection.rs index 87f545fe..f6068a13 100644 --- a/tests-integration/tests/theta_test/intersection.rs +++ b/tests-integration/tests/theta_test/intersection.rs @@ -326,9 +326,9 @@ fn test_seed_mismatch_non_empty_returns_error() { #[test] fn test_intersection_estimated_size() { let mut intersection = ThetaIntersection::default(); - assert_eq!(intersection.estimated_size(), 72); + assert_eq!(intersection.estimated_size(), 0); let sketch = sketch_with_range(0, 1000); intersection.update(&sketch).unwrap(); - assert_eq!(intersection.estimated_size(), 16456); + assert_eq!(intersection.estimated_size(), 16384); } diff --git a/tests-integration/tests/theta_test/sketch.rs b/tests-integration/tests/theta_test/sketch.rs index b900ab1b..cd43f366 100644 --- a/tests-integration/tests/theta_test/sketch.rs +++ b/tests-integration/tests/theta_test/sketch.rs @@ -157,6 +157,7 @@ fn test_reset() { assert_that!(sketch.num_retained(), gt(32)); assert_that!(sketch.theta(), lt(1.0)); + assert_eq!(sketch.estimated_size(), 512); sketch.reset(); assert!(sketch.is_empty()); assert_eq!(sketch.estimate(), 0.0); @@ -165,6 +166,7 @@ fn test_reset() { assert!(!sketch.is_estimation_mode()); assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); + assert_eq!(sketch.estimated_size(), 512); } #[test] diff --git a/tests-integration/tests/theta_test/union.rs b/tests-integration/tests/theta_test/union.rs index 14df5bfe..3935b827 100644 --- a/tests-integration/tests/theta_test/union.rs +++ b/tests-integration/tests/theta_test/union.rs @@ -734,9 +734,20 @@ fn test_corner_case_union_states() { #[test] fn test_union_estimated_size() { let mut union = ThetaUnionBuilder::default().build().unwrap(); - assert_eq!(union.estimated_size(), 1104); + assert_eq!(union.estimated_size(), 1024); let sketch = sketch_with_range(12, 0, 1000); union.update(&sketch).unwrap(); - assert_eq!(union.estimated_size(), 65616); + assert_eq!(union.estimated_size(), 65536); +} + +#[test] +fn compact_estimated_size_includes_retained_capacity() { + let sketch = sketch_with_range(5, 0, 1000); + let mut union = ThetaUnionBuilder::default().lg_k(5).build().unwrap(); + union.update(&sketch).unwrap(); + let result = union.to_sketch(true); + + assert_eq!(result.num_retained(), 32); + assert_eq!(result.estimated_size(), 512); } diff --git a/tests-integration/tests/tuple_test/intersection.rs b/tests-integration/tests/tuple_test/intersection.rs index 9caa0eb8..d01381ab 100644 --- a/tests-integration/tests/tuple_test/intersection.rs +++ b/tests-integration/tests/tuple_test/intersection.rs @@ -185,9 +185,9 @@ fn estimation_bounds_cover_the_true_intersection() { #[test] fn intersection_estimated_size_grows_with_updates() { let mut intersection = TupleIntersection::new(SumPolicy); - assert_eq!(intersection.estimated_size(), 72); + assert_eq!(intersection.estimated_size(), 0); let sketch = tuple_sketch_with_range(0, 1000); intersection.update(&sketch).unwrap(); - assert_eq!(intersection.estimated_size(), 32840); + assert_eq!(intersection.estimated_size(), 32768); } diff --git a/tests-integration/tests/tuple_test/sketch.rs b/tests-integration/tests/tuple_test/sketch.rs index 194060d2..694644e1 100644 --- a/tests-integration/tests/tuple_test/sketch.rs +++ b/tests-integration/tests/tuple_test/sketch.rs @@ -131,6 +131,67 @@ fn custom_update_policy_accepts_multiple_value_representations() { ); } +#[test] +fn estimated_size_is_shallow_for_generic_summaries() { + let mut small_summary_sketch = TupleSketchBuilder::new(ArraySumPolicy { num_values: 1 }) + .build() + .unwrap(); + small_summary_sketch.update("key", [1.0]); + + let mut large_summary_sketch = TupleSketchBuilder::new(ArraySumPolicy { num_values: 1024 }) + .build() + .unwrap(); + large_summary_sketch.update("key", vec![1.0; 1024]); + + assert_eq!( + small_summary_sketch.estimated_size(), + large_summary_sketch.estimated_size() + ); + assert_eq!( + small_summary_sketch.compact(false).estimated_size(), + large_summary_sketch.compact(false).estimated_size() + ); +} + +struct PaddedArraySumPolicy { + inner: ArraySumPolicy, + _padding: [u8; 1024], +} + +impl SummaryPolicy for PaddedArraySumPolicy { + type Summary = Vec; + + fn create(&self) -> Self::Summary { + self.inner.create() + } +} + +impl SummaryUpdatePolicy for PaddedArraySumPolicy +where + U: AsRef<[f64]>, +{ + fn update(&self, summary: &mut Self::Summary, value: U) { + self.inner.update(summary, value); + } +} + +#[test] +fn estimated_size_includes_policy_inline_storage() { + let mut small_policy_sketch = TupleSketchBuilder::new(ArraySumPolicy { num_values: 1 }) + .build() + .unwrap(); + small_policy_sketch.update("key", [1.0]); + + let policy = PaddedArraySumPolicy { + inner: ArraySumPolicy { num_values: 1 }, + _padding: [0; 1024], + }; + let mut large_policy_sketch = TupleSketchBuilder::new(policy).build().unwrap(); + large_policy_sketch.update("key", [1.0]); + + assert!(large_policy_sketch.estimated_size() > small_policy_sketch.estimated_size()); +} + #[test] fn trim_and_reset_update_public_state() { let mut sketch = default_tuple_sketch_builder().lg_k(5).build().unwrap(); diff --git a/tests-integration/tests/tuple_test/union.rs b/tests-integration/tests/tuple_test/union.rs index 90b68b01..6c23c100 100644 --- a/tests-integration/tests/tuple_test/union.rs +++ b/tests-integration/tests/tuple_test/union.rs @@ -186,9 +186,20 @@ fn estimation_bounds_cover_the_true_union() { #[test] fn union_estimated_size_grows_with_updates() { let mut union = default_union_builder().build().unwrap(); - assert_eq!(union.estimated_size(), 2128); + assert_eq!(union.estimated_size(), 2048); let sketch = tuple_sketch_with_range(0, 1000); union.update(&sketch).unwrap(); - assert_eq!(union.estimated_size(), 131152); + assert_eq!(union.estimated_size(), 131072); +} + +#[test] +fn compact_estimated_size_includes_retained_capacity() { + let sketch = tuple_sketch_with_range(0, 1000); + let mut union = default_union_builder().lg_k(5).build().unwrap(); + union.update(&sketch).unwrap(); + let result = union.to_sketch(true); + + assert_eq!(result.num_retained(), 32); + assert_eq!(result.estimated_size(), 1024); }