refactor: simplify estimated_size accounting

CHANGELOG.md

@@ -11,6 +11,7 @@ All significant changes to this project will be documented in this file.
 * `ReqSketch` queries now use `datasketches::common::SearchCriteria` instead of `datasketches::req::SearchCriteria`. `SearchCriteria` no longer implements `Default`; explicitly choose `Inclusive` or `Exclusive`.
 * `ReqSketch` and `req::SortedView` queries now distinguish empty sketches from invalid arguments. `rank` returns `Option<f64>`; `quantile`, `quantiles`, `cdf`, and `pmf`, where available, return `Result<Option<_>, Error>`. Handle `None` for valid queries on an empty sketch, and propagate or handle `InvalidArgument` for invalid ranks or split points, which are checked before emptiness.
 * `TDigest` and `TDigestMut` rank, quantile, CDF, and PMF queries now return `Result<Option<_>, Error>`. Invalid arguments return `InvalidArgument` instead of panicking, including on empty digests. Propagate or handle the `Result` before handling `None` for an empty digest; `min_value` and `max_value` still return `Option<f64>`.
+* `estimated_size` now returns an implementation-dependent estimate of memory usage. Returned values may change as implementations evolve and should not be treated as exact totals or a stable accounting contract.
 
 ### New features
 

datasketches/src/bloom/sketch.rs

@@ -639,9 +639,9 @@ impl BloomFilter {
         was_set
     }
 
-    /// Returns the estimated size of the filter in bytes.
+    /// Returns an estimate of the filter's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.bit_array.len() * size_of::<u64>()
+        self.bit_array.len() * size_of::<u64>()
     }
 }
 

datasketches/src/countmin/sketch.rs

@@ -495,11 +495,9 @@ impl<T: CountMinValue> CountMinSketch<T> {
         Ok(sketch)
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>()
-            + self.counts.capacity() * size_of::<T>()
-            + self.hash_seeds.capacity() * size_of::<u64>()
+        self.counts.capacity() * size_of::<T>() + self.hash_seeds.capacity() * size_of::<u64>()
     }
 
     fn make(num_hashes: u8, num_buckets: u32, seed: u64, seed_hash: u16, entries: usize) -> Self {

datasketches/src/cpc/pair_table.rs

@@ -258,7 +258,7 @@ impl PairTable {
         }
     }
 
-    /// Returns the estimated size of the heap allocations in bytes
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.slots.capacity() * size_of::<u32>()
     }

datasketches/src/cpc/sketch.rs

@@ -459,16 +459,14 @@ impl CpcSketch {
         matrix
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        let heap_size = self.sliding_window.capacity()
+        self.sliding_window.capacity()
             + self
                 .surprising_value_table
                 .as_ref()
                 .map(|t| t.estimated_size())
-                .unwrap_or(0);
-
-        size_of::<Self>() + heap_size
+                .unwrap_or(0)
     }
 }
 

datasketches/src/cpc/union.rs

@@ -347,14 +347,12 @@ impl CpcUnion {
         }
     }
 
-    /// Returns the estimated size of the union in bytes.
+    /// Returns an estimate of the union's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        // The state's inline size is already covered by size_of::<Self>().
-        let heap_size = match &self.state {
-            UnionState::Accumulator(sketch) => sketch.estimated_size() - size_of::<CpcSketch>(),
+        match &self.state {
+            UnionState::Accumulator(sketch) => sketch.estimated_size(),
             UnionState::BitMatrix(matrix) => matrix.capacity() * size_of::<u64>(),
-        };
-        size_of::<Self>() + heap_size
+        }
     }
 }
 

datasketches/src/frequencies/reverse_purge_item_hash_map.rs

@@ -197,7 +197,7 @@ impl<T: Eq + Hash> ReversePurgeItemHashMap<T> {
         self.num_active
     }
 
-    /// Returns the estimated size of the heap allocations in bytes.
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.keys.capacity() * size_of::<Option<T>>()
             + self.values.capacity() * size_of::<u64>()

datasketches/src/frequencies/sketch.rs

@@ -323,9 +323,9 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
         self.hash_map.lg_length()
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.hash_map.estimated_size()
+        self.hash_map.estimated_size()
     }
 
     /// Updates the sketch with a count of one.

datasketches/src/hll/array4.rs

@@ -419,7 +419,7 @@ impl Array4 {
         bytes.into_bytes()
     }
 
-    /// Returns the estimated size of the heap allocations in bytes
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.bytes.len()
             + self

datasketches/src/hll/array6.rs

@@ -233,7 +233,7 @@ impl Array6 {
         bytes.into_bytes()
     }
 
-    /// Returns the estimated size of the heap allocations in bytes
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.bytes.len()
     }

datasketches/src/hll/array8.rs

@@ -278,7 +278,7 @@ impl Array8 {
         bytes.into_bytes()
     }
 
-    /// Returns the estimated size of the heap allocations in bytes
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.bytes.len()
     }

datasketches/src/hll/aux_map.rs

@@ -192,7 +192,7 @@ impl AuxMap {
         })
     }
 
-    /// Returns the estimated size of the heap allocations in bytes
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.entries.len() * size_of::<Coupon>()
     }

datasketches/src/hll/container.rs

@@ -121,7 +121,7 @@ impl Container {
         self.coupons.iter().filter(|&&c| !c.is_empty()).copied()
     }
 
-    /// Returns the estimated size of the heap allocations in bytes
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.coupons.len() * size_of::<Coupon>()
     }

datasketches/src/hll/sketch.rs

@@ -470,17 +470,15 @@ impl HllSketch {
         }
     }
 
-    /// Returns the estimated memory usage in bytes, including owned heap allocations.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        let heap_size = match &self.mode {
+        match &self.mode {
             Mode::List { list, .. } => list.container().estimated_size(),
             Mode::Set { set, .. } => set.container().estimated_size(),
             Mode::Array4(arr) => arr.estimated_size(),
             Mode::Array6(arr) => arr.estimated_size(),
             Mode::Array8(arr) => arr.estimated_size(),
-        };
-
-        size_of::<Self>() + heap_size
+        }
     }
 }
 

datasketches/src/hll/union.rs

@@ -365,10 +365,9 @@ impl HllUnion {
         self.gadget.lower_bound(num_std_dev)
     }
 
-    /// Returns the estimated memory usage in bytes, including owned heap allocations.
+    /// Returns an estimate of the union's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        // The gadget's inline size is already covered by size_of::<Self>().
-        size_of::<Self>() - size_of::<HllSketch>() + self.gadget.estimated_size()
+        self.gadget.estimated_size()
     }
 }
 

datasketches/src/tdigest/sketch.rs

@@ -156,6 +156,7 @@ impl TDigestBuffer {
         self.centroids
     }
 
+    /// Returns an estimate of memory usage in bytes.
     fn estimated_size(&self) -> usize {
         self.centroids.capacity() * size_of::<Centroid>()
     }
@@ -1012,9 +1013,9 @@ impl TDigestMut {
         centroids.shrink_to(target_capacity);
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.buffer.estimated_size()
+        self.buffer.estimated_size()
     }
 }
 
@@ -1630,9 +1631,9 @@ impl TDigest {
         )
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.centroids.capacity() * size_of::<Centroid>()
+        self.centroids.capacity() * size_of::<Centroid>()
     }
 }
 

datasketches/src/thetafamily/common/hash_table.rs

@@ -344,7 +344,7 @@ where
             })
     }
 
-    /// Returns the estimated size of the heap allocations in bytes.
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
         self.entries.capacity() * size_of::<Option<E>>()
     }

datasketches/src/thetafamily/common/intersection.rs

@@ -234,9 +234,9 @@ where
         self.result_state != IntersectionResultState::Uninitialized
     }
 
-    /// Returns the estimated size of the heap allocations in bytes.
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        self.table.estimated_size()
+        size_of::<P>() + self.table.estimated_size()
     }
 
     /// Returns the current intersection as canonical compact-sketch state.

datasketches/src/thetafamily/common/union.rs

@@ -150,8 +150,8 @@ where
         self.result_theta = None;
     }
 
-    /// Returns the estimated size of the heap allocations in bytes.
+    /// Returns an estimate of memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        self.table.estimated_size()
+        size_of::<P>() + self.table.estimated_size()
     }
 }

datasketches/src/thetafamily/theta/intersection.rs

@@ -78,9 +78,9 @@ impl ThetaIntersection {
         self.state.has_result()
     }
 
-    /// Returns the estimated size of the intersection in bytes.
+    /// Returns an estimate of the intersection's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.state.estimated_size()
+        self.state.estimated_size()
     }
 
     /// Returns the current intersection as a compact theta sketch.

datasketches/src/thetafamily/theta/sketch.rs

@@ -427,9 +427,9 @@ impl ThetaSketch {
         .expect("theta should always be valid")
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.table.estimated_size()
+        self.table.estimated_size()
     }
 }
 
@@ -1027,9 +1027,9 @@ impl CompactThetaSketch {
         Ok(Self::from_compact_state(compact_state))
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.compact_state.retained_entries_capacity() * size_of::<u64>()
+        self.compact_state.retained_entries_capacity() * size_of::<u64>()
     }
 }
 

datasketches/src/thetafamily/theta/union.rs

@@ -64,9 +64,9 @@ impl ThetaUnion {
         self.state.reset();
     }
 
-    /// Returns the estimated size of the union in bytes.
+    /// Returns an estimate of the union's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.state.estimated_size()
+        self.state.estimated_size()
     }
 }
 

datasketches/src/thetafamily/tuple/intersection.rs

@@ -138,9 +138,9 @@ where
         self.state.has_result()
     }
 
-    /// Returns the estimated size of the intersection in bytes.
+    /// Returns an estimate of the intersection's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.state.estimated_size()
+        self.state.estimated_size()
     }
 
     /// Returns the current intersection as a compact Tuple sketch.

datasketches/src/thetafamily/tuple/sketch.rs

@@ -404,9 +404,9 @@ where
         .expect("theta should always be valid")
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.table.estimated_size()
+        size_of::<P>() + self.table.estimated_size()
     }
 }
 
@@ -541,10 +541,9 @@ impl<S> CompactTupleSketch<S> {
         .expect("compact theta should always be valid")
     }
 
-    /// Returns the estimated size of the sketch in bytes.
+    /// Returns an estimate of the sketch's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>()
-            + self.compact_state.retained_entries_capacity() * size_of::<TupleEntry<S>>()
+        self.compact_state.retained_entries_capacity() * size_of::<TupleEntry<S>>()
     }
 
     fn preamble_longs(&self) -> u8 {

datasketches/src/thetafamily/tuple/union.rs

@@ -112,9 +112,9 @@ where
         self.state.reset();
     }
 
-    /// Returns the estimated size of the union in bytes.
+    /// Returns an estimate of the union's memory usage in bytes.
     pub fn estimated_size(&self) -> usize {
-        size_of::<Self>() + self.state.estimated_size()
+        self.state.estimated_size()
     }
 }
 

tests-integration/tests/bloom_test/sketch.rs

@@ -211,6 +211,12 @@ fn test_requested_size_rounds_to_word_boundary() {
     assert_eq!(filter.capacity(), 128);
 }
 
+#[test]
+fn test_estimated_size() {
+    let filter = BloomFilterBuilder::with_size(65, 3).build().unwrap();
+    assert_eq!(filter.estimated_size(), 16);
+}
+
 #[test]
 fn test_accuracy_builder_rejects_zero_items_at_build() {
     let error = BloomFilterBuilder::with_accuracy(0, 0.01)

tests-integration/tests/countmin_test/sketch.rs

@@ -445,11 +445,11 @@ fn test_increment_single_key_like_rust_count_min_sketch() {
 #[test]
 fn test_estimated_size() {
     let mut sketch = CountMinSketch::<i64>::new(4, 128).unwrap();
-    assert_eq!(sketch.estimated_size(), 4224);
+    assert_eq!(sketch.estimated_size(), 4128);
 
     // The backing tables are allocated up front; updates do not grow the sketch.
     sketch.update("apple");
-    assert_eq!(sketch.estimated_size(), 4224);
+    assert_eq!(sketch.estimated_size(), 4128);
 }
 
 #[test]

tests-integration/tests/cpc_test/union.rs

@@ -183,12 +183,12 @@ fn test_lg_k_too_large_returns_error() {
 #[test]
 fn test_union_estimated_size() {
     let mut union = CpcUnion::new(11).unwrap();
-    assert_eq!(union.estimated_size(), 112);
+    assert_eq!(union.estimated_size(), 0);
 
     let mut sketch = CpcSketch::new(11).unwrap();
     for i in 0..1000 {
         sketch.update(i);
     }
     union.update(&sketch).unwrap();
-    assert_eq!(union.estimated_size(), 16496);
+    assert_eq!(union.estimated_size(), 16384);
 }