From c8a52ca4397d943bce90ecccf0306327fefec2a3 Mon Sep 17 00:00:00 2001 From: "edouard.mulliez" Date: Thu, 8 Oct 2026 15:44:46 +0200 Subject: [PATCH 1/3] feat(hll): expose composite cardinality estimates --- CHANGELOG.md | 1 + datasketches/src/hll/array4.rs | 5 + datasketches/src/hll/array6.rs | 5 + datasketches/src/hll/array8.rs | 5 + datasketches/src/hll/estimator.rs | 2 +- datasketches/src/hll/sketch.rs | 37 ++++++ datasketches/src/hll/union.rs | 22 ++++ tests-integration/tests/hll_test/estimate.rs | 130 +++++++++++++++++++ tests-integration/tests/hll_test/main.rs | 1 + 9 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 tests-integration/tests/hll_test/estimate.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f366e11..5cfc3dfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All significant changes to this project will be documented in this file. ### New features +* Add `HllSketch::composite_estimate` and `HllUnion::composite_estimate` to query cardinality without relying on HIP update history, matching Java's `getCompositeEstimate()`. * `KllSketch` is now available behind the `kll` feature, with rank, quantile, PMF, and CDF queries, merging, serialization, custom ordered item types, and a `KllFloat` adapter for non-NaN floating-point values. * Add `TDigestMut::quantiles` and `TDigest::quantiles` for querying several ranks in one centroid scan while preserving input order. * Add `FromIterator` as an alternative to incremental `TDigestMut::merge` calls for combining owned partial sketches in a single compression pass. diff --git a/datasketches/src/hll/array4.rs b/datasketches/src/hll/array4.rs index 1a130593..19c96320 100644 --- a/datasketches/src/hll/array4.rs +++ b/datasketches/src/hll/array4.rs @@ -241,6 +241,11 @@ impl Array4 { .estimate(self.lg_config_k, self.cur_min, self.num_at_cur_min) } + pub fn composite_estimate(&self) -> f64 { + self.estimator + .composite_estimate(self.lg_config_k, self.cur_min, self.num_at_cur_min) + } + pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 { self.estimator.upper_bound( self.lg_config_k, diff --git a/datasketches/src/hll/array6.rs b/datasketches/src/hll/array6.rs index a1b00327..c29a9015 100644 --- a/datasketches/src/hll/array6.rs +++ b/datasketches/src/hll/array6.rs @@ -123,6 +123,11 @@ impl Array6 { self.estimator.estimate(self.lg_config_k, 0, self.num_zeros) } + pub fn composite_estimate(&self) -> f64 { + self.estimator + .composite_estimate(self.lg_config_k, 0, self.num_zeros) + } + pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 { self.estimator .upper_bound(self.lg_config_k, 0, self.num_zeros, num_std_dev) diff --git a/datasketches/src/hll/array8.rs b/datasketches/src/hll/array8.rs index a2f60aa5..355bcc18 100644 --- a/datasketches/src/hll/array8.rs +++ b/datasketches/src/hll/array8.rs @@ -87,6 +87,11 @@ impl Array8 { self.estimator.estimate(self.lg_config_k, 0, self.num_zeros) } + pub fn composite_estimate(&self) -> f64 { + self.estimator + .composite_estimate(self.lg_config_k, 0, self.num_zeros) + } + pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 { self.estimator .upper_bound(self.lg_config_k, 0, self.num_zeros, num_std_dev) diff --git a/datasketches/src/hll/estimator.rs b/datasketches/src/hll/estimator.rs index 1a228091..457dff8e 100644 --- a/datasketches/src/hll/estimator.rs +++ b/datasketches/src/hll/estimator.rs @@ -182,7 +182,7 @@ impl Estimator { } /// Selects between bias-corrected HLL and bitmap estimates using their empirical crossover. - fn composite_estimate(&self, lg_config_k: u8, cur_min: u8, num_at_cur_min: u32) -> f64 { + pub fn composite_estimate(&self, lg_config_k: u8, cur_min: u8, num_at_cur_min: u32) -> f64 { let raw_estimate = self.raw_estimate(lg_config_k); let x_arr = composite_interpolation::get_x_arr(lg_config_k); diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 437062f3..de54bc42 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -209,6 +209,10 @@ impl HllSketch { /// Returns the current cardinality estimate. /// + /// In HLL mode, uses the more accurate HIP estimator when the update history is available, + /// and the composite estimator otherwise. Use [`composite_estimate`](Self::composite_estimate) + /// to query the composite estimator regardless of update history. + /// /// # Examples /// /// ``` @@ -232,6 +236,39 @@ impl HllSketch { } } + /// Returns the cardinality estimate using the composite estimator. + /// + /// In HLL mode, this derives the estimate from the registers rather than the HIP update + /// history. It is less accurate than HIP but allows callers to compare estimates without + /// depending on insertion order. In LIST and SET modes, it returns the same coupon-count + /// estimate as [`estimate`](Self::estimate). + /// + /// This is equivalent to Java's `HllSketch.getCompositeEstimate()`. It does not modify the + /// sketch or invalidate the HIP estimator. + /// + /// # Examples + /// + /// ``` + /// use datasketches::hll::HllSketch; + /// use datasketches::hll::HllType; + /// + /// let mut sketch = HllSketch::new(12, HllType::Hll4).unwrap(); + /// for value in 0..10_000 { + /// sketch.update(value); + /// } + /// let estimate = sketch.composite_estimate(); + /// assert!((estimate - 10_000.0).abs() < 500.0); + /// ``` + pub fn composite_estimate(&self) -> f64 { + match &self.mode { + Mode::List { list, .. } => list.container().estimate(), + Mode::Set { set, .. } => set.container().estimate(), + Mode::Array4(arr) => arr.composite_estimate(), + Mode::Array6(arr) => arr.composite_estimate(), + Mode::Array8(arr) => arr.composite_estimate(), + } + } + /// Returns the upper confidence bound for `num_std_dev` standard deviations. /// /// # Examples diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index f16cebcc..752c2e9d 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -307,6 +307,28 @@ impl HllUnion { self.gadget.estimate() } + /// Returns the union's cardinality estimate using the composite estimator. + /// + /// Uses [`HllSketch::composite_estimate`] regardless of whether the union retains valid HIP + /// update history. This is equivalent to Java's `Union.getCompositeEstimate()` and does not + /// modify the union. + /// + /// # Examples + /// + /// ``` + /// use datasketches::hll::HllUnion; + /// + /// let mut union = HllUnion::new(12).unwrap(); + /// for value in 0..10_000 { + /// union.update_value(value); + /// } + /// let estimate = union.composite_estimate(); + /// assert!((estimate - 10_000.0).abs() < 500.0); + /// ``` + pub fn composite_estimate(&self) -> f64 { + self.gadget.composite_estimate() + } + /// Returns the upper confidence bound for `num_std_dev` standard deviations. /// /// # Examples diff --git a/tests-integration/tests/hll_test/estimate.rs b/tests-integration/tests/hll_test/estimate.rs new file mode 100644 index 00000000..37242598 --- /dev/null +++ b/tests-integration/tests/hll_test/estimate.rs @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datasketches::hll::HllSketch; +use datasketches::hll::HllType; +use datasketches::hll::HllUnion; +use googletest::assert_that; +use googletest::prelude::near; + +const HLL_TYPES: [HllType; 3] = [HllType::Hll4, HllType::Hll6, HllType::Hll8]; + +fn make_sketch(hll_type: HllType, values: impl Iterator) -> HllSketch { + let mut sketch = HllSketch::new(11, hll_type).unwrap(); + for value in values { + sketch.update(value); + } + sketch +} + +#[test] +fn composite_estimate_matches_coupon_estimate_in_list_and_set_modes() { + for hll_type in HLL_TYPES { + for n in [0, 3, 100] { + let sketch = make_sketch(hll_type, 0..n); + assert_eq!(sketch.composite_estimate(), sketch.estimate()); + } + } +} + +#[test] +fn composite_estimate_is_independent_of_insertion_order() { + for hll_type in HLL_TYPES { + for n in [1_000, 10_000, 100_000] { + let forward = make_sketch(hll_type, 0..n); + let reverse = make_sketch(hll_type, (0..n).rev()); + assert_ne!(forward.estimate(), reverse.estimate()); + assert_that!( + forward.composite_estimate(), + near(reverse.composite_estimate(), 1e-9), + "type={hll_type:?}, n={n}" + ); + } + } +} + +#[test] +fn composite_estimate_matches_estimate_after_register_merge() { + for hll_type in HLL_TYPES { + for n in [1_000, 10_000, 100_000] { + let sketch = make_sketch(hll_type, 0..n); + let mut union = HllUnion::new(sketch.lg_config_k()).unwrap(); + union.update(&sketch); + union.update(&sketch); + assert_that!(sketch.composite_estimate(), near(union.estimate(), 1e-9)); + assert_eq!(union.composite_estimate(), union.estimate()); + for result_type in HLL_TYPES { + assert_that!( + union.to_sketch(result_type).composite_estimate(), + near(union.estimate(), 1e-9) + ); + } + } + } +} + +#[test] +fn composite_estimate_preserves_hip_state_and_subsequent_updates() { + for hll_type in HLL_TYPES { + let mut sketch = make_sketch(hll_type, 0..10_000); + let mut control = sketch.clone(); + let estimate = sketch.estimate(); + let bytes = sketch.serialize(); + assert_ne!(sketch.composite_estimate(), estimate); + assert_eq!(sketch.estimate(), estimate); + assert_eq!(sketch.serialize(), bytes); + + let decoded = HllSketch::deserialize(&bytes).unwrap(); + assert_eq!(decoded.composite_estimate(), sketch.composite_estimate()); + assert_eq!(decoded.estimate(), estimate); + + for value in 10_000..20_000_u64 { + sketch.update(value); + control.update(value); + } + assert_eq!(sketch.serialize(), control.serialize()); + } +} + +#[test] +fn union_composite_estimate_works_with_direct_updates_and_single_sketch() { + for n in [0, 3, 100, 10_000] { + let sketch = make_sketch(HllType::Hll8, 0..n); + let mut direct = HllUnion::new(11).unwrap(); + for value in 0..n { + direct.update_value(value); + } + let direct_estimate = direct.estimate(); + assert_eq!(direct.composite_estimate(), sketch.composite_estimate()); + assert_eq!(direct.estimate(), direct_estimate); + + for hll_type in HLL_TYPES { + let sketch = make_sketch(hll_type, 0..n); + let mut union = HllUnion::new(11).unwrap(); + union.update(&sketch); + let hip_estimate = union.estimate(); + assert_eq!(hip_estimate, sketch.estimate()); + assert_that!( + union.composite_estimate(), + near(sketch.composite_estimate(), 1e-9) + ); + assert_eq!(union.estimate(), hip_estimate); + union.reset(); + assert_eq!(union.composite_estimate(), 0.0); + } + } +} diff --git a/tests-integration/tests/hll_test/main.rs b/tests-integration/tests/hll_test/main.rs index 30fb7dce..5a892ff7 100644 --- a/tests-integration/tests/hll_test/main.rs +++ b/tests-integration/tests/hll_test/main.rs @@ -16,5 +16,6 @@ // under the License. mod bounds; +mod estimate; mod union; mod update; From 24a5bf38fdc5cd6d22deadce358f45e96ec469c8 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 8 Oct 2026 23:25:15 +0800 Subject: [PATCH 2/3] docs(hll): drop implementation details from composite estimate docs User-facing docs intentionally avoid internal modes and Java method references; describe only the observable contract instead. --- datasketches/src/hll/sketch.rs | 15 ++++----------- datasketches/src/hll/union.rs | 4 +--- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index de54bc42..491fbcf8 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -209,10 +209,6 @@ impl HllSketch { /// Returns the current cardinality estimate. /// - /// In HLL mode, uses the more accurate HIP estimator when the update history is available, - /// and the composite estimator otherwise. Use [`composite_estimate`](Self::composite_estimate) - /// to query the composite estimator regardless of update history. - /// /// # Examples /// /// ``` @@ -238,13 +234,10 @@ impl HllSketch { /// Returns the cardinality estimate using the composite estimator. /// - /// In HLL mode, this derives the estimate from the registers rather than the HIP update - /// history. It is less accurate than HIP but allows callers to compare estimates without - /// depending on insertion order. In LIST and SET modes, it returns the same coupon-count - /// estimate as [`estimate`](Self::estimate). - /// - /// This is equivalent to Java's `HllSketch.getCompositeEstimate()`. It does not modify the - /// sketch or invalidate the HIP estimator. + /// The estimate is derived only from the sketch's registers, so it does not depend on the + /// order in which values were inserted. This can be less accurate than + /// [`estimate`](Self::estimate); use it to compare estimates across sketches that saw the + /// same values in different orders. This method does not modify the sketch. /// /// # Examples /// diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index 752c2e9d..4807465e 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -309,9 +309,7 @@ impl HllUnion { /// Returns the union's cardinality estimate using the composite estimator. /// - /// Uses [`HllSketch::composite_estimate`] regardless of whether the union retains valid HIP - /// update history. This is equivalent to Java's `Union.getCompositeEstimate()` and does not - /// modify the union. + /// See [`HllSketch::composite_estimate`]. This method does not modify the union. /// /// # Examples /// From b9f79700b5caea1e824f0954071eaaf980f1a3c1 Mon Sep 17 00:00:00 2001 From: tison Date: Fri, 9 Oct 2026 00:55:39 +0800 Subject: [PATCH 3/3] Apply batched suggestions from code review Co-authored-by: tison --- datasketches/src/hll/sketch.rs | 8 +++----- datasketches/src/hll/union.rs | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs index 491fbcf8..06dd823a 100644 --- a/datasketches/src/hll/sketch.rs +++ b/datasketches/src/hll/sketch.rs @@ -232,12 +232,10 @@ impl HllSketch { } } - /// Returns the cardinality estimate using the composite estimator. + /// Returns the current cardinality estimate that never depends on + /// the order in which values were inserted. /// - /// The estimate is derived only from the sketch's registers, so it does not depend on the - /// order in which values were inserted. This can be less accurate than - /// [`estimate`](Self::estimate); use it to compare estimates across sketches that saw the - /// same values in different orders. This method does not modify the sketch. + /// This can be less accurate than [`estimate`](Self::estimate). /// /// # Examples /// diff --git a/datasketches/src/hll/union.rs b/datasketches/src/hll/union.rs index 4807465e..0f6e20ec 100644 --- a/datasketches/src/hll/union.rs +++ b/datasketches/src/hll/union.rs @@ -309,7 +309,7 @@ impl HllUnion { /// Returns the union's cardinality estimate using the composite estimator. /// - /// See [`HllSketch::composite_estimate`]. This method does not modify the union. + /// See [`HllSketch::composite_estimate`]. /// /// # Examples ///