diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 841f8509..59a69ac7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,16 @@ Prepare the cross-language serialization test data: cargo x prepare-testdata ``` +Generate deterministic snapshots from the Rust implementations: + +```shell +cargo x generate-snapshots +``` + +The command writes local files under `serialization/rust/snapshots`. These +generated files are ignored by Git, so the command can be rerun without +changing the working tree. + Test: ```shell diff --git a/Cargo.lock b/Cargo.lock index ab02dc69..b9087396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -696,6 +696,14 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "snapshot_generator" +version = "0.0.0" +dependencies = [ + "clap", + "datasketches", +] + [[package]] name = "socks" version = "0.3.4" diff --git a/Cargo.toml b/Cargo.toml index 92c47753..677d1c4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,13 @@ # under the License. [workspace] -members = ["benchmarks", "datasketches", "tests-integration", "xtask"] +members = [ + "benchmarks", + "datasketches", + "snapshot_generator", + "tests-integration", + "xtask", +] resolver = "3" [workspace.package] diff --git a/snapshot_generator/Cargo.toml b/snapshot_generator/Cargo.toml new file mode 100644 index 00000000..7a74fa47 --- /dev/null +++ b/snapshot_generator/Cargo.toml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "snapshot_generator" +publish = false + +edition.workspace = true +homepage.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +clap = { workspace = true } +datasketches = { workspace = true, features = ["bloom", "countmin"] } + +[lints] +workspace = true diff --git a/snapshot_generator/src/main.rs b/snapshot_generator/src/main.rs new file mode 100644 index 00000000..0c754ec2 --- /dev/null +++ b/snapshot_generator/src/main.rs @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fs; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +use clap::Parser; +use datasketches::bloom::BloomFilterBuilder; +use datasketches::countmin::CountMinSketch; + +#[derive(Parser)] +#[command(about = "Generate deterministic Rust serialization snapshots")] +struct Arguments { + /// Directory in which snapshot files are written. + #[arg(long, default_value = "serialization/rust/snapshots")] + output: PathBuf, +} + +fn main() -> Result<(), Box> { + let arguments = Arguments::parse(); + fs::create_dir_all(&arguments.output)?; + + write_snapshot( + &arguments.output, + "bloom_empty_rust.sk", + &BloomFilterBuilder::with_accuracy(128, 0.01) + .build()? + .serialize(), + )?; + + let mut bloom = BloomFilterBuilder::with_accuracy(128, 0.01).build()?; + for value in ["alpha", "beta", "gamma"] { + bloom.insert(value); + } + write_snapshot( + &arguments.output, + "bloom_non_empty_rust.sk", + &bloom.serialize(), + )?; + + let empty_countmin = CountMinSketch::::with_seed(4, 32, 9001)?; + write_snapshot( + &arguments.output, + "count_min_empty_rust.sk", + &empty_countmin.serialize(), + )?; + + let mut countmin = CountMinSketch::::with_seed(4, 32, 9001)?; + for (value, weight) in [("alpha", 3), ("beta", 2), ("gamma", 5)] { + for _ in 0..weight { + countmin.update(value); + } + } + write_snapshot( + &arguments.output, + "count_min_non_empty_rust.sk", + &countmin.serialize(), + )?; + + Ok(()) +} + +fn write_snapshot(output: &Path, name: &str, bytes: &[u8]) -> io::Result<()> { + let path = output.join(name); + fs::write(&path, bytes)?; + println!("wrote {} ({} bytes)", path.display(), bytes.len()); + Ok(()) +} diff --git a/snapshot_generator/tests/snapshots.rs b/snapshot_generator/tests/snapshots.rs new file mode 100644 index 00000000..ec0c06f5 --- /dev/null +++ b/snapshot_generator/tests/snapshots.rs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fs; +use std::process::Command; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +#[test] +fn generates_identical_snapshots_across_processes() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "datasketches-snapshots-{}-{nonce}", + std::process::id() + )); + let first = root.join("first"); + let second = root.join("second"); + for output in [&first, &second] { + let status = Command::new(env!("CARGO_BIN_EXE_snapshot_generator")) + .arg("--output") + .arg(output) + .status() + .unwrap(); + assert!(status.success(), "snapshot generation failed"); + assert_eq!(fs::read_dir(output).unwrap().count(), 4); + } + for name in [ + "bloom_empty_rust.sk", + "bloom_non_empty_rust.sk", + "count_min_empty_rust.sk", + "count_min_non_empty_rust.sk", + ] { + let first_bytes = fs::read(first.join(name)).unwrap(); + assert!(!first_bytes.is_empty(), "empty snapshot: {name}"); + assert_eq!( + first_bytes, + fs::read(second.join(name)).unwrap(), + "nondeterministic snapshot: {name}" + ); + } + fs::remove_dir_all(root).unwrap(); +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9b3fb1f2..dc788cb2 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -49,6 +49,7 @@ impl Command { SubCommand::Lint(cmd) => cmd.run(), SubCommand::Test(cmd) => cmd.run(), SubCommand::PrepareTestData(cmd) => cmd.run(), + SubCommand::GenerateSnapshots(cmd) => cmd.run(), } } } @@ -70,6 +71,11 @@ enum SubCommand { about = "Prepare serialization compatibility test data." )] PrepareTestData(CommandPrepareTestData), + #[clap( + name = "generate-snapshots", + about = "Generate deterministic Rust serialization snapshots." + )] + GenerateSnapshots(CommandGenerateSnapshots), } #[derive(Parser)] @@ -409,3 +415,15 @@ impl CommandPrepareTestData { languages } } + +#[derive(Parser)] +#[clap(name = "generate-snapshots")] +struct CommandGenerateSnapshots {} + +impl CommandGenerateSnapshots { + fn run(self) { + let mut command = find_command("cargo"); + command.args(["run", "--package", "snapshot_generator", "--"]); + run_command(command); + } +}