diff --git a/.gitignore b/.gitignore index 4533f07..770f3fd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ worktrees/* .idea/ .vscode/ *.swp +/live-module-coverage.json diff --git a/Cargo.lock b/Cargo.lock index f41156b..fe72438 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,6 +63,18 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -222,6 +234,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "foreign-types" version = "0.3.2" @@ -400,6 +418,20 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mp4" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9ef834d5ed55e494a2ae350220314dc4aacd1c43a9498b00e320e0ea352a5c3" +dependencies = [ + "byteorder", + "bytes", + "num-rational", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "ndk" version = "0.9.0" @@ -429,6 +461,25 @@ dependencies = [ "jni-sys 0.3.1", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-derive" version = "0.4.2" @@ -440,6 +491,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -576,6 +648,41 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "opus-decoder" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ae32c275dabb0cd2c863460bf349e9abc3568ba2abf0f9eb7b7d2edeeed07e" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "opus-pure" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de7c2ad44427a636b27bf0b03eb29e713775ba1b45ff31a0618c19ee5adb215" + +[[package]] +name = "oxideav-aac" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35453ffa827080f322dba5d2233879f008627d4ffb07c496c0c6fe3190b35048" +dependencies = [ + "oxideav-core", +] + +[[package]] +name = "oxideav-core" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64502aa4210ff04a5d574ca86f2c22e064a6c3bda812914b138c777e8ce6b023" +dependencies = [ + "bytemuck", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -663,6 +770,12 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "rustc_version" version = "0.4.1" @@ -770,6 +883,117 @@ version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-pcm", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "2.0.119" @@ -836,12 +1060,18 @@ dependencies = [ name = "tinyvoice" version = "0.3.2" dependencies = [ + "bytes", "cpal", "log", + "mp4", + "opus-decoder", + "opus-pure", + "oxideav-aac", "parking_lot", "rdev", "serde", "serde_json", + "symphonia", "thiserror 2.0.21", "tinyvoice-bus", "tokio", diff --git a/MODULE.md b/MODULE.md index 89e0090..a4fb654 100644 --- a/MODULE.md +++ b/MODULE.md @@ -4,12 +4,15 @@ This package contains the native `tinyvoice` module for TinyBus module ABI v1. Install only the archive matching the host operating system and architecture. The module claims `ai.tinyhumans.tinyvoice.Voice`, serves the object at -`/ai/tinyhumans/tinyvoice/Voice`, and provides 53 methods under contract 1.10. +`/ai/tinyhumans/tinyvoice/Voice`, and provides 56 methods under contract 1.11. ## Methods | Method | Arguments | Returns | | --- | --- | --- | +| `CodecProbe` | `CodecInput { audio_base64 }` | `CodecInfo`: detected codec, duration, sample rate, channels | +| `CodecTranscode` | `CodecTranscodeRequest { audio_base64, from, target }` | `CodecOutput { audio_base64 }` | +| `CodecWaveform` | `WaveformRequest { audio_base64, buckets }` | `WaveformResult { peaks }` | | `Route` | `transcript` | JSON `VoiceIntent`, tagged by an `intent` field | | `ExtractCommand` | `transcript`, `wake_word` | The command after the wake word, or `""` | | `WakeWordPresent` | `transcript`, `wake_word` | `bool` | diff --git a/README.md b/README.md index 19e4f3a..6465c29 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TinyVoice -Host-agnostic voice primitives, extracted from [OpenHuman](https://github.com/tinyhumansai/openhuman): WAV framing, silence gating, voice-activity segmentation, wake-word gating, fast-path command routing, STT hallucination detection, and relay-turn preparation. +Host-agnostic voice primitives, extracted from [OpenHuman](https://github.com/tinyhumansai/openhuman): WAV framing and voice-note codecs, silence gating, voice-activity segmentation, wake-word gating, fast-path command routing, STT hallucination detection, and relay-turn preparation. Ships two things from one repository: @@ -25,6 +25,7 @@ permission check. | Here | With the host | | --- | --- | | WAV framing, RMS, resampling, downmix, silence gate; module-owned capture threads | Microphone permission decisions through the computer module | +| In-memory WAV/PCM16, Ogg/Opus, and M4A/AAC-LC probe, transcode, and waveform APIs | Codec calls through the `tinyvoice-bus` module contract | | VAD segmentation | The capture loop that drives it | | Wake-word gate, intent routing | What to *do* with an intent | | Hallucination detection | The STT transport, credentials, and retry policy | @@ -66,6 +67,11 @@ turn. It performs no inference or tool execution. A module host uses the OpenHuman supplies its local agent, acknowledgment policy and delivery. See the [relay contract](docs/specs/relay-turn-preparation.md). +The [`tinyvoice::codec` API](docs/specs/voice-codecs.md) detects, probes, +transcodes, and extracts waveform peaks from supported voice-note formats. It +uses pure-Rust codecs and in-memory containers; it does not require FFmpeg or +system codec libraries. + ### Optional features - `hotkey` (off by default): `tinyvoice::hotkey`, a system-wide push-to-talk / diff --git a/crates/tinyvoice-bus/src/codec.rs b/crates/tinyvoice-bus/src/codec.rs new file mode 100644 index 0000000..9fda4ba --- /dev/null +++ b/crates/tinyvoice-bus/src/codec.rs @@ -0,0 +1,71 @@ +//! Typed, transport-neutral requests for voice-note codec operations. + +use serde::{Deserialize, Serialize}; + +/// Supported encoded voice-note media. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AudioCodec { + /// Signed 16-bit little-endian PCM in WAV. + WavPcm16, + /// Opus audio in Ogg. + OggOpus, + /// AAC-LC audio in M4A. + M4aAac, + /// MPEG Layer III audio (source format). + Mp3, +} + +/// Input bytes encoded as standard Base64. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CodecInput { + /// Base64-encoded complete media file. + pub audio_base64: String, +} + +/// Convert media to the requested voice-note codec. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CodecTranscodeRequest { + /// Base64-encoded source media. + pub audio_base64: String, + /// Declared source container and codec, checked against the payload. + pub from: AudioCodec, + /// Destination container and codec. + pub target: AudioCodec, +} + +/// Encoded media returned from a transcode operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CodecOutput { + /// Base64-encoded complete media file. + pub audio_base64: String, +} + +/// Audio properties decoded from a media file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct CodecInfo { + /// Detected codec. + pub codec: AudioCodec, + /// Duration rounded down to whole milliseconds. + pub duration_ms: u64, + /// Original sample rate in Hz. + pub sample_rate: u32, + /// Original audio channel count. + pub channels: u16, +} + +/// Extract a fixed-size normalized waveform. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WaveformRequest { + /// Base64-encoded source media. + pub audio_base64: String, + /// Requested number of buckets, limited to 1 through 4096. + pub buckets: usize, +} + +/// Normalized peak amplitude bytes, one per requested bucket. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WaveformResult { + /// Peak values in the range 0 through 255. + pub peaks: Vec, +} diff --git a/crates/tinyvoice-bus/src/lib.rs b/crates/tinyvoice-bus/src/lib.rs index 8dac566..7c4b53a 100644 --- a/crates/tinyvoice-bus/src/lib.rs +++ b/crates/tinyvoice-bus/src/lib.rs @@ -13,6 +13,7 @@ mod error; pub mod capture; +pub mod codec; pub mod hotkey; pub mod intent; pub mod live; @@ -24,6 +25,10 @@ pub mod transcript; pub mod vad; pub mod version; +pub use codec::{ + AudioCodec, CodecInfo, CodecInput, CodecOutput, CodecTranscodeRequest, WaveformRequest, + WaveformResult, +}; pub use error::{Error, Result}; pub use hotkey::*; pub use intent::VoiceIntent; diff --git a/crates/tinyvoice-bus/src/names/mod.rs b/crates/tinyvoice-bus/src/names/mod.rs index 89c9704..7364645 100644 --- a/crates/tinyvoice-bus/src/names/mod.rs +++ b/crates/tinyvoice-bus/src/names/mod.rs @@ -8,6 +8,12 @@ pub const OBJECT_PATH: &str = "/ai/tinyhumans/tinyvoice/Voice"; /// One constant per member of [`BUS_NAME`]. pub mod methods { + /// Decode media and return its container and audio properties. + pub const CODEC_PROBE: &str = "CodecProbe"; + /// Convert an in-memory voice-note audio payload. + pub const CODEC_TRANSCODE: &str = "CodecTranscode"; + /// Decode media and return normalized waveform peak bytes. + pub const CODEC_WAVEFORM: &str = "CodecWaveform"; /// Fix explicit host concurrency and reservation bounds. pub const RELAY_CONFIGURE: &str = "RelayConfigure"; /// Reserve a relay turn before local-brain startup. @@ -128,6 +134,9 @@ pub const METHODS: &[&str] = &[ methods::CAPTURE_SHUTDOWN, methods::CAPTURE_START, methods::CAPTURE_STOP, + methods::CODEC_PROBE, + methods::CODEC_TRANSCODE, + methods::CODEC_WAVEFORM, methods::ENCODE_WAV, methods::ENCODE_WAV_PCM16, methods::EXTRACT_COMMAND, diff --git a/crates/tinyvoice-bus/src/version/mod.rs b/crates/tinyvoice-bus/src/version/mod.rs index 88c0748..e2df0e6 100644 --- a/crates/tinyvoice-bus/src/version/mod.rs +++ b/crates/tinyvoice-bus/src/version/mod.rs @@ -1,7 +1,7 @@ //! The `TinyVoice` `TinyBus` contract version and its binding rule. /// The wire contract version this crate defines. -pub const CONTRACT_VERSION: (u32, u32) = (1, 10); +pub const CONTRACT_VERSION: (u32, u32) = (1, 11); /// Returns whether a host using [`CONTRACT_VERSION`] can bind to `module`. /// diff --git a/crates/tinyvoice-module/Cargo.lock b/crates/tinyvoice-module/Cargo.lock index 82aaf2b..2171c70 100644 --- a/crates/tinyvoice-module/Cargo.lock +++ b/crates/tinyvoice-module/Cargo.lock @@ -170,6 +170,18 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -554,6 +566,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.5.0" @@ -1261,6 +1279,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mp4" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9ef834d5ed55e494a2ae350220314dc4aacd1c43a9498b00e320e0ea352a5c3" +dependencies = [ + "byteorder", + "bytes", + "num-rational", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "ndk" version = "0.9.0" @@ -1290,6 +1322,25 @@ dependencies = [ "jni-sys 0.3.1", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-derive" version = "0.4.2" @@ -1301,6 +1352,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1449,6 +1521,32 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus-pure" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de7c2ad44427a636b27bf0b03eb29e713775ba1b45ff31a0618c19ee5adb215" + +[[package]] +name = "oxideav-aac" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35453ffa827080f322dba5d2233879f008627d4ffb07c496c0c6fe3190b35048" +dependencies = [ + "oxideav-core", +] + +[[package]] +name = "oxideav-core" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64502aa4210ff04a5d574ca86f2c22e064a6c3bda812914b138c777e8ce6b023" +dependencies = [ + "bytemuck", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1736,6 +1834,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -2086,6 +2190,117 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-pcm", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "2.0.119" @@ -2146,7 +2361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -2466,12 +2681,17 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" name = "tinyvoice" version = "0.3.2" dependencies = [ + "bytes", "cpal", "log", + "mp4", + "opus-pure", + "oxideav-aac", "parking_lot", "rdev", "serde", "serde_json", + "symphonia", "thiserror 2.0.20", "tinyvoice-bus", "tokio", diff --git a/crates/tinyvoice-module/src/service/live/prepare_tests.rs b/crates/tinyvoice-module/src/service/live/prepare_tests.rs index 1e549b1..92eccd6 100644 --- a/crates/tinyvoice-module/src/service/live/prepare_tests.rs +++ b/crates/tinyvoice-module/src/service/live/prepare_tests.rs @@ -45,7 +45,7 @@ async fn native_configuration_preserves_all_explicit_host_fields() -> anyhow::Re assert_eq!(prepared.config.provider_options, json!({"host": true})); assert_eq!( prepared.options.tool_timeout, - std::time::Duration::from_millis(1000) + std::time::Duration::from_secs(1) ); assert!(!prepared.options.include_deferred_tools); Ok(()) diff --git a/crates/tinyvoice-module/src/service/mod.rs b/crates/tinyvoice-module/src/service/mod.rs index bbb4429..7f21bec 100644 --- a/crates/tinyvoice-module/src/service/mod.rs +++ b/crates/tinyvoice-module/src/service/mod.rs @@ -31,6 +31,15 @@ use tinyvoice::vad::{VadConfig, VadSegmenter}; /// an allocation failure. pub const MAX_AUDIO_BYTES: usize = 8 * 1024 * 1024; +/// Concurrent in-memory codec operations; excess calls fail fast instead of +/// allowing attacker-controlled encoded payloads to build an unbounded queue. +const MAX_CODEC_WORKERS: usize = 2; + +static CODEC_WORKERS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| { + std::sync::Arc::new(tokio::sync::Semaphore::new(MAX_CODEC_WORKERS)) + }); + /// Most sessions a caller may hold open at once. /// /// A session is opened by `VadOpen` and only released by `VadClose`, and a @@ -152,6 +161,26 @@ fn encode_samples(samples: &[f32]) -> String { BASE64.encode(bytes) } +/// Run CPU-heavy, bounded codec work away from the async module worker. +async fn run_codec_worker( + semaphore: std::sync::Arc, + operation: F, +) -> TinyBusResult +where + T: Send + 'static, + F: FnOnce() -> TinyBusResult + Send + 'static, +{ + let permit = semaphore + .try_acquire_owned() + .map_err(|_| tinybus::Error::failed("voice codec worker pool is at capacity"))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + operation() + }) + .await + .map_err(|_| tinybus::Error::failed("voice codec worker failed"))? +} + /// The error for a session id that is not open. /// /// Its own function so the wording is identical across the four methods that @@ -198,9 +227,10 @@ fn to_json(value: &T) -> TinyBusResult { clippy::unused_async, reason = "TinyBus interface methods require async signatures" )] -#[expect( +#[allow(unknown_lints)] +#[allow( clippy::unused_async_trait_impl, - reason = "TinyBus interface methods must retain async signatures at this dynamic module boundary" + reason = "TinyBus interface dispatch awaits every declared async method" )] #[tinybus::interface(name = "ai.tinyhumans.tinyvoice.Voice")] impl VoiceService { @@ -752,6 +782,61 @@ impl VoiceService { Ok(BASE64.encode(wav)) } + /// Decode a complete in-memory media file and report its audio properties. + async fn codec_probe( + &self, + request: tinyvoice_bus::codec::CodecInput, + ) -> TinyBusResult { + run_codec_worker(CODEC_WORKERS.clone(), move || { + let bytes = decode_audio(&request.audio_base64)?; + let info = tinyvoice::codec::probe(&bytes).map_err(|error| failed(&error))?; + Ok(tinyvoice_bus::codec::CodecInfo { + codec: bus_codec(info.codec)?, + duration_ms: info.duration_ms, + sample_rate: info.sample_rate, + channels: info.channels, + }) + }) + .await + } + + /// Transcode one complete in-memory voice note to the requested format. + async fn codec_transcode( + &self, + request: tinyvoice_bus::codec::CodecTranscodeRequest, + ) -> TinyBusResult { + run_codec_worker(CODEC_WORKERS.clone(), move || { + let bytes = decode_audio(&request.audio_base64)?; + let from = library_codec(request.from); + let target = library_codec(request.target); + let output = tinyvoice::codec::transcode(&bytes, from, target) + .map_err(|error| failed(&error))?; + if output.len() > MAX_AUDIO_BYTES { + return Err(tinybus::Error::failed(format!( + "transcoded audio exceeds the {MAX_AUDIO_BYTES} byte limit" + ))); + } + Ok(tinyvoice_bus::codec::CodecOutput { + audio_base64: BASE64.encode(output), + }) + }) + .await + } + + /// Decode audio and return normalized peak amplitudes for fixed buckets. + async fn codec_waveform( + &self, + request: tinyvoice_bus::codec::WaveformRequest, + ) -> TinyBusResult { + run_codec_worker(CODEC_WORKERS.clone(), move || { + let bytes = decode_audio(&request.audio_base64)?; + let peaks = tinyvoice::codec::waveform(&bytes, request.buckets) + .map_err(|error| failed(&error))?; + Ok(tinyvoice_bus::codec::WaveformResult { peaks }) + }) + .await + } + /// Downmix, resample, and silence-gate a captured buffer, returning WAV. /// /// This is the whole capture-side pipeline in one call, because a host that @@ -844,6 +929,29 @@ impl VoiceService { } } +fn library_codec(codec: tinyvoice_bus::codec::AudioCodec) -> tinyvoice::codec::AudioCodec { + match codec { + tinyvoice_bus::codec::AudioCodec::WavPcm16 => tinyvoice::codec::AudioCodec::WavPcm16, + tinyvoice_bus::codec::AudioCodec::OggOpus => tinyvoice::codec::AudioCodec::OggOpus, + tinyvoice_bus::codec::AudioCodec::M4aAac => tinyvoice::codec::AudioCodec::M4aAac, + tinyvoice_bus::codec::AudioCodec::Mp3 => tinyvoice::codec::AudioCodec::Mp3, + } +} + +fn bus_codec( + codec: tinyvoice::codec::AudioCodec, +) -> TinyBusResult { + match codec { + tinyvoice::codec::AudioCodec::WavPcm16 => Ok(tinyvoice_bus::codec::AudioCodec::WavPcm16), + tinyvoice::codec::AudioCodec::OggOpus => Ok(tinyvoice_bus::codec::AudioCodec::OggOpus), + tinyvoice::codec::AudioCodec::M4aAac => Ok(tinyvoice_bus::codec::AudioCodec::M4aAac), + tinyvoice::codec::AudioCodec::Mp3 => Ok(tinyvoice_bus::codec::AudioCodec::Mp3), + _ => Err(tinybus::Error::failed( + "codec is not available on the voice bus", + )), + } +} + /// Claim the bus name and serve the interface. async fn setup(connection: Connection) -> TinyBusResult<()> { connection @@ -903,6 +1011,9 @@ tinybus_module::module_export_optional_static! { "CaptureShutdown", "EncodeWav", "EncodeWavPcm16", + "CodecProbe", + "CodecTranscode", + "CodecWaveform", "PrepareCapture", "ListInputDevices", "RecordingStart", diff --git a/crates/tinyvoice-module/src/service/mod_tests.rs b/crates/tinyvoice-module/src/service/mod_tests.rs index 893d1e9..7ad3844 100644 --- a/crates/tinyvoice-module/src/service/mod_tests.rs +++ b/crates/tinyvoice-module/src/service/mod_tests.rs @@ -20,11 +20,33 @@ use tinybus::broker::Broker; use tinybus::transport::memory::MemoryBus; use tinybus::{Connection, Interface}; use tinyvoice_bus::{ - ActivationMode, HostKeyFact, HotkeyFeedRequest, HotkeyHandleRequest, HotkeyReadRequest, - HotkeyRequest, HotkeyReserveRequest, HotkeySource, SequencedHostFact, + ActivationMode, AudioCodec, CodecInfo, CodecInput, CodecOutput, CodecTranscodeRequest, + HostKeyFact, HotkeyFeedRequest, HotkeyHandleRequest, HotkeyReadRequest, HotkeyRequest, + HotkeyReserveRequest, HotkeySource, SequencedHostFact, WaveformRequest, WaveformResult, }; use tinyvoice_bus::{BUS_NAME, OBJECT_PATH, names}; +#[tokio::test(flavor = "current_thread")] +async fn codec_work_is_offloaded_and_rejects_when_capacity_is_full() { + let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = semaphore + .clone() + .try_acquire_owned() + .expect("test takes the only codec worker slot"); + assert!( + super::run_codec_worker(semaphore.clone(), || Ok(())) + .await + .is_err() + ); + drop(held); + + let executor_thread = std::thread::current().id(); + let worker_thread = super::run_codec_worker(semaphore, || Ok(std::thread::current().id())) + .await + .expect("available codec worker runs"); + assert_ne!(worker_thread, executor_thread); +} + /// A live bus with the interface served on it. /// /// The service `Connection` is held here rather than dropped at the end of @@ -97,6 +119,52 @@ async fn route_returns_a_tagged_intent() -> tinybus::Result<()> { Ok(()) } +#[tokio::test] +async fn voice_note_codec_methods_are_available_over_the_module_contract() -> tinybus::Result<()> { + let proxy = connect().await?; + let wav = include_bytes!("../../../tinyvoice/tests/fixtures/voice-envelope.wav"); + let encoded = BASE64.encode(wav); + + let info: CodecInfo = proxy + .call( + "CodecProbe", + (CodecInput { + audio_base64: encoded.clone(), + },), + ) + .await?; + assert_eq!(info.codec, AudioCodec::WavPcm16); + assert_eq!(info.channels, 1); + + let waveform: WaveformResult = proxy + .call( + "CodecWaveform", + (WaveformRequest { + audio_base64: encoded.clone(), + buckets: 256, + },), + ) + .await?; + assert_eq!(waveform.peaks.len(), 256); + assert!(waveform.peaks.iter().any(|peak| *peak > 0)); + + let opus: CodecOutput = proxy + .call( + "CodecTranscode", + (CodecTranscodeRequest { + audio_base64: encoded, + from: AudioCodec::WavPcm16, + target: AudioCodec::OggOpus, + },), + ) + .await?; + let opus = base64::engine::general_purpose::STANDARD + .decode(opus.audio_base64) + .expect("module returns valid base64"); + assert!(opus.starts_with(b"OggS")); + Ok(()) +} + #[tokio::test] async fn route_carries_the_payload_for_a_parameterised_intent() -> tinybus::Result<()> { let proxy = connect().await?; diff --git a/crates/tinyvoice/Cargo.toml b/crates/tinyvoice/Cargo.toml index bc90375..e471810 100644 --- a/crates/tinyvoice/Cargo.toml +++ b/crates/tinyvoice/Cargo.toml @@ -19,6 +19,16 @@ serde_json = "1" tinyvoice-bus = { version = "0.1.4", path = "../tinyvoice-bus" } # Derive macros for the crate-wide error type in `src/error/mod.rs`. thiserror = "2" +# Pure-Rust Opus encoding, decoding, and Ogg encapsulation for voice clips. +opus-pure = "0.2.2" +# Pure-Rust AAC-LC encoder for M4A output. +oxideav-aac = "0.1.7" +# Pure-Rust audio container/codecs for AAC-in-M4A and PCM WAV input. +symphonia = { version = "0.6.1", default-features = false, features = ["aac", "isomp4", "mp3", "pcm", "wav"] } +# In-memory ISO-MP4 muxing for AAC access units. +mp4 = "0.14" +# MP4 sample payload buffer type used by the in-memory muxer. +bytes = "1" # `hotkey` only: the global keyboard listener and its plumbing. rdev = { version = "0.5", optional = true } @@ -32,6 +42,7 @@ tokio = { version = "1", default-features = false, features = ["sync"], optional [dev-dependencies] # The `capture` tests drive async handles with `#[tokio::test]`. tokio = { version = "1", features = ["macros", "rt", "sync"] } +opus-decoder = "0.1.1" [features] default = [] diff --git a/crates/tinyvoice/src/codec/README.md b/crates/tinyvoice/src/codec/README.md new file mode 100644 index 0000000..7c6b0d7 --- /dev/null +++ b/crates/tinyvoice/src/codec/README.md @@ -0,0 +1,22 @@ +# Voice codec module + +`mod.rs` owns the pure, in-memory public API for probing, transcoding, and +waveform extraction. It detects WAV/PCM16, Ogg/Opus, M4A/AAC-LC, and MP3 input; +transcode callers also provide the source format so mismatches are rejected. +MP3 is decode-only. Compressed destinations are encoded as mono 48 kHz; WAV to +WAV preserves the bytes. + +`codec_tests.rs` covers internal format constraints and malformed muxer data. +The public integration coverage is in `tests/codecs.rs`, including fixture +provenance and interoperability checks in `tests/fixtures/README.md`. + +Codec operations use pure-Rust dependencies: `opus-pure` for Opus/Ogg, +Symphonia for AAC/M4A, MP3, and WAV decoding, `oxideav-aac` for AAC-LC encoding, +and `mp4` for in-memory M4A muxing. Input is limited to 32 MiB; decoded audio is +limited to five minutes and 32 million samples, with sample rates between 8 and +192 kHz. No codec path uses filesystem, network, process, or device I/O. + +The host-facing surface is the typed `tinyvoice-bus` contract and its compiled +`tinyvoice-module` adapter. Hosts must not call this implementation crate +directly for module-owned behavior. See the [codec specification](../../../../docs/specs/voice-codecs.md) +for backend and bus details. diff --git a/crates/tinyvoice/src/codec/codec_tests.rs b/crates/tinyvoice/src/codec/codec_tests.rs new file mode 100644 index 0000000..9692b18 --- /dev/null +++ b/crates/tinyvoice/src/codec/codec_tests.rs @@ -0,0 +1,500 @@ +//! Internal codec boundaries, limits, and container-repair behavior. + +use super::*; + +#[test] +fn m4a_repair_rejects_invalid_muxer_output_without_panicking() { + assert!(repair_m4a(Vec::new(), 1).is_err()); + assert!(repair_m4a(b"not an MP4".to_vec(), 1).is_err()); +} + +#[test] +fn m4a_sl_config_descriptor_has_its_required_predefined_byte() -> Result<()> { + let (raw_m4a, duration_samples) = stereo_aac_muxed_fixture()?; + let raw_esds_type = find_last(&raw_m4a, b"esds").ok_or_else(|| codec_error("missing esds"))?; + let raw_esds_end = raw_esds_type - 4 + read_be_u32(&raw_m4a, raw_esds_type - 4)? as usize; + let raw_descriptors = raw_m4a + .get(raw_esds_type + 8..raw_esds_end) + .ok_or_else(|| codec_error("truncated raw esds"))?; + let raw_sl_config = raw_descriptors + .windows(2) + .position(|window| window == [0x06, 0x00]) + .ok_or_else(|| codec_error("missing raw SLConfig descriptor"))?; + assert_eq!( + &raw_descriptors[raw_sl_config..raw_sl_config + 3], + &[0x06, 0x00, 0x00] + ); + + let m4a = repair_m4a(raw_m4a, duration_samples)?; + let esds_type = find_last(&m4a, b"esds").ok_or_else(|| codec_error("missing esds"))?; + let esds_start = esds_type - 4; + let esds_end = esds_start + read_be_u32(&m4a, esds_start)? as usize; + let descriptor_start = esds_type + 8; + let descriptors = m4a + .get(descriptor_start..esds_end) + .ok_or_else(|| codec_error("truncated esds"))?; + let sl_config = descriptors + .windows(2) + .position(|window| window == [0x06, 0x01]) + .ok_or_else(|| codec_error("missing SLConfig descriptor"))?; + assert_eq!(&descriptors[sl_config..sl_config + 3], &[0x06, 0x01, 0x02]); + assert!( + probe(&m4a).is_ok(), + "repaired M4A remains independently parseable" + ); + Ok(()) +} + +#[test] +fn normalized_waveform_peak_uses_the_full_byte_range() { + assert_eq!(normalized_peak_byte(0.0), 0); + assert_eq!(normalized_peak_byte(0.5), 128); + assert_eq!(normalized_peak_byte(1.0), 255); + assert_eq!(normalized_peak_byte(2.0), 255); +} + +#[test] +fn decoded_duration_limit_scales_with_input_sample_rate() { + assert!(within_duration(8_000 * 300, 8_000)); + assert!(!within_duration(8_000 * 300 + 1, 8_000)); +} + +#[test] +fn aac_frame_padding_is_bounded_and_only_applies_to_the_decode_stage() { + let five_minutes = 48_000usize * 300; + assert!(within_duration_with_padding( + five_minutes + AAC_FRAME_SAMPLES, + 48_000, + AAC_FRAME_SAMPLES + )); + assert!(!within_duration_with_padding( + five_minutes + AAC_FRAME_SAMPLES + 1, + 48_000, + AAC_FRAME_SAMPLES + )); + assert!(!within_duration(five_minutes + 1, 48_000)); +} + +#[test] +fn five_minute_m4a_trims_real_aac_frame_padding_and_rejects_longer_duration() -> Result<()> { + let exact = five_minute_silent_aac_fixture(48_000 * 300)?; + assert_eq!(probe(&exact)?.duration_ms, 300_000); + + let over = five_minute_silent_aac_fixture(48_000 * 300 + 1)?; + assert!(probe(&over).is_err()); + Ok(()) +} + +#[test] +fn m4a_duration_uses_the_selected_symphonia_track_id() -> Result<()> { + let bytes = multi_track_aac_fixture()?; + let mut hint = Hint::new(); + hint.with_extension("m4a"); + let source = MediaSourceStream::new( + Box::new(Cursor::new(bytes.clone())), + symphonia::core::io::MediaSourceStreamOptions::default(), + ); + let format = symphonia::default::get_probe() + .probe( + &hint, + source, + FormatOptions::default(), + MetadataOptions::default(), + ) + .map_err(|error| codec_error(format!("test Symphonia probe failed: {error}")))?; + let selected_track_id = format + .default_track(symphonia::core::formats::TrackType::Audio) + .ok_or_else(|| codec_error("test M4A has no default audio track"))? + .id; + let mp4 = mp4::Mp4Reader::read_header(Cursor::new(bytes.clone()), bytes.len() as u64) + .map_err(|error| codec_error(format!("test MP4 parse failed: {error}")))?; + let selected = mp4 + .tracks() + .get(&selected_track_id) + .ok_or_else(|| codec_error("Symphonia ID does not match MP4 track ID"))?; + let other = mp4 + .tracks() + .values() + .find(|track| track.track_id() != selected_track_id) + .ok_or_else(|| codec_error("test M4A is missing its second audio track"))?; + assert_ne!(selected.duration(), other.duration()); + + let (_, samples, rate, _) = decode_symphonia(&bytes, AudioCodec::M4aAac, "m4a") + .map_err(|error| codec_error(format!("test M4A decode failed: {error}")))?; + let expected = m4a_playable_samples(&mp4, selected_track_id, rate, usize::MAX)?; + let other_expected = m4a_playable_samples(&mp4, other.track_id(), rate, usize::MAX)?; + assert_ne!(expected, other_expected); + assert_eq!(samples.len(), expected); + let media_duration = selected.trak.mdia.mdhd.duration; + let media_timescale = selected.trak.mdia.mdhd.timescale; + assert_eq!( + u128::try_from(expected).map_err(codec_error_from)?, + u128::from(media_duration) * u128::from(rate) / u128::from(media_timescale) + ); + Ok(()) +} + +#[test] +fn invalid_wav_sample_rates_and_channel_counts_are_rejected() -> Result<()> { + let mut low_rate = audio::f32_mono_to_wav(&[0.2; 960], VOICE_RATE)?; + low_rate[24..28].copy_from_slice(&1_000u32.to_le_bytes()); + low_rate[28..32].copy_from_slice(&2_000u32.to_le_bytes()); + assert!(probe(&low_rate).is_err()); + + let mut many_channels = audio::f32_mono_to_wav(&[0.2; 9], VOICE_RATE)?; + many_channels[22..24].copy_from_slice(&9u16.to_le_bytes()); + many_channels[28..32].copy_from_slice(&(48_000u32 * 9 * 2).to_le_bytes()); + many_channels[32..34].copy_from_slice(&(9u16 * 2).to_le_bytes()); + assert!(probe(&many_channels).is_err()); + Ok(()) +} + +#[test] +fn symphonia_rejects_the_dedicated_ogg_codec() -> Result<()> { + let wav = audio::f32_mono_to_wav(&[0.2; 960], VOICE_RATE)?; + assert!(decode_symphonia(&wav, AudioCodec::OggOpus, "wav").is_err()); + assert!(decode_symphonia(&wav, AudioCodec::M4aAac, "wav").is_err()); + Ok(()) +} + +#[test] +fn transcode_rejects_a_mismatched_source_declaration() { + assert!(transcode(b"not a wav", AudioCodec::WavPcm16, AudioCodec::OggOpus).is_err()); +} + +#[test] +fn empty_silent_oversized_and_unsupported_codec_inputs_are_rejected() -> Result<()> { + assert!(probe(b"").is_err()); + assert!(waveform(b"", 1).is_err()); + assert!(transcode(b"", AudioCodec::WavPcm16, AudioCodec::OggOpus).is_err()); + let oversized = vec![0; MAX_INPUT_BYTES + 1]; + assert!(probe(&oversized).is_err()); + assert!(transcode(&oversized, AudioCodec::WavPcm16, AudioCodec::OggOpus).is_err()); + + let empty_wav = audio::f32_mono_to_wav(&[], VOICE_RATE)?; + assert!(waveform(&empty_wav, 1).is_err()); + + let silence = audio::f32_mono_to_wav(&vec![0.0; 960], VOICE_RATE)?; + assert_eq!(waveform(&silence, 32)?, vec![0; 32]); + assert!(transcode(&silence, AudioCodec::WavPcm16, AudioCodec::Mp3).is_err()); + assert!(encode_opus(&[]).is_err()); + assert!(encode_aac_m4a(&[]).is_err()); + Ok(()) +} + +#[test] +fn adts_splitter_rejects_bad_and_truncated_frames() { + assert!(split_adts(b"not ADTS").is_err()); + assert!(split_adts(&[0xff, 0xf1, 0, 0, 0, 0]).is_err()); + assert!(split_adts(&[0xff, 0xf1, 0, 0, 0, 0, 0]).is_err()); +} + +#[test] +fn waveform_bucket_indices_use_wide_math_for_32_bit_audio_limits() -> Result<()> { + let clip_samples = 23 * 48_000; + let buckets = 4096; + let (start, end) = waveform_bucket_range(clip_samples, buckets, buckets - 1)?; + assert_eq!(start, (buckets - 1) * clip_samples / buckets); + assert_eq!(end, clip_samples); + + let sample_count = usize::try_from(u32::MAX).map_err(codec_error_from)?; + let index = buckets - 1; + assert!( + u32::try_from(sample_count) + .map_err(codec_error_from)? + .checked_mul(u32::try_from(index).map_err(codec_error_from)?) + .is_none() + ); + + let expected_start = usize::try_from( + (u128::try_from(index).map_err(codec_error_from)? + * u128::try_from(sample_count).map_err(codec_error_from)?) + / u128::try_from(buckets).map_err(codec_error_from)?, + ) + .map_err(codec_error_from)?; + let expected_end = usize::try_from( + (u128::try_from(index + 1).map_err(codec_error_from)? + * u128::try_from(sample_count).map_err(codec_error_from)?) + / u128::try_from(buckets).map_err(codec_error_from)?, + ) + .map_err(codec_error_from)?; + assert_eq!( + waveform_bucket_range(sample_count, buckets, index)?, + (expected_start, expected_end) + ); + Ok(()) +} + +#[test] +fn an_empty_ogg_opus_stream_has_no_playable_audio() -> Result<()> { + let head = OpusHead { + channel_count: 1, + pre_skip: 0, + input_sample_rate: VOICE_RATE, + output_gain_q8: 0, + mapping_family: 0, + stream_count: 1, + coupled_count: 0, + channel_mapping: vec![0], + }; + let bytes = OggOpusWriter::new(Vec::new(), head) + .map_err(codec_error_from)? + .finish() + .map_err(codec_error_from)?; + assert!(decode_opus(&bytes).is_err()); + Ok(()) +} + +#[test] +fn same_format_aac_transcode_normalizes_stereo_to_mono_48khz() -> Result<()> { + let stereo = stereo_aac_fixture()?; + let input = probe(&stereo)?; + assert_eq!(input.codec, AudioCodec::M4aAac); + assert_eq!(input.channels, 2); + + let mono = transcode(&stereo, AudioCodec::M4aAac, AudioCodec::M4aAac)?; + let output = probe(&mono)?; + assert_eq!(output.channels, 1); + assert_eq!(output.sample_rate, 48_000); + Ok(()) +} + +fn stereo_aac_fixture() -> Result> { + let (bytes, duration_samples) = stereo_aac_muxed_fixture()?; + repair_m4a(bytes, duration_samples) +} + +fn stereo_aac_muxed_fixture() -> Result<(Vec, usize)> { + use bytes::Bytes; + use mp4::{ + AacConfig, AudioObjectType, ChannelConfig, FourCC, MediaConfig, Mp4Config, Mp4Sample, + Mp4Writer, SampleFreqIndex, TrackConfig, TrackType, + }; + use oxideav_aac::encoder::{EncoderConfig, StreamEncoder}; + + let pcm: Vec = (0..9_600).flat_map(|_| [6_000, 6_000]).collect(); + let mut encoder = StreamEncoder::new(EncoderConfig { + sample_rate: 48_000, + channels: 2, + bitrate: 128_000, + }) + .map_err(codec_error_from)?; + let adts = encoder.encode_all(&pcm).map_err(codec_error_from)?; + let frames = aac_frames(&adts)?; + let track = TrackConfig { + track_type: TrackType::Audio, + timescale: VOICE_RATE, + language: "und".into(), + media_conf: MediaConfig::AacConfig(AacConfig { + bitrate: 128_000, + profile: AudioObjectType::AacLowComplexity, + freq_index: SampleFreqIndex::Freq48000, + chan_conf: ChannelConfig::Stereo, + }), + }; + let config = Mp4Config { + major_brand: FourCC::from(*b"M4A "), + minor_version: 0, + compatible_brands: vec![FourCC::from(*b"M4A "), FourCC::from(*b"isom")], + timescale: VOICE_RATE, + }; + let mut writer = + Mp4Writer::write_start(Cursor::new(Vec::new()), &config).map_err(codec_error_from)?; + writer.add_track(&track).map_err(codec_error_from)?; + let mut start_time = 0u64; + for frame in frames.iter().skip(1) { + writer + .write_sample( + 1, + &Mp4Sample { + start_time, + duration: 1024, + rendering_offset: 0, + is_sync: true, + bytes: Bytes::copy_from_slice(frame), + }, + ) + .map_err(codec_error_from)?; + start_time += 1024; + } + writer.write_end().map_err(codec_error_from)?; + Ok((writer.into_writer().into_inner(), 9_600)) +} + +fn five_minute_silent_aac_fixture(duration_samples: usize) -> Result> { + use bytes::Bytes; + use mp4::{ + AacConfig, AudioObjectType, ChannelConfig, FourCC, MediaConfig, Mp4Config, Mp4Sample, + Mp4Writer, SampleFreqIndex, TrackConfig, TrackType, + }; + use oxideav_aac::encoder::{EncoderConfig, StreamEncoder}; + + let mut encoder = StreamEncoder::new(EncoderConfig { + sample_rate: 48_000, + channels: 1, + bitrate: 64_000, + }) + .map_err(codec_error_from)?; + let adts = encoder + .encode_all(&vec![0; AAC_FRAME_SAMPLES * 2]) + .map_err(codec_error_from)?; + let frame = *aac_frames(&adts)? + .get(1) + .ok_or_else(|| codec_error("test AAC encoder returned too few frames"))?; + let track = TrackConfig { + track_type: TrackType::Audio, + timescale: VOICE_RATE, + language: "und".into(), + media_conf: MediaConfig::AacConfig(AacConfig { + bitrate: 64_000, + profile: AudioObjectType::AacLowComplexity, + freq_index: SampleFreqIndex::Freq48000, + chan_conf: ChannelConfig::Mono, + }), + }; + let config = Mp4Config { + major_brand: FourCC::from(*b"M4A "), + minor_version: 0, + compatible_brands: vec![FourCC::from(*b"M4A "), FourCC::from(*b"isom")], + timescale: VOICE_RATE, + }; + let mut writer = + Mp4Writer::write_start(Cursor::new(Vec::new()), &config).map_err(codec_error_from)?; + writer.add_track(&track).map_err(codec_error_from)?; + let full_frames = duration_samples / AAC_FRAME_SAMPLES; + let final_duration = duration_samples % AAC_FRAME_SAMPLES; + let sample_count = full_frames + usize::from(final_duration > 0); + for index in 0..sample_count { + let start_time = u64::try_from(index * AAC_FRAME_SAMPLES).map_err(codec_error_from)?; + let duration = if index == full_frames && final_duration > 0 { + final_duration + } else { + AAC_FRAME_SAMPLES + }; + writer + .write_sample( + 1, + &Mp4Sample { + start_time, + duration: u32::try_from(duration).map_err(codec_error_from)?, + rendering_offset: 0, + is_sync: true, + bytes: Bytes::copy_from_slice(frame), + }, + ) + .map_err(codec_error_from)?; + } + writer.write_end().map_err(codec_error_from)?; + repair_m4a(writer.into_writer().into_inner(), duration_samples) +} + +fn multi_track_aac_fixture() -> Result> { + use bytes::Bytes; + use mp4::{ + AacConfig, AudioObjectType, ChannelConfig, FourCC, MediaConfig, Mp4Config, Mp4Sample, + Mp4Writer, SampleFreqIndex, TrackConfig, TrackType, + }; + use oxideav_aac::encoder::{EncoderConfig, StreamEncoder}; + + let mut encoder = StreamEncoder::new(EncoderConfig { + sample_rate: 48_000, + channels: 2, + bitrate: 128_000, + }) + .map_err(codec_error_from)?; + let adts = encoder + .encode_all(&vec![4_000; AAC_FRAME_SAMPLES * 12 * 2]) + .map_err(codec_error_from)?; + let frames = aac_frames(&adts)?; + let track = TrackConfig { + track_type: TrackType::Audio, + timescale: VOICE_RATE, + language: "und".into(), + media_conf: MediaConfig::AacConfig(AacConfig { + bitrate: 128_000, + profile: AudioObjectType::AacLowComplexity, + freq_index: SampleFreqIndex::Freq48000, + chan_conf: ChannelConfig::Stereo, + }), + }; + let config = Mp4Config { + major_brand: FourCC::from(*b"M4A "), + minor_version: 0, + compatible_brands: vec![FourCC::from(*b"M4A "), FourCC::from(*b"isom")], + timescale: VOICE_RATE, + }; + let mut writer = + Mp4Writer::write_start(Cursor::new(Vec::new()), &config).map_err(codec_error_from)?; + writer.add_track(&track).map_err(codec_error_from)?; + writer.add_track(&track).map_err(codec_error_from)?; + for (track_id, frame_count) in [(1, 4), (2, 9)] { + let mut start_time = 0u64; + for frame in frames.iter().skip(1).take(frame_count) { + writer + .write_sample( + track_id, + &Mp4Sample { + start_time, + duration: u32::try_from(AAC_FRAME_SAMPLES).map_err(codec_error_from)?, + rendering_offset: 0, + is_sync: true, + bytes: Bytes::copy_from_slice(frame), + }, + ) + .map_err(codec_error_from)?; + start_time += AAC_FRAME_SAMPLES as u64; + } + } + writer.write_end().map_err(codec_error_from)?; + let mut bytes = writer.into_writer().into_inner(); + repair_test_sl_config_descriptors(&mut bytes)?; + Ok(bytes) +} + +fn repair_test_sl_config_descriptors(bytes: &mut [u8]) -> Result<()> { + let esds_types: Vec = bytes + .windows(4) + .enumerate() + .filter_map(|(index, bytes)| (bytes == b"esds").then_some(index)) + .collect(); + for esds_type in esds_types { + let atom_start = esds_type + .checked_sub(4) + .ok_or_else(|| codec_error("invalid test esds atom"))?; + let atom_size = read_be_u32(bytes, atom_start)? as usize; + let atom_end = atom_start + .checked_add(atom_size) + .ok_or_else(|| codec_error("test esds size overflow"))?; + let descriptor_start = esds_type + 8; + let descriptors = bytes + .get(descriptor_start..atom_end) + .ok_or_else(|| codec_error("truncated test esds"))?; + let sl_config = descriptors + .windows(2) + .position(|window| window == [0x06, 0x00]) + .ok_or_else(|| codec_error("missing test SLConfig descriptor"))?; + bytes[descriptor_start + sl_config + 1] = 1; + bytes[descriptor_start + sl_config + 2] = 2; + } + Ok(()) +} + +fn aac_frames(adts: &[u8]) -> Result> { + let mut frames = Vec::new(); + let mut position = 0; + while position < adts.len() { + if adts.len() - position < 7 || adts[position] != 0xff { + return Err(codec_error("invalid test AAC framing")); + } + let header = if adts[position + 1] & 1 == 0 { 9 } else { 7 }; + let length = (usize::from(adts[position + 3] & 3) << 11) + | (usize::from(adts[position + 4]) << 3) + | usize::from(adts[position + 5] >> 5); + if length < header || position + length > adts.len() { + return Err(codec_error("truncated test AAC frame")); + } + frames.push(&adts[position + header..position + length]); + position += length; + } + Ok(frames) +} diff --git a/crates/tinyvoice/src/codec/mod.rs b/crates/tinyvoice/src/codec/mod.rs new file mode 100644 index 0000000..7cf17f9 --- /dev/null +++ b/crates/tinyvoice/src/codec/mod.rs @@ -0,0 +1,731 @@ +//! In-memory voice-note decoding, encoding, probing, and waveform extraction. +//! +//! The public functions accept byte slices and return owned values. They do no +//! filesystem, device, network, process, or runtime work. Audio is normalized +//! to mono 48 kHz for encoded voice-note output. + +use std::io::Cursor; + +use opus_pure::{ + Application, MAX_PACKET_BYTES, MAX_PACKET_SAMPLES, OggOpusReader, OggOpusWriter, OpusEncoder, + OpusHead, Trim, +}; +use symphonia::core::{ + codecs::audio::AudioDecoder, + formats::{FormatOptions, FormatReader, probe::Hint}, + io::MediaSourceStream, + meta::MetadataOptions, +}; + +use crate::{Error, Result, audio}; + +/// The supported voice-note container and codec combinations. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum AudioCodec { + /// PCM signed 16-bit little-endian WAV. + WavPcm16, + /// Opus audio in an Ogg container. + OggOpus, + /// AAC-LC audio in an M4A/ISO-BMFF container. + M4aAac, + /// MPEG Layer III audio (typically returned by text-to-speech services). + Mp3, +} + +/// Container facts obtained by [`probe`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AudioInfo { + /// Detected container and codec. + pub codec: AudioCodec, + /// Decoded duration, rounded down to whole milliseconds. + pub duration_ms: u64, + /// Decoded audio sample rate in Hz. + pub sample_rate: u32, + /// Decoded channel count. + pub channels: u16, +} + +const MAX_INPUT_BYTES: usize = 32 * 1024 * 1024; +const MAX_DECODED_SAMPLES: usize = 32_000_000; +const MAX_DURATION_SECONDS: u64 = 300; +const VOICE_RATE: u32 = 48_000; +const OPUS_FRAME: usize = 960; +const AAC_FRAME_SAMPLES: usize = 1024; +const AAC_BITRATE: u32 = 64_000; + +/// Detect a supported input, validate and decode it, then report its facts. +/// +/// # Errors +/// +/// Returns [`Error::AudioCodec`] for unsupported, corrupt, truncated, or +/// over-limit media. +pub fn probe(bytes: &[u8]) -> Result { + let (codec, samples, rate, channels) = decode(bytes)?; + let duration_ms = (samples.len() as u64).saturating_mul(1000) / u64::from(rate); + Ok(AudioInfo { + codec, + duration_ms, + sample_rate: rate, + channels, + }) +} + +/// Convert bytes in `from` format to the requested voice-note format. +/// +/// The declared source format is checked against the encoded input. Encoded +/// output is mono 48 kHz; WAV-to-WAV requests preserve original bytes. MP3 is +/// accepted as a source format but is not an encoding target. +/// +/// # Errors +/// +/// Returns [`Error::AudioCodec`] when input or output audio cannot be handled. +pub fn transcode(bytes: &[u8], from: AudioCodec, target: AudioCodec) -> Result> { + let (source, samples, rate, _) = decode_as(bytes, from)?; + if source == target && source == AudioCodec::WavPcm16 { + return Ok(bytes.to_vec()); + } + let samples = if rate == VOICE_RATE { + samples + } else { + audio::resample(&samples, rate, VOICE_RATE)? + }; + match target { + AudioCodec::WavPcm16 => audio::f32_mono_to_wav(&samples, VOICE_RATE), + AudioCodec::OggOpus => encode_opus(&samples), + AudioCodec::M4aAac => encode_aac_m4a(&samples), + AudioCodec::Mp3 => Err(codec_error("MP3 encoding is not supported")), + } +} + +/// Return normalized peak-amplitude bytes for a decoded waveform. +/// +/// The output has exactly `buckets` entries. Each entry is a peak amplitude +/// normalized against the clip's loudest bucket, matching the compact byte +/// shape used by voice-note clients. +/// +/// # Errors +/// +/// Returns [`Error::AudioCodec`] for invalid audio or when `buckets` is zero or +/// exceeds 4096. +pub fn waveform(bytes: &[u8], buckets: usize) -> Result> { + if buckets == 0 || buckets > 4096 { + return Err(codec_error( + "waveform bucket count must be between 1 and 4096", + )); + } + let (_, samples, rate, _) = decode(bytes)?; + let samples = if rate == VOICE_RATE { + samples + } else { + audio::resample(&samples, rate, VOICE_RATE)? + }; + if samples.is_empty() { + return Err(codec_error("audio contains no samples")); + } + let mut peaks = Vec::with_capacity(buckets); + for bucket in 0..buckets { + let (start, end) = waveform_bucket_range(samples.len(), buckets, bucket)?; + let peak = samples[start..end] + .iter() + .fold(0.0f32, |largest, sample| largest.max(sample.abs())); + peaks.push(peak); + } + let max_peak = peaks.iter().copied().fold(0.0f32, f32::max); + if max_peak == 0.0 { + return Ok(vec![0; buckets]); + } + Ok(peaks + .into_iter() + .map(|peak| normalized_peak_byte(peak / max_peak)) + .collect()) +} + +fn waveform_bucket_range( + sample_count: usize, + bucket_count: usize, + index: usize, +) -> Result<(usize, usize)> { + let sample_count_u128 = u128::try_from(sample_count).map_err(codec_error_from)?; + let bucket_count_u128 = u128::try_from(bucket_count).map_err(codec_error_from)?; + let index_u128 = u128::try_from(index).map_err(codec_error_from)?; + let start = usize::try_from(index_u128 * sample_count_u128 / bucket_count_u128) + .map_err(codec_error_from)?; + let end = usize::try_from((index_u128 + 1) * sample_count_u128 / bucket_count_u128) + .map_err(codec_error_from)?; + Ok((start, end.max(start.saturating_add(1)).min(sample_count))) +} + +/// Convert a known finite amplitude ratio in `[0, 1]` to its byte scale. +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn normalized_peak_byte(ratio: f32) -> u8 { + (ratio.clamp(0.0, 1.0) * 255.0).round() as u8 +} + +fn decode(bytes: &[u8]) -> Result<(AudioCodec, Vec, u32, u16)> { + if bytes.is_empty() || bytes.len() > MAX_INPUT_BYTES { + return Err(codec_error( + "audio input is empty or exceeds the 32 MiB limit", + )); + } + if bytes.starts_with(b"OggS") { + decode_opus(bytes) + .map(|(samples, rate, channels)| (AudioCodec::OggOpus, samples, rate, channels)) + } else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WAVE") { + decode_symphonia(bytes, AudioCodec::WavPcm16, "wav") + } else if bytes.get(4..8) == Some(b"ftyp") { + decode_symphonia(bytes, AudioCodec::M4aAac, "m4a") + } else if looks_like_mp3(bytes) { + decode_symphonia(bytes, AudioCodec::Mp3, "mp3") + } else { + Err(codec_error("unsupported audio container")) + } +} + +fn decode_as(bytes: &[u8], expected: AudioCodec) -> Result<(AudioCodec, Vec, u32, u16)> { + if bytes.is_empty() || bytes.len() > MAX_INPUT_BYTES { + return Err(codec_error( + "audio input is empty or exceeds the 32 MiB limit", + )); + } + let decoded = match expected { + AudioCodec::OggOpus if bytes.starts_with(b"OggS") => decode_opus(bytes) + .map(|(samples, rate, channels)| (AudioCodec::OggOpus, samples, rate, channels)), + AudioCodec::WavPcm16 if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WAVE") => { + decode_symphonia(bytes, AudioCodec::WavPcm16, "wav") + } + AudioCodec::M4aAac if bytes.get(4..8) == Some(b"ftyp") => { + decode_symphonia(bytes, AudioCodec::M4aAac, "m4a") + } + AudioCodec::Mp3 if looks_like_mp3(bytes) => decode_symphonia(bytes, AudioCodec::Mp3, "mp3"), + _ => { + return Err(codec_error( + "audio input does not match the declared source codec", + )); + } + }?; + Ok(decoded) +} + +fn looks_like_mp3(bytes: &[u8]) -> bool { + bytes.starts_with(b"ID3") + || bytes + .get(..2) + .is_some_and(|header| header[0] == 0xff && header[1] & 0xe0 == 0xe0) +} + +fn decode_opus(bytes: &[u8]) -> Result<(Vec, u32, u16)> { + let mut reader = OggOpusReader::new(Cursor::new(bytes.to_vec())).map_err(codec_error_from)?; + let head = reader.head().clone(); + let channels = usize::from(head.channel_count); + if channels == 0 || channels > 2 { + return Err(codec_error("voice Opus supports one or two channels")); + } + let mut decoder = head.decoder(48_000).map_err(codec_error_from)?; + let mut trim = Trim::new(&head, 48_000, channels).map_err(codec_error_from)?; + let mut pcm = vec![0.0; MAX_PACKET_SAMPLES * channels]; + let mut mono = Vec::new(); + let mut saw_packet = false; + let mut saw_eos = false; + while let Some(packet) = reader.read_packet().map_err(codec_error_from)? { + saw_packet = true; + saw_eos |= packet.end_of_stream; + let count = decoder + .decode(&packet.data, MAX_PACKET_SAMPLES, &mut pcm) + .map_err(codec_error_from)?; + let trimmed = trim.keep(&packet, &pcm[..count * channels]); + for frame in trimmed.chunks_exact(channels) { + mono.push(frame.iter().sum::() / f32::from(head.channel_count)); + } + if !within_duration(mono.len(), VOICE_RATE) { + return Err(codec_error("decoded audio exceeds the five minute limit")); + } + } + if !saw_packet || !saw_eos || mono.is_empty() { + return Err(codec_error( + "Ogg Opus stream is truncated or contains no audio", + )); + } + Ok(( + mono, + VOICE_RATE, + u16::try_from(channels).map_err(codec_error_from)?, + )) +} + +fn decode_symphonia( + bytes: &[u8], + codec: AudioCodec, + extension: &str, +) -> Result<(AudioCodec, Vec, u32, u16)> { + let mut hint = Hint::new(); + hint.with_extension(extension); + let source = MediaSourceStream::new( + Box::new(Cursor::new(bytes.to_vec())), + symphonia::core::io::MediaSourceStreamOptions::default(), + ); + let mut format = symphonia::default::get_probe() + .probe( + &hint, + source, + FormatOptions::default(), + MetadataOptions::default(), + ) + .map_err(|error| codec_error(format!("audio container probe failed: {error}")))?; + let track = format + .default_track(symphonia::core::formats::TrackType::Audio) + .ok_or_else(|| codec_error("audio stream has no default track"))?; + let track_id = track.id; + let codec_params = track + .codec_params + .as_ref() + .and_then(|params| params.audio()) + .ok_or_else(|| codec_error("audio codec parameters are missing"))? + .clone(); + let expected_codec = match codec { + AudioCodec::WavPcm16 => symphonia::core::codecs::audio::well_known::CODEC_ID_PCM_S16LE, + AudioCodec::M4aAac => symphonia::core::codecs::audio::well_known::CODEC_ID_AAC, + AudioCodec::Mp3 => symphonia::core::codecs::audio::well_known::CODEC_ID_MP3, + AudioCodec::OggOpus => return Err(codec_error("Ogg Opus uses its dedicated decoder")), + }; + if codec_params.codec != expected_codec { + return Err(codec_error( + "audio stream does not match its declared codec", + )); + } + let rate = codec_params + .sample_rate + .ok_or_else(|| codec_error("audio sample rate is missing"))?; + if !(8_000..=192_000).contains(&rate) { + return Err(codec_error( + "audio sample rate must be between 8 and 192 kHz", + )); + } + let mut decoder = symphonia::default::get_codecs() + .make_audio_decoder( + &codec_params, + &symphonia::core::codecs::audio::AudioDecoderOptions::default(), + ) + .map_err(codec_error_from)?; + let decoder_padding = if codec == AudioCodec::M4aAac { + AAC_FRAME_SAMPLES + } else { + 0 + }; + let (mut samples, channel_count) = + decode_packets(&mut *format, track_id, &mut *decoder, rate, decoder_padding)?; + if samples.is_empty() { + return Err(codec_error("audio contains no samples")); + } + if codec == AudioCodec::M4aAac { + let mp4 = mp4::Mp4Reader::read_header(Cursor::new(bytes.to_vec()), bytes.len() as u64) + .map_err(codec_error_from)?; + let playable_samples = m4a_playable_samples(&mp4, track_id, rate, samples.len())?; + if !within_duration(playable_samples, rate) || playable_samples > MAX_DECODED_SAMPLES { + return Err(codec_error("decoded audio exceeds the five minute limit")); + } + samples.truncate(playable_samples); + if samples.is_empty() { + return Err(codec_error("M4A audio duration is empty")); + } + } + Ok(( + codec, + samples, + rate, + u16::try_from(channel_count).map_err(codec_error_from)?, + )) +} + +fn m4a_playable_samples( + mp4: &mp4::Mp4Reader>>, + track_id: u32, + rate: u32, + decoded_samples: usize, +) -> Result { + let audio_track = mp4 + .tracks() + .get(&track_id) + .filter(|track| { + track + .track_type() + .is_ok_and(|kind| kind == mp4::TrackType::Audio) + }) + .ok_or_else(|| codec_error("decoded M4A audio track metadata is missing"))?; + let edit_list = audio_track + .trak + .edts + .as_ref() + .and_then(|edts| edts.elst.as_ref()); + let (duration_units, duration_timescale) = if let Some(elst) = edit_list { + let duration_units = elst + .entries + .iter() + .try_fold(0u64, |total, entry| { + total.checked_add(entry.segment_duration) + }) + .ok_or_else(|| codec_error("M4A edit list duration overflows"))?; + (duration_units, mp4.timescale()) + } else { + ( + audio_track.trak.mdia.mdhd.duration, + audio_track.trak.mdia.mdhd.timescale, + ) + }; + if duration_timescale == 0 + || u128::from(duration_units) + > u128::from(MAX_DURATION_SECONDS) * u128::from(duration_timescale) + { + return Err(codec_error("M4A duration exceeds the five minute limit")); + } + Ok( + (u128::from(duration_units) * u128::from(rate) / u128::from(duration_timescale)) + .min(decoded_samples as u128) as usize, + ) +} + +fn decode_packets( + format: &mut dyn FormatReader, + track_id: u32, + decoder: &mut dyn AudioDecoder, + rate: u32, + decoder_padding: usize, +) -> Result<(Vec, usize)> { + let mut samples = Vec::new(); + let mut channel_count = 0usize; + loop { + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + Err(error) => { + return Err(codec_error(format!( + "audio container packet read failed: {error}" + ))); + } + }; + if packet.track_id != track_id { + continue; + } + let audio_buffer = decoder + .decode(&packet) + .map_err(|error| codec_error(format!("audio packet decode failed: {error}")))?; + let channels = audio_buffer.spec().channels().count(); + if channels == 0 || channels > 8 { + return Err(codec_error("audio channel count is unsupported")); + } + channel_count = channels; + let mut interleaved = vec![0.0f32; audio_buffer.samples_interleaved()]; + audio_buffer.copy_to_slice_interleaved::(&mut interleaved); + for frame in interleaved.chunks_exact(channels) { + samples.push( + frame.iter().sum::() + / f32::from(u8::try_from(channels).map_err(codec_error_from)?), + ); + } + if !within_duration_with_padding(samples.len(), rate, decoder_padding) + || samples.len() > MAX_DECODED_SAMPLES + { + return Err(codec_error("decoded audio exceeds the five minute limit")); + } + } + Ok((samples, channel_count)) +} + +fn encode_opus(samples: &[f32]) -> Result> { + if samples.is_empty() { + return Err(codec_error("audio contains no samples")); + } + let mut encoder = OpusEncoder::new(48_000, 1, Application::Voip).map_err(codec_error_from)?; + encoder.bitrate_bps = 24_000; + let head = OpusHead::for_encoder(&encoder, VOICE_RATE); + let mut writer = OggOpusWriter::new(Vec::new(), head.clone()).map_err(codec_error_from)?; + let needed_granule = u64::from(head.pre_skip) + samples.len() as u64; + let packet_count = usize::try_from(needed_granule.div_ceil(960)).map_err(codec_error_from)?; + let mut packet = vec![0; MAX_PACKET_BYTES]; + for index in 0..packet_count { + let start = index * OPUS_FRAME; + let end = (start + OPUS_FRAME).min(samples.len()); + let mut frame = [0.0f32; OPUS_FRAME]; + if start < samples.len() { + frame[..end - start].copy_from_slice(&samples[start..end]); + } + let count = encoder + .encode(&frame, OPUS_FRAME, &mut packet) + .map_err(codec_error_from)?; + let duration = if index + 1 == packet_count { + u32::try_from(needed_granule.saturating_sub(writer.granule().cast_unsigned())) + .map_err(codec_error_from)? + } else { + 960 + }; + writer + .write_packet_with_duration(&packet[..count], duration) + .map_err(codec_error_from)?; + } + writer.finish().map_err(codec_error_from) +} + +fn encode_aac_m4a(samples: &[f32]) -> Result> { + use bytes::Bytes; + use mp4::{ + AacConfig, AudioObjectType, ChannelConfig, FourCC, MediaConfig, Mp4Config, Mp4Sample, + Mp4Writer, SampleFreqIndex, TrackConfig, TrackType, + }; + use oxideav_aac::encoder::{EncoderConfig, StreamEncoder}; + + if samples.is_empty() { + return Err(codec_error("audio contains no samples")); + } + let mut encoder = StreamEncoder::new(EncoderConfig { + sample_rate: VOICE_RATE, + channels: 1, + bitrate: AAC_BITRATE, + }) + .map_err(codec_error_from)?; + let pcm: Vec = samples.iter().map(|sample| pcm16_sample(*sample)).collect(); + let adts = encoder.encode_all(&pcm).map_err(codec_error_from)?; + let frames = split_adts(&adts)?; + if frames.len() < 2 { + return Err(codec_error("AAC encoder returned too few frames")); + } + let config = AacConfig { + bitrate: AAC_BITRATE, + profile: AudioObjectType::AacLowComplexity, + freq_index: SampleFreqIndex::Freq48000, + chan_conf: ChannelConfig::Mono, + }; + let track = TrackConfig { + track_type: TrackType::Audio, + timescale: VOICE_RATE, + language: "und".into(), + media_conf: MediaConfig::AacConfig(config), + }; + let mp4_config = Mp4Config { + major_brand: FourCC::from(*b"M4A "), + minor_version: 0, + compatible_brands: vec![FourCC::from(*b"M4A "), FourCC::from(*b"isom")], + timescale: VOICE_RATE, + }; + let mut writer = + Mp4Writer::write_start(Cursor::new(Vec::new()), &mp4_config).map_err(codec_error_from)?; + writer.add_track(&track).map_err(codec_error_from)?; + let usable = &frames[1..]; + let mut start_time = 0u64; + for frame in usable { + let duration = 1024; + writer + .write_sample( + 1, + &Mp4Sample { + start_time, + duration, + rendering_offset: 0, + is_sync: true, + bytes: Bytes::copy_from_slice(frame), + }, + ) + .map_err(codec_error_from)?; + start_time += u64::from(duration); + } + writer.write_end().map_err(codec_error_from)?; + repair_m4a(writer.into_writer().into_inner(), samples.len()) +} + +/// Repair the MP4 muxer's empty `SLConfig` descriptor and add a gapless edit. +/// +/// `mp4` 0.14 writes an empty-valued `SLConfig` descriptor placeholder, while +/// ISO/IEC 14496-1 requires its one-byte predefined value. Fill that existing +/// byte without changing atom sizes. The edit list clips AAC's 1024-sample +/// frame padding to the original PCM duration. +#[allow(clippy::too_many_lines)] +fn repair_m4a(mut bytes: Vec, duration_samples: usize) -> Result> { + let esds_type = find_last(&bytes, b"esds") + .ok_or_else(|| codec_error("M4A muxer omitted the AAC decoder configuration"))?; + let esds_start = esds_type + .checked_sub(4) + .ok_or_else(|| codec_error("invalid M4A AAC decoder configuration"))?; + let descriptor_end = esds_start + .checked_add(read_be_u32(&bytes, esds_start)? as usize) + .ok_or_else(|| codec_error("M4A AAC decoder configuration size overflow"))?; + let descriptor_start = esds_type + .checked_add(8) + .ok_or_else(|| codec_error("M4A descriptor offset overflow"))?; + let descriptors = bytes + .get(descriptor_start..descriptor_end) + .ok_or_else(|| codec_error("truncated M4A AAC descriptors"))?; + let sl_config = descriptors + .windows(3) + .rposition(|window| window == [0x06, 0x00, 0x00]) + .and_then(|offset| descriptor_start.checked_add(offset)) + .ok_or_else(|| codec_error("M4A AAC decoder configuration has no SLConfig descriptor"))?; + *bytes + .get_mut(sl_config + 1) + .ok_or_else(|| codec_error("truncated M4A SLConfig descriptor"))? = 1; + *bytes + .get_mut(sl_config + 2) + .ok_or_else(|| codec_error("truncated M4A SLConfig predefined value"))? = 2; + + let mdia_type = + find_last(&bytes, b"mdia").ok_or_else(|| codec_error("M4A media atom is missing"))?; + let trak_type = find_last(&bytes[..mdia_type], b"trak") + .ok_or_else(|| codec_error("M4A track atom is missing"))?; + let movie_type = + find_last(&bytes, b"moov").ok_or_else(|| codec_error("M4A movie atom is missing"))?; + let edit = edit_list_atom(duration_samples)?; + let edit_len = u32::try_from(edit.len()).map_err(codec_error_from)?; + for atom_type in [b"trak".as_slice(), b"moov"] { + let position = if atom_type == b"trak" { + trak_type + } else { + movie_type + }; + let size_position = position + .checked_sub(4) + .ok_or_else(|| codec_error("invalid M4A atom position"))?; + let size = read_be_u32(&bytes, size_position)?; + let size_end = size_position + .checked_add(4) + .ok_or_else(|| codec_error("M4A atom position overflow"))?; + bytes + .get_mut(size_position..size_end) + .ok_or_else(|| codec_error("truncated M4A atom size"))? + .copy_from_slice( + &size + .checked_add(edit_len) + .ok_or_else(|| codec_error("M4A atom is too large"))? + .to_be_bytes(), + ); + } + let tkhd_type = find_last(&bytes[..mdia_type], b"tkhd") + .ok_or_else(|| codec_error("M4A track header is missing"))?; + let tkhd_duration = tkhd_type + .checked_add(24) + .ok_or_else(|| codec_error("M4A track header offset overflow"))?; + let tkhd_end = tkhd_duration + .checked_add(4) + .ok_or_else(|| codec_error("M4A track header offset overflow"))?; + bytes + .get_mut(tkhd_duration..tkhd_end) + .ok_or_else(|| codec_error("truncated M4A track header"))? + .copy_from_slice( + &u32::try_from(duration_samples) + .map_err(codec_error_from)? + .to_be_bytes(), + ); + let mvhd_type = find_last(&bytes[..mdia_type], b"mvhd") + .ok_or_else(|| codec_error("M4A movie header is missing"))?; + let mvhd_duration = mvhd_type + .checked_add(20) + .ok_or_else(|| codec_error("M4A movie header offset overflow"))?; + let mvhd_end = mvhd_duration + .checked_add(4) + .ok_or_else(|| codec_error("M4A movie header offset overflow"))?; + bytes + .get_mut(mvhd_duration..mvhd_end) + .ok_or_else(|| codec_error("truncated M4A movie header"))? + .copy_from_slice( + &u32::try_from(duration_samples) + .map_err(codec_error_from)? + .to_be_bytes(), + ); + let edit_position = mdia_type + .checked_sub(4) + .ok_or_else(|| codec_error("invalid M4A media atom position"))?; + bytes.splice(edit_position..edit_position, edit); + Ok(bytes) +} + +fn edit_list_atom(duration_samples: usize) -> Result> { + let duration = u32::try_from(duration_samples).map_err(codec_error_from)?; + let mut elst = Vec::with_capacity(28); + elst.extend_from_slice(&[0, 0, 0, 0]); // version and flags + elst.extend_from_slice(&1u32.to_be_bytes()); + elst.extend_from_slice(&duration.to_be_bytes()); + elst.extend_from_slice(&0i32.to_be_bytes()); // no leading media-time trim + elst.extend_from_slice(&1i16.to_be_bytes()); // playback rate 1.0 + elst.extend_from_slice(&0i16.to_be_bytes()); + let elst = atom(*b"elst", &elst)?; + atom(*b"edts", &elst) +} + +fn atom(kind: [u8; 4], payload: &[u8]) -> Result> { + let size = u32::try_from( + payload + .len() + .checked_add(8) + .ok_or_else(|| codec_error("M4A atom size overflow"))?, + ) + .map_err(codec_error_from)?; + let mut out = Vec::with_capacity(size as usize); + out.extend_from_slice(&size.to_be_bytes()); + out.extend_from_slice(&kind); + out.extend_from_slice(payload); + Ok(out) +} + +fn find_last(bytes: &[u8], needle: &[u8]) -> Option { + bytes + .windows(needle.len()) + .rposition(|window| window == needle) +} + +fn read_be_u32(bytes: &[u8], offset: usize) -> Result { + let end = offset + .checked_add(4) + .ok_or_else(|| codec_error("M4A atom offset overflow"))?; + let data = bytes + .get(offset..end) + .ok_or_else(|| codec_error("truncated M4A atom"))?; + Ok(u32::from_be_bytes([data[0], data[1], data[2], data[3]])) +} + +fn split_adts(bytes: &[u8]) -> Result> { + let mut frames = Vec::new(); + let mut position = 0; + while position < bytes.len() { + if bytes.len() - position < 7 + || bytes[position] != 0xff + || bytes[position + 1] & 0xf6 != 0xf0 + { + return Err(codec_error("AAC encoder returned invalid ADTS framing")); + } + let header = if bytes[position + 1] & 1 == 0 { 9 } else { 7 }; + let length = (usize::from(bytes[position + 3] & 3) << 11) + | (usize::from(bytes[position + 4]) << 3) + | usize::from(bytes[position + 5] >> 5); + if length < header || position + length > bytes.len() { + return Err(codec_error("AAC encoder returned a truncated ADTS frame")); + } + frames.push(&bytes[position + header..position + length]); + position += length; + } + Ok(frames) +} + +fn codec_error(message: impl Into) -> Error { + Error::AudioCodec(message.into()) +} +fn codec_error_from(error: impl std::fmt::Display) -> Error { + codec_error(error.to_string()) +} + +fn within_duration(samples: usize, sample_rate: u32) -> bool { + within_duration_with_padding(samples, sample_rate, 0) +} + +fn within_duration_with_padding(samples: usize, sample_rate: u32, padding_samples: usize) -> bool { + let maximum = u64::from(sample_rate) + .saturating_mul(MAX_DURATION_SECONDS) + .saturating_add(u64::try_from(padding_samples).unwrap_or(u64::MAX)); + u64::try_from(samples).is_ok_and(|count| count <= maximum) +} + +/// Convert clamped normalized PCM to the encoder's signed integer range. +#[allow(clippy::cast_possible_truncation)] +fn pcm16_sample(sample: f32) -> i16 { + (sample.clamp(-1.0, 1.0) * 32767.0).round() as i16 +} + +#[cfg(test)] +#[path = "codec_tests.rs"] +mod tests; diff --git a/crates/tinyvoice/src/error/mod.rs b/crates/tinyvoice/src/error/mod.rs index 85be001..14eb7a8 100644 --- a/crates/tinyvoice/src/error/mod.rs +++ b/crates/tinyvoice/src/error/mod.rs @@ -57,6 +57,10 @@ pub enum Error { /// A microphone capture operation failed. #[error("capture failed: {0}")] Capture(String), + + /// An encoded audio container or codec could not be read or written. + #[error("audio codec error: {0}")] + AudioCodec(String), } /// The result type returned by every fallible function in this crate. diff --git a/crates/tinyvoice/src/lib.rs b/crates/tinyvoice/src/lib.rs index 3759b41..b0d7218 100644 --- a/crates/tinyvoice/src/lib.rs +++ b/crates/tinyvoice/src/lib.rs @@ -59,6 +59,7 @@ mod error; pub mod audio; #[cfg(feature = "capture")] pub mod capture; +pub mod codec; #[cfg(feature = "hotkey")] pub mod hotkey; pub mod intent; diff --git a/crates/tinyvoice/tests/codecs.rs b/crates/tinyvoice/tests/codecs.rs new file mode 100644 index 0000000..9779d50 --- /dev/null +++ b/crates/tinyvoice/tests/codecs.rs @@ -0,0 +1,241 @@ +//! Public API coverage for voice-note codecs using a checked-in WAV fixture. + +use std::io::Cursor; +use tinyvoice::Result; +use tinyvoice::codec::{self, AudioCodec}; + +const VOICE_ENVELOPE_FIXTURE: &[u8] = include_bytes!("fixtures/voice-envelope.wav"); +const OPUS_INTEROP_FIXTURE: &[u8] = include_bytes!("fixtures/opus-interop.wav"); +const MP3_FIXTURE: &[u8] = include_bytes!("fixtures/mp3-test.bit"); + +fn window_rms(bytes: &[u8], sample_rate: u32, window_ms: usize) -> Vec { + let (sample_bytes, _trailing_byte) = bytes.as_chunks::<2>(); + let samples = sample_bytes + .iter() + .map(|sample| f64::from(i16::from_le_bytes(*sample)) / f64::from(i16::MAX)) + .collect::>(); + let window = sample_rate as usize * window_ms / 1_000; + samples + .chunks(window) + .map(|window| { + let (sum, count) = window.iter().fold((0.0, 0.0), |(sum, count), sample| { + (sum + sample * sample, count + 1.0) + }); + (sum / count).sqrt() + }) + .collect() +} + +fn assert_envelope_retained(reference: &[u8], reference_rate: u32, decoded: &[u8]) { + let expected = window_rms(reference, reference_rate, 40); + let actual = window_rms(decoded, 48_000, 40); + assert_eq!( + actual.len(), + expected.len(), + "decoded clip duration changed" + ); + let scale = expected.iter().sum::() / actual.iter().sum::(); + let normalized_error = expected + .iter() + .zip(actual.iter()) + .map(|(expected, actual)| (expected - actual * scale).abs()) + .sum::() + / expected.iter().sum::(); + assert!( + normalized_error < 0.22, + "voice envelope drifted: {normalized_error:.3}" + ); + assert!( + actual.iter().any(|rms| *rms > 0.05), + "decoded signal is silent" + ); +} + +#[test] +fn probe_waveform_and_passthrough_work_for_a_real_wav_fixture() -> Result<()> { + let info = codec::probe(VOICE_ENVELOPE_FIXTURE)?; + assert_eq!(info.codec, AudioCodec::WavPcm16); + assert_eq!(info.sample_rate, 44_100); + assert_eq!(info.channels, 1); + assert!(info.duration_ms > 2_300); + + let peaks = codec::waveform(VOICE_ENVELOPE_FIXTURE, 256)?; + assert_eq!(peaks.len(), 256); + assert!(peaks.iter().any(|peak| *peak > 0)); + assert_eq!( + codec::transcode( + VOICE_ENVELOPE_FIXTURE, + AudioCodec::WavPcm16, + AudioCodec::WavPcm16 + )?, + VOICE_ENVELOPE_FIXTURE + ); + Ok(()) +} + +#[test] +fn opus_and_aac_voice_notes_round_trip_through_independent_container_decoders() -> Result<()> { + let opus = codec::transcode( + OPUS_INTEROP_FIXTURE, + AudioCodec::WavPcm16, + AudioCodec::OggOpus, + )?; + assert!(opus.starts_with(b"OggS")); + let opus_info = codec::probe(&opus)?; + assert_eq!(opus_info.codec, AudioCodec::OggOpus); + assert_eq!(opus_info.sample_rate, 48_000); + assert_eq!(opus_info.channels, 1); + assert!(opus_info.duration_ms > 0); + + // Decode the emitted packets with a separately implemented Opus decoder; + // this guards against an encoder/decoder pair agreeing on invalid packets. + let mut packet_reader = opus_pure::OggOpusReader::new(Cursor::new(opus.clone())) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + let mut independent = opus_decoder::OpusDecoder::new(48_000, 1) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + let mut independent_samples = Vec::new(); + while let Some(packet) = packet_reader + .read_packet() + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))? + { + let mut frame = vec![0.0; opus_decoder::OpusDecoder::MAX_FRAME_SIZE_48K]; + let count = independent + .decode_float(&packet.data, &mut frame, false) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + independent_samples.extend_from_slice(&frame[..count]); + } + assert!( + independent_samples + .iter() + .any(|sample| sample.abs() > 0.001) + ); + + let m4a = codec::transcode(&opus, AudioCodec::OggOpus, AudioCodec::M4aAac)?; + assert!(m4a.windows(4).any(|window| window == b"ftyp")); + let aac_info = codec::probe(&m4a)?; + assert_eq!(aac_info.codec, AudioCodec::M4aAac); + assert_eq!(aac_info.sample_rate, 48_000); + assert_eq!(aac_info.channels, 1); + assert!(aac_info.duration_ms > 0); + assert!(aac_info.duration_ms.abs_diff(opus_info.duration_ms) <= 1); + assert!(codec::waveform(&m4a, 256)?.iter().any(|peak| *peak > 0)); + + let wav = codec::transcode(&m4a, AudioCodec::M4aAac, AudioCodec::WavPcm16)?; + assert!(wav.starts_with(b"RIFF")); + let wav_info = codec::probe(&wav)?; + assert_eq!(wav_info.codec, AudioCodec::WavPcm16); + assert_eq!(wav_info.sample_rate, 48_000); + assert!(wav_info.duration_ms.abs_diff(opus_info.duration_ms) <= 1); + Ok(()) +} + +#[test] +fn multiframe_voice_survives_opus_aac_round_trip_with_gapless_duration() -> Result<()> { + let original = codec::transcode( + VOICE_ENVELOPE_FIXTURE, + AudioCodec::WavPcm16, + AudioCodec::WavPcm16, + )?; + let original_info = codec::probe(&original)?; + assert!(original_info.duration_ms > 2_300); + + let opus = codec::transcode(&original, AudioCodec::WavPcm16, AudioCodec::OggOpus)?; + let aac = codec::transcode(&opus, AudioCodec::OggOpus, AudioCodec::M4aAac)?; + let wav = codec::transcode(&aac, AudioCodec::M4aAac, AudioCodec::WavPcm16)?; + let decoded_info = codec::probe(&wav)?; + assert_eq!(decoded_info.sample_rate, 48_000); + assert!(decoded_info.duration_ms.abs_diff(original_info.duration_ms) <= 1); + assert_envelope_retained(&original[44..], original_info.sample_rate, &wav[44..]); + Ok(()) +} + +#[test] +fn mp3_tts_source_decodes_and_transcodes_to_voice_formats() -> Result<()> { + let info = codec::probe(MP3_FIXTURE)?; + assert_eq!(info.codec, AudioCodec::Mp3); + assert!(info.duration_ms > 0); + + let opus = codec::transcode(MP3_FIXTURE, AudioCodec::Mp3, AudioCodec::OggOpus)?; + let opus_info = codec::probe(&opus)?; + assert_eq!(opus_info.channels, 1); + assert_eq!(opus_info.sample_rate, 48_000); + + let wav = codec::transcode(MP3_FIXTURE, AudioCodec::Mp3, AudioCodec::WavPcm16)?; + let wav_info = codec::probe(&wav)?; + assert_eq!(wav_info.channels, 1); + assert_eq!(wav_info.sample_rate, 48_000); + assert!(wav_info.duration_ms.abs_diff(info.duration_ms) <= 30); + let aac = codec::transcode(MP3_FIXTURE, AudioCodec::Mp3, AudioCodec::M4aAac)?; + let aac_info = codec::probe(&aac)?; + assert_eq!(aac_info.codec, AudioCodec::M4aAac); + assert_eq!(aac_info.sample_rate, 48_000); + assert!(aac_info.duration_ms.abs_diff(wav_info.duration_ms) <= 1); + let aac_wav = codec::transcode(&aac, AudioCodec::M4aAac, AudioCodec::WavPcm16)?; + assert_envelope_retained(&wav[44..], wav_info.sample_rate, &aac_wav[44..]); + assert!(codec::transcode(MP3_FIXTURE, AudioCodec::WavPcm16, AudioCodec::OggOpus).is_err()); + Ok(()) +} + +#[test] +fn same_format_opus_transcode_normalizes_stereo_to_mono_48khz() -> Result<()> { + let mut encoder = opus_pure::OpusEncoder::new(48_000, 2, opus_pure::Application::Voip) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + let head = opus_pure::OpusHead::for_encoder(&encoder, 48_000); + let mut writer = opus_pure::OggOpusWriter::new(Vec::new(), head.clone()) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + let pcm: Vec = (0..960).flat_map(|_| [0.1, 0.1]).collect(); + let mut packet = vec![0; opus_pure::MAX_PACKET_BYTES]; + let count = encoder + .encode(&pcm, 960, &mut packet) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + writer + .write_packet_with_duration(&packet[..count], 960) + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + let stereo = writer + .finish() + .map_err(|error| tinyvoice::Error::AudioCodec(error.to_string()))?; + + let mono = codec::transcode(&stereo, AudioCodec::OggOpus, AudioCodec::OggOpus)?; + let info = codec::probe(&mono)?; + assert_eq!(info.codec, AudioCodec::OggOpus); + assert_eq!(info.channels, 1); + assert_eq!(info.sample_rate, 48_000); + Ok(()) +} + +#[test] +fn malformed_and_truncated_audio_are_rejected() -> Result<()> { + assert!(codec::probe(b"not audio").is_err()); + assert!(codec::probe(&VOICE_ENVELOPE_FIXTURE[..VOICE_ENVELOPE_FIXTURE.len() - 10]).is_err()); + assert!(codec::waveform(VOICE_ENVELOPE_FIXTURE, 0).is_err()); + assert!(codec::waveform(VOICE_ENVELOPE_FIXTURE, 4097).is_err()); + let truncated_opus = codec::transcode( + VOICE_ENVELOPE_FIXTURE, + AudioCodec::WavPcm16, + AudioCodec::OggOpus, + )?; + assert!(codec::probe(&truncated_opus[..truncated_opus.len() / 2]).is_err()); + assert!( + codec::transcode( + &truncated_opus[..truncated_opus.len() / 2], + AudioCodec::OggOpus, + AudioCodec::WavPcm16, + ) + .is_err() + ); + let truncated_aac = codec::transcode( + VOICE_ENVELOPE_FIXTURE, + AudioCodec::WavPcm16, + AudioCodec::M4aAac, + )?; + assert!(codec::probe(&truncated_aac[..truncated_aac.len() / 2]).is_err()); + assert!( + codec::transcode( + &truncated_aac[..truncated_aac.len() / 2], + AudioCodec::M4aAac, + AudioCodec::WavPcm16, + ) + .is_err() + ); + Ok(()) +} diff --git a/crates/tinyvoice/tests/fixtures/CC0-1.0.txt b/crates/tinyvoice/tests/fixtures/CC0-1.0.txt new file mode 100644 index 0000000..670154e --- /dev/null +++ b/crates/tinyvoice/tests/fixtures/CC0-1.0.txt @@ -0,0 +1,116 @@ +CC0 1.0 Universal + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator and +subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for the +purpose of contributing to a commons of creative, cultural and scientific +works ("Commons") that the public can reliably and without fear of later +claims of infringement build upon, modify, incorporate in other works, reuse +and redistribute as freely as possible in any form whatsoever and for any +purposes, including without limitation commercial purposes. These owners may +contribute to the Commons to promote the ideal of a free culture and the +further production of creative, cultural and scientific works, or to gain +reputation or greater distribution for their Work in part through the use and +efforts of others. + +For these and/or other purposes and motivations, and without any expectation +of additional consideration or compensation, the person associating CC0 with a +Work (the "Affirmer"), to the extent that he or she is an owner of Copyright +and Related Rights in the Work, voluntarily elects to apply CC0 to the Work +and publicly distribute the Work under its terms, with knowledge of his or her +Copyright and Related Rights in the Work and the meaning and intended legal +effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not limited +to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, communicate, + and translate a Work; + + ii. moral rights retained by the original author(s) and/or performer(s); + + iii. publicity and privacy rights pertaining to a person's image or likeness + depicted in a Work; + + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + + v. rights protecting the extraction, dissemination, use and reuse of data in + a Work; + + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation thereof, + including any amended or successor version of such directive); and + + vii. other similar, equivalent or corresponding rights throughout the world + based on applicable law or treaty, and any national implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention of, +applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and +unconditionally waives, abandons, and surrenders all of Affirmer's Copyright +and Related Rights and associated claims and causes of action, whether now +known or unknown (including existing as well as future claims and causes of +action), in the Work (i) in all territories worldwide, (ii) for the maximum +duration provided by applicable law or treaty (including future time +extensions), (iii) in any current or future medium and for any number of +copies, and (iv) for any purpose whatsoever, including without limitation +commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes +the Waiver for the benefit of each member of the public at large and to the +detriment of Affirmer's heirs and successors, fully intending that such Waiver +shall not be subject to revocation, rescission, cancellation, termination, or +any other legal or equitable action to disrupt the quiet enjoyment of the Work +by the public as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason be +judged legally invalid or ineffective under applicable law, then the Waiver +shall be preserved to the maximum extent permitted taking into account +Affirmer's express Statement of Purpose. In addition, to the extent the Waiver +is so judged Affirmer hereby grants to each affected person a royalty-free, +non transferable, non sublicensable, non exclusive, irrevocable and +unconditional license to exercise Affirmer's Copyright and Related Rights in +the Work (i) in all territories worldwide, (ii) for the maximum duration +provided by applicable law or treaty (including future time extensions), (iii) +in any current or future medium and for any number of copies, and (iv) for any +purpose whatsoever, including without limitation commercial, advertising or +promotional purposes (the "License"). The License shall be deemed effective as +of the date CC0 was applied by Affirmer to the Work. Should any part of the +License for any reason be judged legally invalid or ineffective under +applicable law, such partial invalidity or ineffectiveness shall not +invalidate the remainder of the License, and in such case Affirmer hereby +affirms that he or she will not (i) exercise any of his or her remaining +Copyright and Related Rights in the Work or (ii) assert any associated claims +and causes of action with respect to the Work, in either case contrary to +Affirmer's express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + + b. Affirmer offers the Work as-is and makes no representations or warranties + of any kind concerning the Work, express, implied, statutory or otherwise, + including without limitation warranties of title, merchantability, fitness + for a particular purpose, non infringement, or the absence of latent or + other defects, accuracy, or the present or absence of errors, whether or not + discoverable, all to the greatest extent permissible under applicable law. + + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without limitation + any person's Copyright and Related Rights in the Work. Further, Affirmer + disclaims responsibility for obtaining any necessary consents, permissions + or other rights required for any use of the Work. + + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to this + CC0 or use of the Work. + +For more information, please see + diff --git a/crates/tinyvoice/tests/fixtures/README.md b/crates/tinyvoice/tests/fixtures/README.md new file mode 100644 index 0000000..47da09e --- /dev/null +++ b/crates/tinyvoice/tests/fixtures/README.md @@ -0,0 +1,16 @@ +# Codec test fixtures + +`voice-envelope.wav` is a deterministic, locally generated 2.371-second mono +PCM voice-band signal. Its harmonics and alternating voiced/silent sections +exercise multiframe encoding, duration clipping, and envelope retention without +network access or third-party audio provenance. + +`opus-interop.wav` is a deterministic, locally generated 2.5-millisecond mono +PCM tone used to exercise the independent Opus decoder against one encoder +packet. Longer multiframe Opus/AAC paths use `voice-envelope.wav`. + +`mp3-test.bit` is the Layer III test stream `vectors/l3-si.bit` from +[minimp3](https://github.com/lieff/minimp3), checked in as a source-format +fixture to cover common `audio/mpeg` text-to-speech responses. The upstream +repository dedicates its contents to CC0; the notice is included as +[`CC0-1.0.txt`](CC0-1.0.txt). diff --git a/crates/tinyvoice/tests/fixtures/mp3-test.bit b/crates/tinyvoice/tests/fixtures/mp3-test.bit new file mode 100644 index 0000000..94a5ef8 Binary files /dev/null and b/crates/tinyvoice/tests/fixtures/mp3-test.bit differ diff --git a/crates/tinyvoice/tests/fixtures/opus-interop.wav b/crates/tinyvoice/tests/fixtures/opus-interop.wav new file mode 100644 index 0000000..f36205f Binary files /dev/null and b/crates/tinyvoice/tests/fixtures/opus-interop.wav differ diff --git a/crates/tinyvoice/tests/fixtures/voice-envelope.wav b/crates/tinyvoice/tests/fixtures/voice-envelope.wav new file mode 100644 index 0000000..a0b69be Binary files /dev/null and b/crates/tinyvoice/tests/fixtures/voice-envelope.wav differ diff --git a/docs/specs/module-capture.md b/docs/specs/module-capture.md index 9d8c315..d905cd7 100644 --- a/docs/specs/module-capture.md +++ b/docs/specs/module-capture.md @@ -76,13 +76,13 @@ names, error detail, handles, audio, paths and request arguments. Local fixtures exercise lifecycle, bounds, faults and in-memory bus dispatch without hardware or external services. The actual compiled artifact verifier -checks all 53 declared members and permission denial without opening a device. +checks all 56 declared members and permission denial without opening a device. The native device bridge follows the repository's existing physical-device coverage exception; manager and processing code remain subject to the 90% per-file threshold. The release workflow installs ALSA development headers for Linux module builds. -OpenHuman must pin a released artifact compatible with current contract 1.10 +OpenHuman must pin a released artifact compatible with current contract 1.11 (this capture slice was introduced in 1.3) and verify its digest before switching callers. Module-owned hotkeys are specified in `hotkey-module-lifecycle.md`. diff --git a/docs/specs/voice-codecs.md b/docs/specs/voice-codecs.md new file mode 100644 index 0000000..b28df68 --- /dev/null +++ b/docs/specs/voice-codecs.md @@ -0,0 +1,36 @@ +# Voice-note codecs + +The `tinyvoice::codec` library API accepts complete in-memory byte slices. It +detects WAV/PCM16, Ogg/Opus, M4A/AAC-LC, and MPEG Layer III input, and exposes +`probe`, `transcode(bytes, from, to)`, and `waveform`. The declared source +format is checked against the input bytes; MP3 is decode-only. It performs no +filesystem, network, process, or device I/O. Decoding is bounded to 32 MiB of +input, five minutes of audio, and 32 million decoded samples. Sample rates +outside 8–192 kHz are rejected. +Voice encodes are mono 48 kHz; Opus uses the VoIP profile at 24 kbps and AAC-LC +uses 64 kbps. WAV-to-WAV passthrough preserves the original bytes. + +Opus encoding, decoding, and Ogg framing use `opus-pure`; M4A demux and AAC +decode use Symphonia; AAC-LC encoding uses `oxideav-aac`; and the `mp4` crate +muxes AAC access units. These are Rust dependencies enabled by default in the +library, so codec support is available without a system FFmpeg install or +native codec library. A Linux x86_64 release build of `tinyvoice-module` +measured 20.8 MiB total / 7.8 MiB `.text` with `cargo bloat`; its report +attributed 618.7 KiB `.text` to `opus-pure` and 118.7 KiB to `oxideav-aac` +(Symphonia's decoder is split across several crates). This is the complete +module size, not the codec-only delta; measure product artifacts before +setting a package-size budget. The codecs remain in the `tinyvoice` library, +while module hosts call them through `CodecProbe`, `CodecTranscode`, and +`CodecWaveform` in the minimal `tinyvoice-bus` contract. + +`tinyvoice-module` transports complete media as Base64 and caps each request +and output at 8 MiB to stay within bus-message limits. The library itself +accepts up to 32 MiB for non-bus consumers. Waveforms contain one normalized +peak byte per requested bucket, with a maximum of 4096 buckets; 256 buckets +match Discord's voice-message waveform shape. + +Integration tests use the checked-in samples documented in +[`tests/fixtures/README.md`](../../crates/tinyvoice/tests/fixtures/README.md), +round-trip through both encoded formats, decode Opus with a second pure-Rust +implementation and M4A with Symphonia, and exercise truncation and bucket +bounds. CI should build the library and module on Windows, macOS, and Linux.