Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ equal to the strings of the specification, dictionaries are typed dataclasses, a
| VP9 and AV1 | ✅ | ❌ | ✅ | ✅ |
| `pip install` on Linux, macOS and Windows | ✅ | ✅ | ✅ | ❌ system packages |

Against aiortc, it opens connections 5–7x faster, moves 4.5–9x more data through a data channel and spends less CPU on
Against aiortc, it opens connections 7–8x faster, moves 4.4–9x more data through a data channel and spends less CPU on
audio. See the [benchmarks](https://wrtc.marshal.dev/benchmarks/), which also measure Pion, libdatachannel and
webrtc-rs.

Expand Down
250 changes: 124 additions & 126 deletions benchmarks/RESULTS.md

Large diffs are not rendered by default.

19 changes: 16 additions & 3 deletions benchmarks/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@
import pathlib
import platform
import sys
import time
from typing import IO, TYPE_CHECKING

from benchmarks import report
from benchmarks.report import SUITES
from benchmarks.runner import BENCHMARKS, LIBRARIES, RESULTS, Driver, Options
from benchmarks.runner import BENCHMARKS, LIBRARIES, RESULTS, Driver, Options, duration
from benchmarks.stats import commit, go_version, machine

if TYPE_CHECKING:
Expand Down Expand Up @@ -153,8 +154,15 @@ def _manifest(options: Options, versions: dict[str, Hello]) -> Manifest:


async def _run(names: list[str], options: Options) -> Run:
today = dt.datetime.now().astimezone()
options.progress(f'started {today:%Y-%m-%d}: {", ".join(names)} of {", ".join(options.libraries)}')
manifest = _manifest(options, await _versions(options.libraries, options.log))
suites = {name: await SUITES[name].run(options) for name in names}
suites: dict[str, object] = {}
for name in names:
started = time.monotonic()
suites[name] = await SUITES[name].run(options)
options.progress(f'{name}: done in {duration(time.monotonic() - started)}')
options.progress(f'done in {duration(time.monotonic() - options.started)}')
return {'manifest': manifest, 'suites': suites}


Expand All @@ -166,7 +174,12 @@ def main() -> None:
stamp = dt.datetime.now().astimezone().strftime('%Y-%m-%dT%H%M%S')
name = f'{stamp}-{platform.system().lower()}-{platform.machine().lower()}'
with (directory / f'{name}.log').open('ab') as log:
results = asyncio.run(_run(command.suites, command.options(log)))
options = command.options(log)
try:
results = asyncio.run(_run(command.suites, options))
except KeyboardInterrupt:
options.progress(f'interrupted, no results saved; drivers logged to {log.name}')
sys.exit(130)

path = directory / f'{name}.json'
path.write_text(json.dumps(results, indent=1) + '\n', encoding='utf-8')
Expand Down
Binary file modified benchmarks/charts/datachannel-cpu.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/datachannel-rtt.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/datachannel-throughput.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/media-audio.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/media-video-call.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/media-video-latency.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/media-video-receive.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/media-video-send.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/memory.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmarks/charts/setup.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
16 changes: 15 additions & 1 deletion benchmarks/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,16 @@

import asyncio
import contextlib
import datetime as dt
import fnmatch
import itertools
import os
import pathlib
import shutil
import subprocess
import sys
from dataclasses import dataclass
import time
from dataclasses import dataclass, field
from typing import IO, TYPE_CHECKING, TypedDict, cast

import psutil
Expand Down Expand Up @@ -189,11 +191,22 @@ class Options:
repeat: int
log: IO[bytes]
cases: Sequence[str] = ()
started: float = field(default_factory=time.monotonic)

def selects(self, case: str) -> bool:
"""Whether a case of that name runs."""
return len(self.cases) == 0 or any(fnmatch.fnmatchcase(case, pattern) for pattern in self.cases)

def progress(self, message: str) -> None:
"""Writes a line to stderr after the time of day and the time since the run started."""
elapsed = duration(time.monotonic() - self.started)
print(f'{dt.datetime.now().astimezone():%H:%M:%S} +{elapsed} {message}', file=sys.stderr, flush=True)


def duration(seconds: float) -> str:
"""A span of seconds as H:MM:SS."""
return str(dt.timedelta(seconds=round(seconds)))


@dataclass
class Usage:
Expand Down Expand Up @@ -269,6 +282,7 @@ async def start(cls, library: str, log: IO[bytes]) -> Self:
cwd=ROOT,
env={**os.environ, 'WRTC_ALLOW_LOOPBACK': '1'},
limit=LINE_LIMIT,
start_new_session=True,
)

driver = cls(library, process)
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/suites/datachannel.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ async def run(options: Options) -> dict[str, Results]:
}

for library, cell in _cases(options):
print(f'datachannel: {library} {_name(cell)}', file=sys.stderr, flush=True)
options.progress(f'datachannel: {library} {_name(cell)}')
if cell is None:
results[library]['ping'].append(await ping_run(library, timing, options.log))
else:
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/suites/media.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,6 @@ async def _attempt(run: Awaitable[Run]) -> Run | None:

async def _record(case: Case, plan: Plan, log: IO[bytes], *, results: Results) -> None:
"""Runs a case into the results, None if it failed."""
print(f'media: {case.library} {case.kind} {case.name}', file=sys.stderr, flush=True)
if case.kind in AUDIO_KINDS:
audio = await _attempt(_audio(case, plan, log))
results['audio'].setdefault(case.kind, {}).setdefault(case.name, []).append(audio)
Expand Down Expand Up @@ -438,6 +437,7 @@ async def run(options: Options) -> MediaResults:
shuffled = cases * options.repeat
random.shuffle(shuffled)
for case in shuffled:
options.progress(f'media: {case.library} {case.kind} {case.name}')
await _record(case, plan, options.log, results=results[case.library])

order = list(dict.fromkeys(case.name for case in cases))
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/suites/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ async def run(options: Options) -> dict[str, Results]:
library: {'states': {state: [] for state in STATES}, 'growth': []} for library in options.libraries
}
for library, state in _cases(options):
print(f'memory: {library} {_name(state)}', file=sys.stderr, flush=True)
options.progress(f'memory: {library} {_name(state)}')
if state is None:
results[library]['growth'].append(await growth_run(library, sizes, options.log))
else:
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/suites/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ async def run(options: Options) -> dict[Mode, dict[str, Results]]:
}

for case in _cases(options):
print(f'setup: {case.library} {case.name}', file=sys.stderr, flush=True)
options.progress(f'setup: {case.library} {case.name}')
result, trickle = results[case.mode][case.library], case.mode == 'trickle'
if case.cold:
result['cold'].append(await cold_trial(case.library, options.log, trickle=trickle))
Expand Down
16 changes: 8 additions & 8 deletions docs/source/benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ processes, and the charts show the medians of each library's newest run.

Against aiortc, Python WebRTC:

- opens a connection **5–7x faster**;
- moves **4.5–9x more data** through a data channel, with **4–6x less CPU** per megabyte;
- spends **less CPU on audio** at every stream count, up to 4.5x less when sending;
- opens a connection **7–8x faster**;
- moves **4.4–9x more data** through a data channel, with **4–6x less CPU** per megabyte;
- spends **less CPU on audio** at every stream count, up to 4x less when sending;
- encodes H.264 in hardware on macOS, at **a tenth of the CPU** per frame;
- delivers call video with **about half the latency**;
- holds **more connections** in one process (1450 against 1100).
Expand All @@ -37,7 +37,7 @@ read worse because of that, and the sections below explain each one.

![Connection setup](https://raw.githubusercontent.com/MarshalX/python-webrtc/main/benchmarks/charts/setup.png)

The time from creating a connection to an open data channel: about 5 ms, against 29 ms for aiortc. Cold runs start
The time from creating a connection to an open data channel: about 3.5 ms, against 29 ms for aiortc. Cold runs start
fresh processes. The benchmark keeps the defaults: DTLS 1.3 is on, while SNAP and SPED, the other parts of
[WARP](guides/field-trials.md#warp), are off and would save more round trips.

Expand Down Expand Up @@ -72,7 +72,7 @@ This chart favors aiortc, for two reasons that come from the engine rather than
Memory stays reserved after connections close, because the allocator keeps it for reuse: opening and closing 100
connections over and over doesn't grow the process past the first round.

The process itself starts smaller, 91 MB against 126 MB, and holds 1450 connections where aiortc stops at 1100.
The process itself starts smaller, 92 MB against 128 MB, and holds 1450 connections where aiortc stops at 1100.

## Media

Expand All @@ -81,7 +81,7 @@ The process itself starts smaller, 91 MB against 126 MB, and holds 1450 connecti
![Video receive](https://raw.githubusercontent.com/MarshalX/python-webrtc/main/benchmarks/charts/media-video-receive.png)

CPU for each frame received, decoded and read in Python. VP8 decodes with the same library in both, so the bars stay
close. H.264 at 1080p is cheaper, decoded by VideoToolbox on macOS.
close. H.264 is cheaper at every resolution, decoded by VideoToolbox on macOS.

### Video send

Expand All @@ -102,13 +102,13 @@ encoder on macOS and costs a tenth of the CPU per frame.
![Video call](https://raw.githubusercontent.com/MarshalX/python-webrtc/main/benchmarks/charts/media-video-call.png)

Two peers of the same library call each other, and each library's congestion control picks the bitrate. Python WebRTC
reaches the bitrate the video asks for, 2.5 or 5 Mbps, while aiortc stays near 1.5–2.3 Mbps. The panel under the bars
reaches the bitrate the video asks for, 2.5 or 5 Mbps, while aiortc stays near 1.5–2.4 Mbps. The panel under the bars
shows it: Python WebRTC receives up to three times the bits for every frame, so a better picture, and decoding more
bits costs more CPU per frame.

![Video call latency](https://raw.githubusercontent.com/MarshalX/python-webrtc/main/benchmarks/charts/media-video-latency.png)

From writing a frame on one side to reading it on the other: 21–30 ms against aiortc's 43 ms, at the higher
From writing a frame on one side to reading it on the other: 22–28 ms against aiortc's 44 ms, at the higher
bitrate.

### Audio
Expand Down
22 changes: 6 additions & 16 deletions docs/source/guides/aiortc.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,22 +75,12 @@ Behaves differently:

## Performance

Code moved to the layer runs several times faster than on aiortc. aiortc's own benchmark driver, run once on aiortc
1.15 and once on the layer, in the same session: Apple M1, medians of 3 runs.

| Case | Layer against aiortc |
|------|----------------------|
| Connection setup, to an open data channel | **5.4x faster**: 5.8 ms against 31.4 ms |
| Data channel throughput | **3.9–7.6x more** data a second |
| Audio send, CPU per stream | **3–4x less** |
| Connections in one process | 1100 against 850 |
| Memory per connection | **more**: 3.6x bare, 2x with a data channel, 1.45x with audio |

Memory per connection is where aiortc is lighter: the engine keeps a full browser's state for each connection. See
[Benchmarks](../benchmarks.md#memory). Video send isn't compared, since the two encode different resolutions.

Against the same code written for the native API, the layer costs 0–8% more. The difference is one copy per received
video frame into a PyAV frame, and about 24 MB at startup from importing PyAV.
Python WebRTC opens connections 7–8x faster than aiortc, moves 4.4–9x more data through a data channel and spends
less CPU on audio. See the [benchmarks](../benchmarks.md).

The layer gives up some of that for compatibility: it costs up to 8% more than the same code on the native API. Each
received video frame is copied into a PyAV frame, and importing PyAV adds about 24 MB at startup. The native API has
neither cost, see [Moving to the native API](#moving-to-the-native-api).

## Moving to the native API

Expand Down
30 changes: 28 additions & 2 deletions docs/source/guides/field-trials.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Field trials

The native WebRTC engine ships some features turned off, behind field trials. A trial has a name, like
`WebRTC-Sctp-Snap`, and a group, usually `Enabled`. Browsers set them with a command-line flag. In python-webrtc,
`WebRTC-Sctp-Snap`, and a group, usually `Enabled`. Browsers set them with a command-line flag. In Python WebRTC,
`webrtc.field_trials` sets them: a mapping of names to groups that works like {data}`os.environ`.

```python
Expand Down Expand Up @@ -32,6 +32,32 @@ The engine reads the trials once, when it starts: on the first {class}`~webrtc.R
Trial names change between releases of the engine, which is libwebrtc M152 in this release. An unknown name is
accepted and does nothing, as it does in the browsers, so check the spelling.

## Socket receive buffer

Python WebRTC gives every UDP socket a receive buffer of at least 8 MB, so a burst of data channel messages isn't
dropped while the engine is busy. Setting `WebRTC-SetSocketReceiveBuffer` to any group turns this off: `Enabled-<kb>`
sets your size, and `Disabled` leaves it to the engine and the system.

Linux caps the buffer at `net.core.rmem_max`, often about 200 KB. With that cap, a flood of data channel messages
overflows the buffer, and the channel can stall for seconds while SCTP recovers the lost packets. To lift the cap:

```bash
sudo sysctl -w net.core.rmem_max=8388608
```

Add `net.core.rmem_max = 8388608` to `/etc/sysctl.conf` to keep it after a reboot.

## Connected sockets

On macOS and Linux, when one remote address both sends to a UDP socket and receives from it, like the peer of a busy
data channel, Python WebRTC gives that peer a second socket on the same local address, connected to it. The kernel
then skips its route and socket lookups on every datagram, which makes data channels faster. Packets on the wire don't
change, so the peer can't tell, and every other address keeps using the first socket.

Set `PythonWebRTC-ConnectedSockets` to `Disabled` to keep one socket per address, as the engine does on its own: to
rule this out while debugging a connection, or on a machine shared with other users, as the second socket shares the
address while it binds.

## WARP

[WARP](https://datatracker.ietf.org/doc/draft-uberti-tsvwg-warp/) connects faster by saving round trips. It's made of four parts, and each of them works on its own: a peer that
Expand All @@ -55,5 +81,5 @@ The server has to know the `id` of the channel, so the signalling sends it along
[OpenAI's WARP guide](https://developers.openai.com/api/docs/guides/realtime-webrtc-warp) describes. The
[GPT-Live example](../examples/openai_live.md) connects this way.

With both trials, a data channel between two python-webrtc peers opens 50% sooner, and a GPT-Live session starts 62%
With both trials, a data channel between two Python WebRTC peers opens 50% sooner, and a GPT-Live session starts 62%
sooner once the API has answered.
2 changes: 1 addition & 1 deletion docs/source/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ Prebuilt wheels for Linux, macOS and Windows, including free-threaded Python. No
| VP9 and AV1 | ✅ | ❌ | ✅ | ✅ |
| `pip install` on Linux, macOS and Windows | ✅ | ✅ | ✅ | ❌ system packages |

Against aiortc, it opens connections 5–7x faster, moves 4.5–9x more data through a data channel and spends less CPU on
Against aiortc, it opens connections 7–8x faster, moves 4.4–9x more data through a data channel and spends less CPU on
audio. See the [benchmarks](benchmarks.md), which also measure Pion, libdatachannel and webrtc-rs.

## Where to start
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"

[project]
name = "wrtc" # "webrtc" isn't allowed on PyPI
version = "0.0.2"
version = "0.0.3"
description = "Chrome's WebRTC engine (libwebrtc M152), with the browser API, in Python"
readme = "README.md"
license = "BSD-3-Clause"
Expand Down
3 changes: 2 additions & 1 deletion python-webrtc/cpp/NATIVE.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ Python garbage.

- Every change of binding or activity ends in an update of the root on the loop that holds it. A change made from
Python goes through `_changing()` (a spent `once` handler too), so roots checks on other loops wait for it. An
unbind updates roots after unbinding, and `LoopState.update` re-reads the binding after storing a root
unbind updates roots after unbinding, on the loop of the mailbox the unbind itself found (`Unbind()` returns its
id), as a loop read before it may be stale. `LoopState.update` re-reads the binding after storing a root
(`_recheck`): whichever runs last sees the other.
- An object is kept for the events still owed to it, not for the state it shows (an announced data channel reads
open before its open event).
Expand Down
Loading
Loading