Fix generator frame drops, adapt relayed video and keep local sinks unadapted

python-webrtc/cpp/src/interfaces/rtc_video_track_source.cpp

@@ -14,6 +14,9 @@
 #include <api/video/i420_buffer.h>
 #include <rtc_base/time_utils.h>
 
+#include "../media/frame_adaptation.h"
+#include "../media/local_video_sink.h"
+
 namespace python_webrtc {
 
   RTCVideoTrackSource::RTCVideoTrackSource(bool isScreencast, std::optional<bool> needsDenoising)
@@ -100,29 +103,16 @@ namespace python_webrtc {
   }
 
   void RTCVideoTrackSource::Broadcast(const webrtc::VideoFrame &frame) {
+    _unadapted.OnFrame(frame);
     if (!_broadcaster.frame_wanted()) {
       return;
     }
-    int croppedWidth = 0;
-    int croppedHeight = 0;
-    int width = 0;
-    int height = 0;
-    // timed by arrival, as generator frames carry the application's timestamps
-    if (!_adapter.AdaptFrameResolution(frame.width(), frame.height(), webrtc::TimeNanos(), &croppedWidth,
-                                       &croppedHeight, &width, &height)) {
+    auto adapted = AdaptFrame(_adapter, frame);
+    if (adapted) {
+      _broadcaster.OnFrame(*adapted);
+    } else {
       _broadcaster.OnDiscardedFrame();
-      return;
     }
-    if (width == frame.width() && height == frame.height()) {
-      _broadcaster.OnFrame(frame);
-      return;
-    }
-    webrtc::VideoFrame adapted = frame;
-    adapted.set_video_frame_buffer(frame.video_frame_buffer()->CropAndScale(
-        (frame.width() - croppedWidth) / 2, (frame.height() - croppedHeight) / 2, croppedWidth, croppedHeight, width,
-        height));
-    adapted.clear_update_rect();
-    _broadcaster.OnFrame(adapted);
   }
 
   void RTCVideoTrackSource::End() {
@@ -142,13 +132,19 @@ namespace python_webrtc {
 
   void RTCVideoTrackSource::AddOrUpdateSink(webrtc::VideoSinkInterface<webrtc::VideoFrame> *sink,
                                             const webrtc::VideoSinkWants &wants) {
+    auto *broadcaster = &_broadcaster;
+    if (wants.black_frames) {
+      broadcaster = &_blackBroadcaster;
+    } else if (LocalVideoSink::Is(sink)) {
+      broadcaster = &_unadapted;
+    }
     const std::scoped_lock lock(_sinksMutex);
-    auto [it, added] = _sinks.try_emplace(sink, wants.black_frames);
-    if (!added && it->second != wants.black_frames) {
-      (it->second ? _blackBroadcaster : _broadcaster).RemoveSink(sink);
-      it->second = wants.black_frames;
+    auto [it, added] = _sinks.try_emplace(sink, broadcaster);
+    if (!added && it->second != broadcaster) {
+      it->second->RemoveSink(sink);
+      it->second = broadcaster;
     }
-    (wants.black_frames ? _blackBroadcaster : _broadcaster).AddOrUpdateSink(sink, wants);
+    broadcaster->AddOrUpdateSink(sink, wants);
     _adapter.OnSinkWants(_broadcaster.wants());
   }
 
@@ -158,7 +154,7 @@ namespace python_webrtc {
     if (it == _sinks.end()) {
       return;
     }
-    (it->second ? _blackBroadcaster : _broadcaster).RemoveSink(sink);
+    it->second->RemoveSink(sink);
     _sinks.erase(it);
     _adapter.OnSinkWants(_broadcaster.wants());
   }

python-webrtc/cpp/src/interfaces/rtc_video_track_source.h

@@ -84,10 +84,11 @@ namespace python_webrtc {
     const std::optional<bool> _needsDenoising;
     webrtc::VideoBroadcaster _broadcaster;
     webrtc::VideoAdapter _adapter;
+    webrtc::VideoBroadcaster _unadapted;
     // sinks of disabled or ended tracks: a black frame a second
     webrtc::VideoBroadcaster _blackBroadcaster;
     std::mutex _sinksMutex;
-    std::map<webrtc::VideoSinkInterface<webrtc::VideoFrame> *, bool> _sinks;
+    std::map<webrtc::VideoSinkInterface<webrtc::VideoFrame> *, webrtc::VideoBroadcaster *> _sinks;
     std::atomic<int64_t> _lastBlackMs{0};
     std::atomic<int> _width{0};
     std::atomic<int> _height{0};

python-webrtc/cpp/src/media/frame_adaptation.h

@@ -0,0 +1,43 @@
+//
+// Copyright 2022-2026 Ilya (Marshal) <https://github.com/MarshalX>. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE.md file in the root of the project.
+//
+
+#ifndef PYTHON_WEBRTC_MEDIA_FRAME_ADAPTATION_H_
+#define PYTHON_WEBRTC_MEDIA_FRAME_ADAPTATION_H_
+
+#include <optional>
+
+#include <api/video/video_adapter.h>
+#include <api/video/video_frame.h>
+#include <rtc_base/time_utils.h>
+
+namespace python_webrtc {
+
+  // The frame as the adapter's sinks want it, timed by its capture time: cropped and scaled, or none when dropped
+  inline std::optional<webrtc::VideoFrame> AdaptFrame(webrtc::VideoAdapter &adapter, const webrtc::VideoFrame &frame) {
+    int croppedWidth = 0;
+    int croppedHeight = 0;
+    int width = 0;
+    int height = 0;
+    if (!adapter.AdaptFrameResolution(frame.width(), frame.height(),
+                                      frame.timestamp_us() * webrtc::kNumNanosecsPerMicrosec, &croppedWidth,
+                                      &croppedHeight, &width, &height)) {
+      return std::nullopt;
+    }
+    if (width == frame.width() && height == frame.height()) {
+      return frame;
+    }
+    webrtc::VideoFrame adapted = frame;
+    adapted.set_video_frame_buffer(frame.video_frame_buffer()->CropAndScale(
+        (frame.width() - croppedWidth) / 2, (frame.height() - croppedHeight) / 2, croppedWidth, croppedHeight, width,
+        height));
+    adapted.clear_update_rect();
+    return adapted;
+  }
+
+} // namespace python_webrtc
+
+#endif // PYTHON_WEBRTC_MEDIA_FRAME_ADAPTATION_H_

python-webrtc/cpp/src/media/local_video_sink.h

@@ -0,0 +1,60 @@
+//
+// Copyright 2022-2026 Ilya (Marshal) <https://github.com/MarshalX>. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE.md file in the root of the project.
+//
+
+#ifndef PYTHON_WEBRTC_MEDIA_LOCAL_VIDEO_SINK_H_
+#define PYTHON_WEBRTC_MEDIA_LOCAL_VIDEO_SINK_H_
+
+#include <mutex>
+#include <set>
+
+#include <api/video/video_frame.h>
+#include <api/video/video_sink_interface.h>
+
+#include "../utils/libwebrtc_thread.h"
+
+namespace python_webrtc {
+
+  // A sink of ours on a track (a processor, a monitor): it sees the frames of the track's source, never the ones a
+  // source adapts for an encoder. Registered while it exists, as sinks reach a source through libwebrtc's tracks.
+  class LocalVideoSink : public webrtc::VideoSinkInterface<webrtc::VideoFrame> {
+  public:
+    LocalVideoSink() {
+      auto &sinks = Sinks();
+      const std::scoped_lock lock(sinks.mutex);
+      sinks.sinks.insert(this);
+    }
+
+    ~LocalVideoSink() override {
+      auto &sinks = Sinks();
+      const std::scoped_lock lock(sinks.mutex);
+      sinks.sinks.erase(this);
+    }
+
+    LocalVideoSink(const LocalVideoSink &) = delete;
+    LocalVideoSink &operator=(const LocalVideoSink &) = delete;
+
+    static bool Is(const webrtc::VideoSinkInterface<webrtc::VideoFrame> *sink) {
+      auto &sinks = Sinks();
+      const std::scoped_lock lock(sinks.mutex);
+      return sinks.sinks.contains(sink);
+    }
+
+  private:
+    struct Registry {
+      std::mutex mutex;
+      std::set<const webrtc::VideoSinkInterface<webrtc::VideoFrame> *> sinks;
+    };
+
+    static Registry &Sinks() {
+      static ForkLocal<Registry> registry;
+      return registry.Get();
+    }
+  };
+
+} // namespace python_webrtc
+
+#endif // PYTHON_WEBRTC_MEDIA_LOCAL_VIDEO_SINK_H_

python-webrtc/cpp/src/media/media_stream_track_processor.cpp

@@ -118,6 +118,8 @@ namespace python_webrtc {
       constexpr int64_t kRtpTicksPerMs = 90;
       timestampUs =
           _firstReceived->first + ((rtp - _firstReceived->second) * webrtc::kNumMicrosecsPerMillisec / kRtpTicksPerMs);
+    } else if (frame.presentation_timestamp()) {
+      timestampUs = frame.presentation_timestamp()->us();
     }
     Push(VideoItem{.buffer = frame.video_frame_buffer(),
                    .timestampUs = timestampUs,

python-webrtc/cpp/src/media/media_stream_track_processor.h

@@ -27,6 +27,7 @@
 #include "../interfaces/media_stream_track.h"
 #include "../utils/mailbox.h"
 #include "../utils/native_object.h"
+#include "local_video_sink.h"
 #include "video_frame_buffer.h"
 
 namespace python_webrtc {
@@ -35,7 +36,7 @@ namespace python_webrtc {
   class MediaStreamTrackProcessor : public NativeObject<MediaStreamTrackProcessor>,
                                     public Emitter<MediaStreamTrackProcessor>,
                                     public TrackEndObserver,
-                                    public webrtc::VideoSinkInterface<webrtc::VideoFrame>,
+                                    public LocalVideoSink,
                                     public webrtc::AudioTrackSinkInterface {
   public:
     static constexpr const char *kName = "MediaStreamTrackProcessor";

python-webrtc/cpp/src/media/relayed_video_track.cpp

@@ -11,6 +11,8 @@
 
 #include <rtc_base/time_utils.h>
 
+#include "frame_adaptation.h"
+
 namespace python_webrtc {
 
   RelayedVideoTrack::RelayedVideoTrack(webrtc::scoped_refptr<webrtc::VideoTrackInterface> track)
@@ -27,7 +29,11 @@ namespace python_webrtc {
     if (!restamper) {
       restamper = std::make_unique<Restamper>(sink);
     }
-    _track->AddOrUpdateSink(restamper.get(), wants);
+    restamper->OnSinkWants(wants);
+    webrtc::VideoSinkWants delivered;
+    delivered.rotation_applied = wants.rotation_applied;
+    delivered.black_frames = wants.black_frames;
+    _track->AddOrUpdateSink(restamper.get(), delivered);
   }
 
   void RelayedVideoTrack::RemoveSink(webrtc::VideoSinkInterface<webrtc::VideoFrame> *sink) {
@@ -42,7 +48,12 @@ namespace python_webrtc {
     webrtc::VideoFrame restamped = frame;
     restamped.set_timestamp_us(_aligner.TranslateTimestamp(frame.timestamp_us(), webrtc::TimeMicros()));
     restamped.set_ntp_time_ms(0);
-    _sink->OnFrame(restamped);
+    auto adapted = AdaptFrame(_adapter, restamped);
+    if (adapted) {
+      _sink->OnFrame(*adapted);
+    } else {
+      _sink->OnDiscardedFrame();
+    }
   }
 
 } // namespace python_webrtc

python-webrtc/cpp/src/media/relayed_video_track.h

@@ -15,14 +15,15 @@
 #include <api/media_stream_interface.h>
 #include <api/scoped_refptr.h>
 #include <api/sequence_checker.h>
+#include <api/video/video_adapter.h>
 #include <api/video/video_frame.h>
 #include <api/video/video_sink_interface.h>
 #include <rtc_base/timestamp_aligner.h>
 
 namespace python_webrtc {
 
   // A track with a remote source as its sender sees it: frames restamped on the local clock without their remote NTP
-  // time, since encoders drop frames whose capture time goes back
+  // time, since encoders drop frames whose capture time goes back, and adapted to what the encoder wants
   class RelayedVideoTrack : public webrtc::VideoTrackInterface {
   public:
     explicit RelayedVideoTrack(webrtc::scoped_refptr<webrtc::VideoTrackInterface> track);
@@ -66,6 +67,8 @@ namespace python_webrtc {
     public:
       explicit Restamper(webrtc::VideoSinkInterface<webrtc::VideoFrame> *sink) : _sink(sink) {}
 
+      void OnSinkWants(const webrtc::VideoSinkWants &wants) { _adapter.OnSinkWants(wants); }
+
       void OnFrame(const webrtc::VideoFrame &frame) override;
 
       void OnDiscardedFrame() override { _sink->OnDiscardedFrame(); }
@@ -77,6 +80,7 @@ namespace python_webrtc {
     private:
       webrtc::VideoSinkInterface<webrtc::VideoFrame> *const _sink;
       webrtc::TimestampAligner _aligner;
+      webrtc::VideoAdapter _adapter;
     };
 
     const webrtc::scoped_refptr<webrtc::VideoTrackInterface> _track;

python-webrtc/cpp/src/media/track_generator.cpp

@@ -15,10 +15,10 @@
 
 #include <api/audio/audio_frame.h>
 #include <api/audio/audio_view.h>
+#include <api/units/timestamp.h>
 #include <api/video/video_frame.h>
 #include <rtc_base/crypto_random.h>
 #include <rtc_base/time_utils.h>
-#include <system_wrappers/include/clock.h>
 
 #include "../exceptions.h"
 #include "../utils/gil.h"
@@ -93,11 +93,18 @@ namespace python_webrtc {
     if (!GetLive() || _muted) {
       return;
     }
-    // sinks see the timestamp of the application, encoders take the capture time from the NTP one
+    // sinks see the timestamp of the application, encoders a capture time on the system clock, spaced as it is
+    const std::scoped_lock lock(_videoMutex);
+    int64_t captureUs = _aligner.TranslateTimestamp(timestampUs, webrtc::TimeMicros());
+    // encoders drop a frame whose capture millisecond isn't after the previous one's
+    if (captureUs / webrtc::kNumMicrosecsPerMillisec <= _lastCaptureMs) {
+      captureUs = (_lastCaptureMs + 1) * webrtc::kNumMicrosecsPerMillisec;
+    }
+    _lastCaptureMs = captureUs / webrtc::kNumMicrosecsPerMillisec;
     auto frame = webrtc::VideoFrame::Builder()
                      .set_video_frame_buffer(buffer->ToWebrtc())
-                     .set_timestamp_us(timestampUs)
-                     .set_ntp_time_ms(webrtc::Clock::GetRealTimeClock()->CurrentNtpInMilliseconds())
+                     .set_timestamp_us(captureUs)
+                     .set_presentation_timestamp(webrtc::Timestamp::Micros(timestampUs))
                      .set_rotation(static_cast<webrtc::VideoRotation>(rotation))
                      .build();
     _videoSource->PushFrame(frame);

python-webrtc/cpp/src/media/track_generator.h

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include <api/scoped_refptr.h>
+#include <rtc_base/timestamp_aligner.h>
 
 #include <pybind11/pybind11.h>
 
@@ -65,6 +66,10 @@ namespace python_webrtc {
     std::shared_ptr<SourceControl> _control = std::make_shared<SourceControl>();
     std::atomic<bool> _muted = false;
 
+    std::mutex _videoMutex;
+    webrtc::TimestampAligner _aligner;
+    int64_t _lastCaptureMs = 0;
+
     // samples short of a 10 ms frame, sent with the next ones of the same format
     std::mutex _audioMutex;
     std::vector<int16_t> _pending;

python-webrtc/cpp/src/media/track_monitor.h

@@ -19,10 +19,12 @@
 #include <api/video/video_sink_interface.h>
 #include <rtc_base/time_utils.h>
 
+#include "local_video_sink.h"
+
 namespace python_webrtc {
 
   // What a track carries, for its settings: the size and rate of its frames, the format of its audio
-  class TrackMonitor : public webrtc::VideoSinkInterface<webrtc::VideoFrame>, public webrtc::AudioTrackSinkInterface {
+  class TrackMonitor : public LocalVideoSink, public webrtc::AudioTrackSinkInterface {
   public:
     struct Video {
       int width;

tests/helpers.py

@@ -356,6 +356,8 @@ async def write_video(
         timestamp += round(interval * 1_000_000)
         await asyncio.sleep(interval)
 
+    writer.release_lock()
+
 
 @contextlib.asynccontextmanager
 async def writing(write: Callable[..., Awaitable[None]], *args: object, **kwargs: object) -> AsyncGenerator[None]: