From dce829510355f895d715b04189fd1274110bf94f Mon Sep 17 00:00:00 2001 From: Rajesh Gopinath Kumar <10174122+rajeshgk@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:12:52 -0500 Subject: [PATCH] Fix hardware decoder selection and software fallback Keep hardware decoder state per FFmpegReader instance. Preserve the selected backend during pixel format negotiation. Fall back to software decoding when hardware device initialization fails. Improve hardware device context cleanup and retry handling. Add deterministic CUDA initialization failure regression coverage. --- src/FFmpegReader.cpp | 180 ++++++++++++++++++++--------------------- src/FFmpegReader.h | 13 ++- tests/FFmpegReader.cpp | 75 +++++++++++++++++ 3 files changed, 173 insertions(+), 95 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index ca32da3fd8..35a8611ea8 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -71,11 +71,6 @@ typedef struct VAAPIDecodeContext { using namespace openshot; -int hw_de_on = 0; -#if USE_HW_ACCEL - AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE; - AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE; -#endif FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader) @@ -140,12 +135,16 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 #if USE_HW_ACCEL // Get hardware pix format -static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) +AVPixelFormat FFmpegReader::GetHardwareDecodeFormat(AVCodecContext *ctx, const AVPixelFormat *pix_fmts) { + FFmpegReader *reader = static_cast(ctx->opaque); + if (!reader) + return AV_PIX_FMT_NONE; + const enum AVPixelFormat *p; // Prefer only the format matching the selected hardware decoder - int selected = openshot::Settings::Instance()->HARDWARE_DECODER; + int selected = reader->selected_hw_decoder; for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { switch (*p) { @@ -153,15 +152,15 @@ static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPi // Linux pix formats case AV_PIX_FMT_VAAPI: if (selected == 1) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_VAAPI; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; return *p; } break; case AV_PIX_FMT_VDPAU: if (selected == 6) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_VDPAU; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU; return *p; } break; @@ -170,15 +169,15 @@ static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPi // Windows pix formats case AV_PIX_FMT_DXVA2_VLD: if (selected == 3) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_DXVA2_VLD; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; return *p; } break; case AV_PIX_FMT_D3D11: if (selected == 4) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_D3D11; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA; return *p; } break; @@ -187,8 +186,8 @@ static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPi // Apple pix formats case AV_PIX_FMT_VIDEOTOOLBOX: if (selected == 5) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; return *p; } break; @@ -196,15 +195,15 @@ static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPi // Cross-platform pix formats case AV_PIX_FMT_CUDA: if (selected == 2) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_CUDA; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; return *p; } break; case AV_PIX_FMT_QSV: if (selected == 7) { - hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV; + reader->hw_de_av_pix_fmt = AV_PIX_FMT_QSV; + reader->hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; return *p; } break; @@ -246,7 +245,8 @@ void FFmpegReader::Open() { // Initialize format context pFormatCtx = NULL; { - hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0); + selected_hw_decoder = openshot::Settings::Instance()->HARDWARE_DECODER; + hw_de_on = (!force_sw_decode && selected_hw_decoder != 0 ? 1 : 0); hw_decode_failed = false; hw_decode_error_count = 0; hw_decode_succeeded = false; @@ -333,74 +333,56 @@ void FFmpegReader::Open() { Logger::Instance()->AppendDebugMethod("Hardware decoding device number", "adapter_num", adapter_num); // Set hardware pix format (callback) - pCodecCtx->get_format = get_hw_dec_format; + pCodecCtx->opaque = this; + pCodecCtx->get_format = FFmpegReader::GetHardwareDecodeFormat; - if (adapter_num < 3 && adapter_num >=0) { #if defined(__linux__) - snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128); - adapter_ptr = adapter; - i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER; - switch (i_decoder_hw) { - case 1: - hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; - break; - case 2: - hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; - break; - case 6: - hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU; - break; - case 7: - hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; - break; - default: - hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; - break; - } - + i_decoder_hw = selected_hw_decoder; + switch (i_decoder_hw) { + case 1: hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; break; + case 2: hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; break; + case 6: hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU; break; + case 7: hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; break; + default: hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; break; + } #elif defined(_WIN32) - adapter_ptr = NULL; - i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER; - switch (i_decoder_hw) { - case 2: - hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; - break; - case 3: - hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; - break; - case 4: - hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA; - break; - case 7: - hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; - break; - default: - hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; - break; - } + i_decoder_hw = selected_hw_decoder; + switch (i_decoder_hw) { + case 2: hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; break; + case 3: hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; break; + case 4: hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA; break; + case 7: hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; break; + default: hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; break; + } #elif defined(__APPLE__) - adapter_ptr = NULL; - i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER; - switch (i_decoder_hw) { - case 5: - hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; - break; - case 7: - hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; - break; - default: - hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; - break; - } + i_decoder_hw = selected_hw_decoder; + switch (i_decoder_hw) { + case 5: hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; break; + case 7: hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; break; + default: hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; break; + } #endif + if (adapter_num >= 0 && adapter_num < 3) { +#if defined(__linux__) + if (i_decoder_hw == 2) { + // CUDA uses a GPU index rather than a DRM render node. + snprintf(adapter, sizeof(adapter), "%d", adapter_num); + } else { + snprintf(adapter, sizeof(adapter), + "/dev/dri/renderD%d", adapter_num + 128); + } + adapter_ptr = adapter; +#endif } else { - adapter_ptr = NULL; // Just to be sure + adapter_ptr = NULL; } // Check if it is there and writable #if defined(__linux__) - if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) { + if (adapter_ptr != NULL && + (hw_de_av_device_type == AV_HWDEVICE_TYPE_CUDA || + access(adapter_ptr, W_OK) == 0)) { #elif defined(_WIN32) if( adapter_ptr != NULL ) { #elif defined(__APPLE__) @@ -413,9 +395,11 @@ void FFmpegReader::Open() { Logger::Instance()->AppendDebugMethod("Decode Device not present using default"); } - hw_device_ctx = NULL; + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + } // Here the first hardware initialisations are made - if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { + if (hardware_device_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type); std::string hw_msg = "HW decode active: "; hw_msg += (hw_name ? hw_name : "unknown"); @@ -453,11 +437,20 @@ void FFmpegReader::Open() { } else { Logger::Instance()->Log("HW decode active: no (falling back to software)"); - throw InvalidCodec("Hardware device create failed.", path); + hw_de_supported = 0; + retry_decode_open = 1; + AV_FREE_CONTEXT(pCodecCtx); + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = NULL; + } } } #endif // USE_HW_ACCEL + if (retry_decode_open) + continue; + // Disable per-frame threading for album arts // Using FF_THREAD_FRAME adds one frame decoding delay per thread, // but there's only one frame in this case. @@ -482,8 +475,10 @@ void FFmpegReader::Open() { // TODO: needs va_config! #if ENABLE_VAAPI - ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config; - constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig); + if (hwconfig) { + ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config; + constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx, hwconfig); + } #endif // ENABLE_VAAPI if (constraints) { if (pCodecCtx->coded_width < constraints->min_width || @@ -505,9 +500,6 @@ void FFmpegReader::Open() { retry_decode_open = 0; } av_hwframe_constraints_free(&constraints); - if (hwconfig) { - av_freep(&hwconfig); - } } else { int max_h, max_w; @@ -535,6 +527,9 @@ void FFmpegReader::Open() { retry_decode_open = 0; } } + if (hwconfig) { + av_freep(&hwconfig); + } } // if hw_de_on && hw_de_supported else { Logger::Instance()->AppendDebugMethod("\nDecode in software is used\n"); @@ -754,11 +749,8 @@ void FFmpegReader::Close() { } AV_FREE_CONTEXT(pCodecCtx); #if USE_HW_ACCEL - if (hw_de_on) { - if (hw_device_ctx) { - av_buffer_unref(&hw_device_ctx); - hw_device_ctx = NULL; - } + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); } #endif // USE_HW_ACCEL if (img_convert_ctx) { @@ -1564,8 +1556,8 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL // Get the format from the variables set in get_hw_dec_format - hw_de_av_pix_fmt = hw_de_av_pix_fmt_global; - hw_de_av_device_type = hw_de_av_device_type_global; + /* Hardware pixel format is maintained per reader. */ + /* Hardware device type is maintained per reader. */ #endif // USE_HW_ACCEL if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) { Logger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts); diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index 7a8143e1b2..fab309b0ac 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -108,7 +108,14 @@ namespace openshot { int videoStream, audioStream; AVCodecContext *pCodecCtx, *aCodecCtx; #if USE_HW_ACCEL - AVBufferRef *hw_device_ctx = NULL; //PM + AVBufferRef *hw_device_ctx = NULL; + using HardwareDeviceCreateFn = int (*)( + AVBufferRef **, + enum AVHWDeviceType, + const char *, + AVDictionary *, + int); + HardwareDeviceCreateFn hardware_device_create = &av_hwdevice_ctx_create; #endif AVStream *pStream, *aStream; AVPacket *packet; @@ -167,6 +174,8 @@ namespace openshot { SWRCONTEXT *avr_ctx = nullptr; ///< Cached audio resample context AVFrame *pFrameRGB_cached = nullptr; ///< Temporary frame used for video conversion + int selected_hw_decoder = 0; + int hw_de_on = 0; int hw_de_supported = 0; // Is set by FFmpegReader bool force_sw_decode = false; bool hw_decode_failed = false; @@ -175,6 +184,8 @@ namespace openshot { #if USE_HW_ACCEL AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE; AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE; + static AVPixelFormat GetHardwareDecodeFormat( + AVCodecContext *ctx, const AVPixelFormat *pix_fmts); int IsHardwareDecodeSupported(int codecid); #endif diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 16c5e12a77..16d227ae14 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -802,6 +802,81 @@ TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[li r.Close(); } + +#if defined(__linux__) && USE_HW_ACCEL +static int forced_hw_device_create_calls = 0; + +static int ForceHardwareDeviceCreationFailure( + AVBufferRef **device, + enum AVHWDeviceType, + const char *, + AVDictionary *, + int) +{ + ++forced_hw_device_create_calls; + if (device) + *device = nullptr; + return AVERROR(ENODEV); +} +#endif + +TEST_CASE("CUDA initialization failure falls back to software decoding", + "[libopenshot][ffmpegreader][hardware][cuda-fallback]") +{ +#if !defined(__linux__) || !USE_HW_ACCEL + WARN("Skipping CUDA fallback test: requires Linux hardware decode support"); + return; +#else + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping CUDA fallback test: ffmpeg executable unavailable"); + return; + } + + std::stringstream fixture_path; + fixture_path << "libopenshot-cuda-fallback-" + << std::chrono::steady_clock::now().time_since_epoch().count() + << ".mp4"; + TemporaryFileGuard fixture_cleanup(fixture_path.str()); + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-f lavfi -i \"testsrc2=size=128x72:rate=30\" " + << "-t 1 -c:v libx264 -pix_fmt yuv420p " + << "\"" << fixture_path.str() << "\""; + + REQUIRE(std::system(command.str().c_str()) == 0); + + HardwareDecoderSettingsGuard settings_guard; + Settings::Instance()->HARDWARE_DECODER = 2; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + FFmpegReader reader(fixture_path.str(), DurationStrategy::VideoPreferred); + + forced_hw_device_create_calls = 0; + reader.hardware_device_create = &ForceHardwareDeviceCreationFailure; + + reader.Open(); + + REQUIRE(forced_hw_device_create_calls > 0); + REQUIRE(reader.info.has_video); + CHECK_FALSE(reader.HardwareDecodeSuccessful()); + + auto frame = reader.GetFrame(1); + REQUIRE(frame != nullptr); + REQUIRE(frame->has_image_data); + CHECK(frame->GetWidth() == 128); + CHECK(frame->GetHeight() == 72); + + // Hardware device creation is deliberately forced to fail. + // Software decoding must still return a valid frame. + if (!reader.HardwareDecodeSuccessful()) { + INFO("CUDA unavailable or unused; frame decoded successfully"); + } + + reader.Close(); +#endif +} + TEST_CASE( "VAAPI_H264_420_Reports_HardwareDecodeSuccess", "[libopenshot][ffmpegreader][hardware]" ) { #if !defined(__linux__) || !USE_HW_ACCEL