summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/video_core/CMakeLists.txt4
-rw-r--r--src/video_core/host1x/codecs/codec.cpp329
-rw-r--r--src/video_core/host1x/codecs/codec.h39
-rw-r--r--src/video_core/host1x/codecs/h264.cpp4
-rw-r--r--src/video_core/host1x/codecs/h264.h1
-rw-r--r--src/video_core/host1x/ffmpeg/ffmpeg.cpp419
-rw-r--r--src/video_core/host1x/ffmpeg/ffmpeg.h213
-rw-r--r--src/video_core/host1x/nvdec.cpp2
-rw-r--r--src/video_core/host1x/nvdec.h2
-rw-r--r--src/video_core/host1x/vic.cpp62
-rw-r--r--src/video_core/host1x/vic.h4
11 files changed, 705 insertions, 374 deletions
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index cf9266d54..b65b9f2a2 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -4,7 +4,7 @@
4add_subdirectory(host_shaders) 4add_subdirectory(host_shaders)
5 5
6if(LIBVA_FOUND) 6if(LIBVA_FOUND)
7 set_source_files_properties(host1x/codecs/codec.cpp 7 set_source_files_properties(host1x/ffmpeg/ffmpeg.cpp
8 PROPERTIES COMPILE_DEFINITIONS LIBVA_FOUND=1) 8 PROPERTIES COMPILE_DEFINITIONS LIBVA_FOUND=1)
9 list(APPEND FFmpeg_LIBRARIES ${LIBVA_LIBRARIES}) 9 list(APPEND FFmpeg_LIBRARIES ${LIBVA_LIBRARIES})
10endif() 10endif()
@@ -66,6 +66,8 @@ add_library(video_core STATIC
66 host1x/codecs/vp9.cpp 66 host1x/codecs/vp9.cpp
67 host1x/codecs/vp9.h 67 host1x/codecs/vp9.h
68 host1x/codecs/vp9_types.h 68 host1x/codecs/vp9_types.h
69 host1x/ffmpeg/ffmpeg.cpp
70 host1x/ffmpeg/ffmpeg.h
69 host1x/control.cpp 71 host1x/control.cpp
70 host1x/control.h 72 host1x/control.h
71 host1x/host1x.cpp 73 host1x/host1x.cpp
diff --git a/src/video_core/host1x/codecs/codec.cpp b/src/video_core/host1x/codecs/codec.cpp
index dbcf508e5..1030db681 100644
--- a/src/video_core/host1x/codecs/codec.cpp
+++ b/src/video_core/host1x/codecs/codec.cpp
@@ -1,11 +1,7 @@
1// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project 1// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
2// SPDX-License-Identifier: GPL-2.0-or-later 2// SPDX-License-Identifier: GPL-2.0-or-later
3 3
4#include <algorithm>
5#include <fstream>
6#include <vector>
7#include "common/assert.h" 4#include "common/assert.h"
8#include "common/scope_exit.h"
9#include "common/settings.h" 5#include "common/settings.h"
10#include "video_core/host1x/codecs/codec.h" 6#include "video_core/host1x/codecs/codec.h"
11#include "video_core/host1x/codecs/h264.h" 7#include "video_core/host1x/codecs/h264.h"
@@ -14,242 +10,17 @@
14#include "video_core/host1x/host1x.h" 10#include "video_core/host1x/host1x.h"
15#include "video_core/memory_manager.h" 11#include "video_core/memory_manager.h"
16 12
17extern "C" {
18#include <libavfilter/buffersink.h>
19#include <libavfilter/buffersrc.h>
20#include <libavutil/opt.h>
21#ifdef LIBVA_FOUND
22// for querying VAAPI driver information
23#include <libavutil/hwcontext_vaapi.h>
24#endif
25}
26
27namespace Tegra { 13namespace Tegra {
28namespace {
29constexpr AVPixelFormat PREFERRED_GPU_FMT = AV_PIX_FMT_NV12;
30constexpr AVPixelFormat PREFERRED_CPU_FMT = AV_PIX_FMT_YUV420P;
31constexpr std::array PREFERRED_GPU_DECODERS = {
32 AV_HWDEVICE_TYPE_CUDA,
33#ifdef _WIN32
34 AV_HWDEVICE_TYPE_D3D11VA,
35 AV_HWDEVICE_TYPE_DXVA2,
36#elif defined(__unix__)
37 AV_HWDEVICE_TYPE_VAAPI,
38 AV_HWDEVICE_TYPE_VDPAU,
39#endif
40 // last resort for Linux Flatpak (w/ NVIDIA)
41 AV_HWDEVICE_TYPE_VULKAN,
42};
43
44void AVPacketDeleter(AVPacket* ptr) {
45 av_packet_free(&ptr);
46}
47
48using AVPacketPtr = std::unique_ptr<AVPacket, decltype(&AVPacketDeleter)>;
49
50AVPixelFormat GetGpuFormat(AVCodecContext* av_codec_ctx, const AVPixelFormat* pix_fmts) {
51 for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) {
52 if (*p == av_codec_ctx->pix_fmt) {
53 return av_codec_ctx->pix_fmt;
54 }
55 }
56 LOG_INFO(Service_NVDRV, "Could not find compatible GPU AV format, falling back to CPU");
57 av_buffer_unref(&av_codec_ctx->hw_device_ctx);
58 av_codec_ctx->pix_fmt = PREFERRED_CPU_FMT;
59 return PREFERRED_CPU_FMT;
60}
61
62// List all the currently available hwcontext in ffmpeg
63std::vector<AVHWDeviceType> ListSupportedContexts() {
64 std::vector<AVHWDeviceType> contexts{};
65 AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE;
66 do {
67 current_device_type = av_hwdevice_iterate_types(current_device_type);
68 contexts.push_back(current_device_type);
69 } while (current_device_type != AV_HWDEVICE_TYPE_NONE);
70 return contexts;
71}
72
73} // namespace
74
75void AVFrameDeleter(AVFrame* ptr) {
76 av_frame_free(&ptr);
77}
78 14
79Codec::Codec(Host1x::Host1x& host1x_, const Host1x::NvdecCommon::NvdecRegisters& regs) 15Codec::Codec(Host1x::Host1x& host1x_, const Host1x::NvdecCommon::NvdecRegisters& regs)
80 : host1x(host1x_), state{regs}, h264_decoder(std::make_unique<Decoder::H264>(host1x)), 16 : host1x(host1x_), state{regs}, h264_decoder(std::make_unique<Decoder::H264>(host1x)),
81 vp8_decoder(std::make_unique<Decoder::VP8>(host1x)), 17 vp8_decoder(std::make_unique<Decoder::VP8>(host1x)),
82 vp9_decoder(std::make_unique<Decoder::VP9>(host1x)) {} 18 vp9_decoder(std::make_unique<Decoder::VP9>(host1x)) {}
83 19
84Codec::~Codec() { 20Codec::~Codec() = default;
85 if (!initialized) {
86 return;
87 }
88 // Free libav memory
89 avcodec_free_context(&av_codec_ctx);
90 av_buffer_unref(&av_gpu_decoder);
91
92 if (filters_initialized) {
93 avfilter_graph_free(&av_filter_graph);
94 }
95}
96
97bool Codec::CreateGpuAvDevice() {
98 static constexpr auto HW_CONFIG_METHOD = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX;
99 static const auto supported_contexts = ListSupportedContexts();
100 for (const auto& type : PREFERRED_GPU_DECODERS) {
101 if (std::none_of(supported_contexts.begin(), supported_contexts.end(),
102 [&type](const auto& context) { return context == type; })) {
103 LOG_DEBUG(Service_NVDRV, "{} explicitly unsupported", av_hwdevice_get_type_name(type));
104 continue;
105 }
106 // Avoid memory leak from not cleaning up after av_hwdevice_ctx_create
107 av_buffer_unref(&av_gpu_decoder);
108 const int hwdevice_res = av_hwdevice_ctx_create(&av_gpu_decoder, type, nullptr, nullptr, 0);
109 if (hwdevice_res < 0) {
110 LOG_DEBUG(Service_NVDRV, "{} av_hwdevice_ctx_create failed {}",
111 av_hwdevice_get_type_name(type), hwdevice_res);
112 continue;
113 }
114#ifdef LIBVA_FOUND
115 if (type == AV_HWDEVICE_TYPE_VAAPI) {
116 // we need to determine if this is an impersonated VAAPI driver
117 AVHWDeviceContext* hwctx =
118 static_cast<AVHWDeviceContext*>(static_cast<void*>(av_gpu_decoder->data));
119 AVVAAPIDeviceContext* vactx = static_cast<AVVAAPIDeviceContext*>(hwctx->hwctx);
120 const char* vendor_name = vaQueryVendorString(vactx->display);
121 if (strstr(vendor_name, "VDPAU backend")) {
122 // VDPAU impersonated VAAPI impl's are super buggy, we need to skip them
123 LOG_DEBUG(Service_NVDRV, "Skipping vdapu impersonated VAAPI driver");
124 continue;
125 } else {
126 // according to some user testing, certain vaapi driver (Intel?) could be buggy
127 // so let's log the driver name which may help the developers/supporters
128 LOG_DEBUG(Service_NVDRV, "Using VAAPI driver: {}", vendor_name);
129 }
130 }
131#endif
132 for (int i = 0;; i++) {
133 const AVCodecHWConfig* config = avcodec_get_hw_config(av_codec, i);
134 if (!config) {
135 LOG_DEBUG(Service_NVDRV, "{} decoder does not support device type {}.",
136 av_codec->name, av_hwdevice_get_type_name(type));
137 break;
138 }
139 if ((config->methods & HW_CONFIG_METHOD) != 0 && config->device_type == type) {
140 LOG_INFO(Service_NVDRV, "Using {} GPU decoder", av_hwdevice_get_type_name(type));
141 av_codec_ctx->pix_fmt = config->pix_fmt;
142 return true;
143 }
144 }
145 }
146 return false;
147}
148
149void Codec::InitializeAvCodecContext() {
150 av_codec_ctx = avcodec_alloc_context3(av_codec);
151 av_opt_set(av_codec_ctx->priv_data, "tune", "zerolatency", 0);
152 av_codec_ctx->thread_count = 0;
153 av_codec_ctx->thread_type &= ~FF_THREAD_FRAME;
154}
155
156void Codec::InitializeGpuDecoder() {
157 if (!CreateGpuAvDevice()) {
158 av_buffer_unref(&av_gpu_decoder);
159 return;
160 }
161 auto* hw_device_ctx = av_buffer_ref(av_gpu_decoder);
162 ASSERT_MSG(hw_device_ctx, "av_buffer_ref failed");
163 av_codec_ctx->hw_device_ctx = hw_device_ctx;
164 av_codec_ctx->get_format = GetGpuFormat;
165}
166
167void Codec::InitializeAvFilters(AVFrame* frame) {
168 const AVFilter* buffer_src = avfilter_get_by_name("buffer");
169 const AVFilter* buffer_sink = avfilter_get_by_name("buffersink");
170 AVFilterInOut* inputs = avfilter_inout_alloc();
171 AVFilterInOut* outputs = avfilter_inout_alloc();
172 SCOPE_EXIT({
173 avfilter_inout_free(&inputs);
174 avfilter_inout_free(&outputs);
175 });
176
177 // Don't know how to get the accurate time_base but it doesn't matter for yadif filter
178 // so just use 1/1 to make buffer filter happy
179 std::string args = fmt::format("video_size={}x{}:pix_fmt={}:time_base=1/1", frame->width,
180 frame->height, frame->format);
181
182 av_filter_graph = avfilter_graph_alloc();
183 int ret = avfilter_graph_create_filter(&av_filter_src_ctx, buffer_src, "in", args.c_str(),
184 nullptr, av_filter_graph);
185 if (ret < 0) {
186 LOG_ERROR(Service_NVDRV, "avfilter_graph_create_filter source error: {}", ret);
187 return;
188 }
189
190 ret = avfilter_graph_create_filter(&av_filter_sink_ctx, buffer_sink, "out", nullptr, nullptr,
191 av_filter_graph);
192 if (ret < 0) {
193 LOG_ERROR(Service_NVDRV, "avfilter_graph_create_filter sink error: {}", ret);
194 return;
195 }
196
197 inputs->name = av_strdup("out");
198 inputs->filter_ctx = av_filter_sink_ctx;
199 inputs->pad_idx = 0;
200 inputs->next = nullptr;
201
202 outputs->name = av_strdup("in");
203 outputs->filter_ctx = av_filter_src_ctx;
204 outputs->pad_idx = 0;
205 outputs->next = nullptr;
206
207 const char* description = "yadif=1:-1:0";
208 ret = avfilter_graph_parse_ptr(av_filter_graph, description, &inputs, &outputs, nullptr);
209 if (ret < 0) {
210 LOG_ERROR(Service_NVDRV, "avfilter_graph_parse_ptr error: {}", ret);
211 return;
212 }
213
214 ret = avfilter_graph_config(av_filter_graph, nullptr);
215 if (ret < 0) {
216 LOG_ERROR(Service_NVDRV, "avfilter_graph_config error: {}", ret);
217 return;
218 }
219
220 filters_initialized = true;
221}
222 21
223void Codec::Initialize() { 22void Codec::Initialize() {
224 const AVCodecID codec = [&] { 23 initialized = decode_api.Initialize(current_codec);
225 switch (current_codec) {
226 case Host1x::NvdecCommon::VideoCodec::H264:
227 return AV_CODEC_ID_H264;
228 case Host1x::NvdecCommon::VideoCodec::VP8:
229 return AV_CODEC_ID_VP8;
230 case Host1x::NvdecCommon::VideoCodec::VP9:
231 return AV_CODEC_ID_VP9;
232 default:
233 UNIMPLEMENTED_MSG("Unknown codec {}", current_codec);
234 return AV_CODEC_ID_NONE;
235 }
236 }();
237 av_codec = avcodec_find_decoder(codec);
238
239 InitializeAvCodecContext();
240 if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) {
241 InitializeGpuDecoder();
242 }
243 if (const int res = avcodec_open2(av_codec_ctx, av_codec, nullptr); res < 0) {
244 LOG_ERROR(Service_NVDRV, "avcodec_open2() Failed with result {}", res);
245 avcodec_free_context(&av_codec_ctx);
246 av_buffer_unref(&av_gpu_decoder);
247 return;
248 }
249 if (!av_codec_ctx->hw_device_ctx) {
250 LOG_INFO(Service_NVDRV, "Using FFmpeg software decoding");
251 }
252 initialized = true;
253} 24}
254 25
255void Codec::SetTargetCodec(Host1x::NvdecCommon::VideoCodec codec) { 26void Codec::SetTargetCodec(Host1x::NvdecCommon::VideoCodec codec) {
@@ -264,14 +35,18 @@ void Codec::Decode() {
264 if (is_first_frame) { 35 if (is_first_frame) {
265 Initialize(); 36 Initialize();
266 } 37 }
38
267 if (!initialized) { 39 if (!initialized) {
268 return; 40 return;
269 } 41 }
42
43 // Assemble bitstream.
270 bool vp9_hidden_frame = false; 44 bool vp9_hidden_frame = false;
271 const auto& frame_data = [&]() { 45 size_t configuration_size = 0;
46 const auto packet_data = [&]() {
272 switch (current_codec) { 47 switch (current_codec) {
273 case Tegra::Host1x::NvdecCommon::VideoCodec::H264: 48 case Tegra::Host1x::NvdecCommon::VideoCodec::H264:
274 return h264_decoder->ComposeFrame(state, is_first_frame); 49 return h264_decoder->ComposeFrame(state, &configuration_size, is_first_frame);
275 case Tegra::Host1x::NvdecCommon::VideoCodec::VP8: 50 case Tegra::Host1x::NvdecCommon::VideoCodec::VP8:
276 return vp8_decoder->ComposeFrame(state); 51 return vp8_decoder->ComposeFrame(state);
277 case Tegra::Host1x::NvdecCommon::VideoCodec::VP9: 52 case Tegra::Host1x::NvdecCommon::VideoCodec::VP9:
@@ -283,89 +58,35 @@ void Codec::Decode() {
283 return std::span<const u8>{}; 58 return std::span<const u8>{};
284 } 59 }
285 }(); 60 }();
286 AVPacketPtr packet{av_packet_alloc(), AVPacketDeleter}; 61
287 if (!packet) { 62 // Send assembled bitstream to decoder.
288 LOG_ERROR(Service_NVDRV, "av_packet_alloc failed"); 63 if (!decode_api.SendPacket(packet_data, configuration_size)) {
289 return;
290 }
291 packet->data = const_cast<u8*>(frame_data.data());
292 packet->size = static_cast<s32>(frame_data.size());
293 if (const int res = avcodec_send_packet(av_codec_ctx, packet.get()); res != 0) {
294 LOG_DEBUG(Service_NVDRV, "avcodec_send_packet error {}", res);
295 return; 64 return;
296 } 65 }
297 // Only receive/store visible frames 66
67 // Only receive/store visible frames.
298 if (vp9_hidden_frame) { 68 if (vp9_hidden_frame) {
299 return; 69 return;
300 } 70 }
301 AVFramePtr initial_frame{av_frame_alloc(), AVFrameDeleter};
302 AVFramePtr final_frame{nullptr, AVFrameDeleter};
303 ASSERT_MSG(initial_frame, "av_frame_alloc initial_frame failed");
304 if (const int ret = avcodec_receive_frame(av_codec_ctx, initial_frame.get()); ret) {
305 LOG_DEBUG(Service_NVDRV, "avcodec_receive_frame error {}", ret);
306 return;
307 }
308 if (initial_frame->width == 0 || initial_frame->height == 0) {
309 LOG_WARNING(Service_NVDRV, "Zero width or height in frame");
310 return;
311 }
312 bool is_interlaced = initial_frame->interlaced_frame != 0;
313 if (av_codec_ctx->hw_device_ctx) {
314 final_frame = AVFramePtr{av_frame_alloc(), AVFrameDeleter};
315 ASSERT_MSG(final_frame, "av_frame_alloc final_frame failed");
316 // Can't use AV_PIX_FMT_YUV420P and share code with software decoding in vic.cpp
317 // because Intel drivers crash unless using AV_PIX_FMT_NV12
318 final_frame->format = PREFERRED_GPU_FMT;
319 const int ret = av_hwframe_transfer_data(final_frame.get(), initial_frame.get(), 0);
320 ASSERT_MSG(!ret, "av_hwframe_transfer_data error {}", ret);
321 } else {
322 final_frame = std::move(initial_frame);
323 }
324 if (final_frame->format != PREFERRED_CPU_FMT && final_frame->format != PREFERRED_GPU_FMT) {
325 UNIMPLEMENTED_MSG("Unexpected video format: {}", final_frame->format);
326 return;
327 }
328 if (!is_interlaced) {
329 av_frames.push(std::move(final_frame));
330 } else {
331 if (!filters_initialized) {
332 InitializeAvFilters(final_frame.get());
333 }
334 if (const int ret = av_buffersrc_add_frame_flags(av_filter_src_ctx, final_frame.get(),
335 AV_BUFFERSRC_FLAG_KEEP_REF);
336 ret) {
337 LOG_DEBUG(Service_NVDRV, "av_buffersrc_add_frame_flags error {}", ret);
338 return;
339 }
340 while (true) {
341 auto filter_frame = AVFramePtr{av_frame_alloc(), AVFrameDeleter};
342 71
343 int ret = av_buffersink_get_frame(av_filter_sink_ctx, filter_frame.get()); 72 // Receive output frames from decoder.
73 decode_api.ReceiveFrames(frames);
344 74
345 if (ret == AVERROR(EAGAIN) || ret == AVERROR(AVERROR_EOF)) 75 while (frames.size() > 10) {
346 break; 76 LOG_DEBUG(HW_GPU, "ReceiveFrames overflow, dropped frame");
347 if (ret < 0) { 77 frames.pop();
348 LOG_DEBUG(Service_NVDRV, "av_buffersink_get_frame error {}", ret);
349 return;
350 }
351
352 av_frames.push(std::move(filter_frame));
353 }
354 }
355 while (av_frames.size() > 10) {
356 LOG_TRACE(Service_NVDRV, "av_frames.push overflow dropped frame");
357 av_frames.pop();
358 } 78 }
359} 79}
360 80
361AVFramePtr Codec::GetCurrentFrame() { 81std::unique_ptr<FFmpeg::Frame> Codec::GetCurrentFrame() {
362 // Sometimes VIC will request more frames than have been decoded. 82 // Sometimes VIC will request more frames than have been decoded.
363 // in this case, return a nullptr and don't overwrite previous frame data 83 // in this case, return a blank frame and don't overwrite previous data.
364 if (av_frames.empty()) { 84 if (frames.empty()) {
365 return AVFramePtr{nullptr, AVFrameDeleter}; 85 return {};
366 } 86 }
367 AVFramePtr frame = std::move(av_frames.front()); 87
368 av_frames.pop(); 88 auto frame = std::move(frames.front());
89 frames.pop();
369 return frame; 90 return frame;
370} 91}
371 92
diff --git a/src/video_core/host1x/codecs/codec.h b/src/video_core/host1x/codecs/codec.h
index 06fe00a4b..f700ae129 100644
--- a/src/video_core/host1x/codecs/codec.h
+++ b/src/video_core/host1x/codecs/codec.h
@@ -4,28 +4,15 @@
4#pragma once 4#pragma once
5 5
6#include <memory> 6#include <memory>
7#include <optional>
7#include <string_view> 8#include <string_view>
8#include <queue> 9#include <queue>
9#include "common/common_types.h" 10#include "common/common_types.h"
11#include "video_core/host1x/ffmpeg/ffmpeg.h"
10#include "video_core/host1x/nvdec_common.h" 12#include "video_core/host1x/nvdec_common.h"
11 13
12extern "C" {
13#if defined(__GNUC__) || defined(__clang__)
14#pragma GCC diagnostic push
15#pragma GCC diagnostic ignored "-Wconversion"
16#endif
17#include <libavcodec/avcodec.h>
18#include <libavfilter/avfilter.h>
19#if defined(__GNUC__) || defined(__clang__)
20#pragma GCC diagnostic pop
21#endif
22}
23
24namespace Tegra { 14namespace Tegra {
25 15
26void AVFrameDeleter(AVFrame* ptr);
27using AVFramePtr = std::unique_ptr<AVFrame, decltype(&AVFrameDeleter)>;
28
29namespace Decoder { 16namespace Decoder {
30class H264; 17class H264;
31class VP8; 18class VP8;
@@ -51,7 +38,7 @@ public:
51 void Decode(); 38 void Decode();
52 39
53 /// Returns next decoded frame 40 /// Returns next decoded frame
54 [[nodiscard]] AVFramePtr GetCurrentFrame(); 41 [[nodiscard]] std::unique_ptr<FFmpeg::Frame> GetCurrentFrame();
55 42
56 /// Returns the value of current_codec 43 /// Returns the value of current_codec
57 [[nodiscard]] Host1x::NvdecCommon::VideoCodec GetCurrentCodec() const; 44 [[nodiscard]] Host1x::NvdecCommon::VideoCodec GetCurrentCodec() const;
@@ -60,25 +47,9 @@ public:
60 [[nodiscard]] std::string_view GetCurrentCodecName() const; 47 [[nodiscard]] std::string_view GetCurrentCodecName() const;
61 48
62private: 49private:
63 void InitializeAvCodecContext();
64
65 void InitializeAvFilters(AVFrame* frame);
66
67 void InitializeGpuDecoder();
68
69 bool CreateGpuAvDevice();
70
71 bool initialized{}; 50 bool initialized{};
72 bool filters_initialized{};
73 Host1x::NvdecCommon::VideoCodec current_codec{Host1x::NvdecCommon::VideoCodec::None}; 51 Host1x::NvdecCommon::VideoCodec current_codec{Host1x::NvdecCommon::VideoCodec::None};
74 52 FFmpeg::DecodeApi decode_api;
75 const AVCodec* av_codec{nullptr};
76 AVCodecContext* av_codec_ctx{nullptr};
77 AVBufferRef* av_gpu_decoder{nullptr};
78
79 AVFilterContext* av_filter_src_ctx{nullptr};
80 AVFilterContext* av_filter_sink_ctx{nullptr};
81 AVFilterGraph* av_filter_graph{nullptr};
82 53
83 Host1x::Host1x& host1x; 54 Host1x::Host1x& host1x;
84 const Host1x::NvdecCommon::NvdecRegisters& state; 55 const Host1x::NvdecCommon::NvdecRegisters& state;
@@ -86,7 +57,7 @@ private:
86 std::unique_ptr<Decoder::VP8> vp8_decoder; 57 std::unique_ptr<Decoder::VP8> vp8_decoder;
87 std::unique_ptr<Decoder::VP9> vp9_decoder; 58 std::unique_ptr<Decoder::VP9> vp9_decoder;
88 59
89 std::queue<AVFramePtr> av_frames{}; 60 std::queue<std::unique_ptr<FFmpeg::Frame>> frames{};
90}; 61};
91 62
92} // namespace Tegra 63} // namespace Tegra
diff --git a/src/video_core/host1x/codecs/h264.cpp b/src/video_core/host1x/codecs/h264.cpp
index ece79b1e2..309a7f1d5 100644
--- a/src/video_core/host1x/codecs/h264.cpp
+++ b/src/video_core/host1x/codecs/h264.cpp
@@ -30,7 +30,7 @@ H264::H264(Host1x::Host1x& host1x_) : host1x{host1x_} {}
30H264::~H264() = default; 30H264::~H264() = default;
31 31
32std::span<const u8> H264::ComposeFrame(const Host1x::NvdecCommon::NvdecRegisters& state, 32std::span<const u8> H264::ComposeFrame(const Host1x::NvdecCommon::NvdecRegisters& state,
33 bool is_first_frame) { 33 size_t* out_configuration_size, bool is_first_frame) {
34 H264DecoderContext context; 34 H264DecoderContext context;
35 host1x.MemoryManager().ReadBlock(state.picture_info_offset, &context, 35 host1x.MemoryManager().ReadBlock(state.picture_info_offset, &context,
36 sizeof(H264DecoderContext)); 36 sizeof(H264DecoderContext));
@@ -39,6 +39,7 @@ std::span<const u8> H264::ComposeFrame(const Host1x::NvdecCommon::NvdecRegisters
39 if (!is_first_frame && frame_number != 0) { 39 if (!is_first_frame && frame_number != 0) {
40 frame.resize_destructive(context.stream_len); 40 frame.resize_destructive(context.stream_len);
41 host1x.MemoryManager().ReadBlock(state.frame_bitstream_offset, frame.data(), frame.size()); 41 host1x.MemoryManager().ReadBlock(state.frame_bitstream_offset, frame.data(), frame.size());
42 *out_configuration_size = 0;
42 return frame; 43 return frame;
43 } 44 }
44 45
@@ -157,6 +158,7 @@ std::span<const u8> H264::ComposeFrame(const Host1x::NvdecCommon::NvdecRegisters
157 frame.resize(encoded_header.size() + context.stream_len); 158 frame.resize(encoded_header.size() + context.stream_len);
158 std::memcpy(frame.data(), encoded_header.data(), encoded_header.size()); 159 std::memcpy(frame.data(), encoded_header.data(), encoded_header.size());
159 160
161 *out_configuration_size = encoded_header.size();
160 host1x.MemoryManager().ReadBlock(state.frame_bitstream_offset, 162 host1x.MemoryManager().ReadBlock(state.frame_bitstream_offset,
161 frame.data() + encoded_header.size(), context.stream_len); 163 frame.data() + encoded_header.size(), context.stream_len);
162 164
diff --git a/src/video_core/host1x/codecs/h264.h b/src/video_core/host1x/codecs/h264.h
index d6b556322..1deaf4632 100644
--- a/src/video_core/host1x/codecs/h264.h
+++ b/src/video_core/host1x/codecs/h264.h
@@ -67,6 +67,7 @@ public:
67 67
68 /// Compose the H264 frame for FFmpeg decoding 68 /// Compose the H264 frame for FFmpeg decoding
69 [[nodiscard]] std::span<const u8> ComposeFrame(const Host1x::NvdecCommon::NvdecRegisters& state, 69 [[nodiscard]] std::span<const u8> ComposeFrame(const Host1x::NvdecCommon::NvdecRegisters& state,
70 size_t* out_configuration_size,
70 bool is_first_frame = false); 71 bool is_first_frame = false);
71 72
72private: 73private:
diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.cpp b/src/video_core/host1x/ffmpeg/ffmpeg.cpp
new file mode 100644
index 000000000..dcd07e6d2
--- /dev/null
+++ b/src/video_core/host1x/ffmpeg/ffmpeg.cpp
@@ -0,0 +1,419 @@
1// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
2// SPDX-License-Identifier: GPL-2.0-or-later
3
4#include "common/assert.h"
5#include "common/logging/log.h"
6#include "common/scope_exit.h"
7#include "common/settings.h"
8#include "video_core/host1x/ffmpeg/ffmpeg.h"
9
10extern "C" {
11#ifdef LIBVA_FOUND
12// for querying VAAPI driver information
13#include <libavutil/hwcontext_vaapi.h>
14#endif
15}
16
17namespace FFmpeg {
18
19namespace {
20
21constexpr AVPixelFormat PreferredGpuFormat = AV_PIX_FMT_NV12;
22constexpr AVPixelFormat PreferredCpuFormat = AV_PIX_FMT_YUV420P;
23constexpr std::array PreferredGpuDecoders = {
24 AV_HWDEVICE_TYPE_CUDA,
25#ifdef _WIN32
26 AV_HWDEVICE_TYPE_D3D11VA,
27 AV_HWDEVICE_TYPE_DXVA2,
28#elif defined(__unix__)
29 AV_HWDEVICE_TYPE_VAAPI,
30 AV_HWDEVICE_TYPE_VDPAU,
31#endif
32 // last resort for Linux Flatpak (w/ NVIDIA)
33 AV_HWDEVICE_TYPE_VULKAN,
34};
35
36AVPixelFormat GetGpuFormat(AVCodecContext* codec_context, const AVPixelFormat* pix_fmts) {
37 for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p) {
38 if (*p == codec_context->pix_fmt) {
39 return codec_context->pix_fmt;
40 }
41 }
42
43 LOG_INFO(HW_GPU, "Could not find compatible GPU AV format, falling back to CPU");
44 av_buffer_unref(&codec_context->hw_device_ctx);
45
46 codec_context->pix_fmt = PreferredCpuFormat;
47 return codec_context->pix_fmt;
48}
49
50std::string AVError(int errnum) {
51 char errbuf[AV_ERROR_MAX_STRING_SIZE] = {};
52 av_make_error_string(errbuf, sizeof(errbuf) - 1, errnum);
53 return errbuf;
54}
55
56} // namespace
57
58Packet::Packet(std::span<const u8> data) {
59 m_packet = av_packet_alloc();
60 m_packet->data = const_cast<u8*>(data.data());
61 m_packet->size = static_cast<s32>(data.size());
62}
63
64Packet::~Packet() {
65 av_packet_free(&m_packet);
66}
67
68Frame::Frame() {
69 m_frame = av_frame_alloc();
70}
71
72Frame::~Frame() {
73 av_frame_free(&m_frame);
74}
75
76Decoder::Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
77 const AVCodecID av_codec = [&] {
78 switch (codec) {
79 case Tegra::Host1x::NvdecCommon::VideoCodec::H264:
80 return AV_CODEC_ID_H264;
81 case Tegra::Host1x::NvdecCommon::VideoCodec::VP8:
82 return AV_CODEC_ID_VP8;
83 case Tegra::Host1x::NvdecCommon::VideoCodec::VP9:
84 return AV_CODEC_ID_VP9;
85 default:
86 UNIMPLEMENTED_MSG("Unknown codec {}", codec);
87 return AV_CODEC_ID_NONE;
88 }
89 }();
90
91 m_codec = avcodec_find_decoder(av_codec);
92}
93
94bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceType type) const {
95 for (int i = 0;; i++) {
96 const AVCodecHWConfig* config = avcodec_get_hw_config(m_codec, i);
97 if (!config) {
98 LOG_DEBUG(HW_GPU, "{} decoder does not support device type {}", m_codec->name,
99 av_hwdevice_get_type_name(type));
100 break;
101 }
102 if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0 &&
103 config->device_type == type) {
104 LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type));
105 *out_pix_fmt = config->pix_fmt;
106 return true;
107 }
108 }
109
110 return false;
111}
112
113std::vector<AVHWDeviceType> HardwareContext::GetSupportedDeviceTypes() {
114 std::vector<AVHWDeviceType> types;
115 AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE;
116
117 while (true) {
118 current_device_type = av_hwdevice_iterate_types(current_device_type);
119 if (current_device_type == AV_HWDEVICE_TYPE_NONE) {
120 return types;
121 }
122
123 types.push_back(current_device_type);
124 }
125}
126
127HardwareContext::~HardwareContext() {
128 av_buffer_unref(&m_gpu_decoder);
129}
130
131bool HardwareContext::InitializeForDecoder(DecoderContext& decoder_context,
132 const Decoder& decoder) {
133 const auto supported_types = GetSupportedDeviceTypes();
134 for (const auto type : PreferredGpuDecoders) {
135 AVPixelFormat hw_pix_fmt;
136
137 if (std::ranges::find(supported_types, type) == supported_types.end()) {
138 LOG_DEBUG(HW_GPU, "{} explicitly unsupported", av_hwdevice_get_type_name(type));
139 continue;
140 }
141
142 if (!this->InitializeWithType(type)) {
143 continue;
144 }
145
146 if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) {
147 decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt);
148 return true;
149 }
150 }
151
152 return false;
153}
154
155bool HardwareContext::InitializeWithType(AVHWDeviceType type) {
156 av_buffer_unref(&m_gpu_decoder);
157
158 if (const int ret = av_hwdevice_ctx_create(&m_gpu_decoder, type, nullptr, nullptr, 0);
159 ret < 0) {
160 LOG_DEBUG(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type),
161 AVError(ret));
162 return false;
163 }
164
165#ifdef LIBVA_FOUND
166 if (type == AV_HWDEVICE_TYPE_VAAPI) {
167 // We need to determine if this is an impersonated VAAPI driver.
168 auto* hwctx = reinterpret_cast<AVHWDeviceContext*>(m_gpu_decoder->data);
169 auto* vactx = static_cast<AVVAAPIDeviceContext*>(hwctx->hwctx);
170 const char* vendor_name = vaQueryVendorString(vactx->display);
171 if (strstr(vendor_name, "VDPAU backend")) {
172 // VDPAU impersonated VAAPI impls are super buggy, we need to skip them.
173 LOG_DEBUG(HW_GPU, "Skipping VDPAU impersonated VAAPI driver");
174 return false;
175 } else {
176 // According to some user testing, certain VAAPI drivers (Intel?) could be buggy.
177 // Log the driver name just in case.
178 LOG_DEBUG(HW_GPU, "Using VAAPI driver: {}", vendor_name);
179 }
180 }
181#endif
182
183 return true;
184}
185
186DecoderContext::DecoderContext(const Decoder& decoder) {
187 m_codec_context = avcodec_alloc_context3(decoder.GetCodec());
188 av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0);
189 m_codec_context->thread_count = 0;
190 m_codec_context->thread_type &= ~FF_THREAD_FRAME;
191}
192
193DecoderContext::~DecoderContext() {
194 av_buffer_unref(&m_codec_context->hw_device_ctx);
195 avcodec_free_context(&m_codec_context);
196}
197
198void DecoderContext::InitializeHardwareDecoder(const HardwareContext& context,
199 AVPixelFormat hw_pix_fmt) {
200 m_codec_context->hw_device_ctx = av_buffer_ref(context.GetBufferRef());
201 m_codec_context->get_format = GetGpuFormat;
202 m_codec_context->pix_fmt = hw_pix_fmt;
203}
204
205bool DecoderContext::OpenContext(const Decoder& decoder) {
206 if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) {
207 LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret));
208 return false;
209 }
210
211 if (!m_codec_context->hw_device_ctx) {
212 LOG_INFO(HW_GPU, "Using FFmpeg software decoding");
213 }
214
215 return true;
216}
217
218bool DecoderContext::SendPacket(const Packet& packet) {
219 if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0) {
220 LOG_ERROR(HW_GPU, "avcodec_send_packet error: {}", AVError(ret));
221 return false;
222 }
223
224 return true;
225}
226
227std::unique_ptr<Frame> DecoderContext::ReceiveFrame(bool* out_is_interlaced) {
228 auto dst_frame = std::make_unique<Frame>();
229
230 const auto ReceiveImpl = [&](AVFrame* frame) {
231 if (const int ret = avcodec_receive_frame(m_codec_context, frame); ret < 0) {
232 LOG_ERROR(HW_GPU, "avcodec_receive_frame error: {}", AVError(ret));
233 return false;
234 }
235
236 *out_is_interlaced = frame->interlaced_frame != 0;
237 return true;
238 };
239
240 if (m_codec_context->hw_device_ctx) {
241 // If we have a hardware context, make a separate frame here to receive the
242 // hardware result before sending it to the output.
243 Frame intermediate_frame;
244
245 if (!ReceiveImpl(intermediate_frame.GetFrame())) {
246 return {};
247 }
248
249 dst_frame->SetFormat(PreferredGpuFormat);
250 if (const int ret =
251 av_hwframe_transfer_data(dst_frame->GetFrame(), intermediate_frame.GetFrame(), 0);
252 ret < 0) {
253 LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret));
254 return {};
255 }
256 } else {
257 // Otherwise, decode the frame as normal.
258 if (!ReceiveImpl(dst_frame->GetFrame())) {
259 return {};
260 }
261 }
262
263 return dst_frame;
264}
265
266DeinterlaceFilter::DeinterlaceFilter(const Frame& frame) {
267 const AVFilter* buffer_src = avfilter_get_by_name("buffer");
268 const AVFilter* buffer_sink = avfilter_get_by_name("buffersink");
269 AVFilterInOut* inputs = avfilter_inout_alloc();
270 AVFilterInOut* outputs = avfilter_inout_alloc();
271 SCOPE_EXIT({
272 avfilter_inout_free(&inputs);
273 avfilter_inout_free(&outputs);
274 });
275
276 // Don't know how to get the accurate time_base but it doesn't matter for yadif filter
277 // so just use 1/1 to make buffer filter happy
278 std::string args = fmt::format("video_size={}x{}:pix_fmt={}:time_base=1/1", frame.GetWidth(),
279 frame.GetHeight(), static_cast<int>(frame.GetPixelFormat()));
280
281 m_filter_graph = avfilter_graph_alloc();
282 int ret = avfilter_graph_create_filter(&m_source_context, buffer_src, "in", args.c_str(),
283 nullptr, m_filter_graph);
284 if (ret < 0) {
285 LOG_ERROR(HW_GPU, "avfilter_graph_create_filter source error: {}", AVError(ret));
286 return;
287 }
288
289 ret = avfilter_graph_create_filter(&m_sink_context, buffer_sink, "out", nullptr, nullptr,
290 m_filter_graph);
291 if (ret < 0) {
292 LOG_ERROR(HW_GPU, "avfilter_graph_create_filter sink error: {}", AVError(ret));
293 return;
294 }
295
296 inputs->name = av_strdup("out");
297 inputs->filter_ctx = m_sink_context;
298 inputs->pad_idx = 0;
299 inputs->next = nullptr;
300
301 outputs->name = av_strdup("in");
302 outputs->filter_ctx = m_source_context;
303 outputs->pad_idx = 0;
304 outputs->next = nullptr;
305
306 const char* description = "yadif=1:-1:0";
307 ret = avfilter_graph_parse_ptr(m_filter_graph, description, &inputs, &outputs, nullptr);
308 if (ret < 0) {
309 LOG_ERROR(HW_GPU, "avfilter_graph_parse_ptr error: {}", AVError(ret));
310 return;
311 }
312
313 ret = avfilter_graph_config(m_filter_graph, nullptr);
314 if (ret < 0) {
315 LOG_ERROR(HW_GPU, "avfilter_graph_config error: {}", AVError(ret));
316 return;
317 }
318
319 m_initialized = true;
320}
321
322bool DeinterlaceFilter::AddSourceFrame(const Frame& frame) {
323 if (const int ret = av_buffersrc_add_frame_flags(m_source_context, frame.GetFrame(),
324 AV_BUFFERSRC_FLAG_KEEP_REF);
325 ret < 0) {
326 LOG_ERROR(HW_GPU, "av_buffersrc_add_frame_flags error: {}", AVError(ret));
327 return false;
328 }
329
330 return true;
331}
332
333std::unique_ptr<Frame> DeinterlaceFilter::DrainSinkFrame() {
334 auto dst_frame = std::make_unique<Frame>();
335 const int ret = av_buffersink_get_frame(m_sink_context, dst_frame->GetFrame());
336
337 if (ret == AVERROR(EAGAIN) || ret == AVERROR(AVERROR_EOF)) {
338 return {};
339 }
340
341 if (ret < 0) {
342 LOG_ERROR(HW_GPU, "av_buffersink_get_frame error: {}", AVError(ret));
343 return {};
344 }
345
346 return dst_frame;
347}
348
349DeinterlaceFilter::~DeinterlaceFilter() {
350 avfilter_graph_free(&m_filter_graph);
351}
352
353void DecodeApi::Reset() {
354 m_deinterlace_filter.reset();
355 m_hardware_context.reset();
356 m_decoder_context.reset();
357 m_decoder.reset();
358}
359
360bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
361 this->Reset();
362 m_decoder.emplace(codec);
363 m_decoder_context.emplace(*m_decoder);
364
365 // Enable GPU decoding if requested.
366 if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) {
367 m_hardware_context.emplace();
368 m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder);
369 }
370
371 // Open the decoder context.
372 if (!m_decoder_context->OpenContext(*m_decoder)) {
373 this->Reset();
374 return false;
375 }
376
377 return true;
378}
379
380bool DecodeApi::SendPacket(std::span<const u8> packet_data, size_t configuration_size) {
381 FFmpeg::Packet packet(packet_data);
382 return m_decoder_context->SendPacket(packet);
383}
384
385void DecodeApi::ReceiveFrames(std::queue<std::unique_ptr<Frame>>& frame_queue) {
386 // Receive raw frame from decoder.
387 bool is_interlaced;
388 auto frame = m_decoder_context->ReceiveFrame(&is_interlaced);
389 if (!frame) {
390 return;
391 }
392
393 if (!is_interlaced) {
394 // If the frame is not interlaced, we can pend it now.
395 frame_queue.push(std::move(frame));
396 } else {
397 // Create the deinterlacer if needed.
398 if (!m_deinterlace_filter) {
399 m_deinterlace_filter.emplace(*frame);
400 }
401
402 // Add the frame we just received.
403 if (!m_deinterlace_filter->AddSourceFrame(*frame)) {
404 return;
405 }
406
407 // Pend output fields.
408 while (true) {
409 auto filter_frame = m_deinterlace_filter->DrainSinkFrame();
410 if (!filter_frame) {
411 break;
412 }
413
414 frame_queue.push(std::move(filter_frame));
415 }
416 }
417}
418
419} // namespace FFmpeg
diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.h b/src/video_core/host1x/ffmpeg/ffmpeg.h
new file mode 100644
index 000000000..1de0bbd83
--- /dev/null
+++ b/src/video_core/host1x/ffmpeg/ffmpeg.h
@@ -0,0 +1,213 @@
1// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
2// SPDX-License-Identifier: GPL-2.0-or-later
3
4#pragma once
5
6#include <memory>
7#include <optional>
8#include <span>
9#include <vector>
10#include <queue>
11
12#include "common/common_funcs.h"
13#include "common/common_types.h"
14#include "video_core/host1x/nvdec_common.h"
15
16extern "C" {
17#if defined(__GNUC__) || defined(__clang__)
18#pragma GCC diagnostic push
19#pragma GCC diagnostic ignored "-Wconversion"
20#endif
21
22#include <libavcodec/avcodec.h>
23#include <libavfilter/avfilter.h>
24#include <libavfilter/buffersink.h>
25#include <libavfilter/buffersrc.h>
26#include <libavutil/avutil.h>
27#include <libavutil/opt.h>
28
29#if defined(__GNUC__) || defined(__clang__)
30#pragma GCC diagnostic pop
31#endif
32}
33
34namespace FFmpeg {
35
36class Packet;
37class Frame;
38class Decoder;
39class HardwareContext;
40class DecoderContext;
41class DeinterlaceFilter;
42
43// Wraps an AVPacket, a container for compressed bitstream data.
44class Packet {
45public:
46 YUZU_NON_COPYABLE(Packet);
47 YUZU_NON_MOVEABLE(Packet);
48
49 explicit Packet(std::span<const u8> data);
50 ~Packet();
51
52 AVPacket* GetPacket() const {
53 return m_packet;
54 }
55
56private:
57 AVPacket* m_packet{};
58};
59
60// Wraps an AVFrame, a container for audio and video stream data.
61class Frame {
62public:
63 YUZU_NON_COPYABLE(Frame);
64 YUZU_NON_MOVEABLE(Frame);
65
66 explicit Frame();
67 ~Frame();
68
69 int GetWidth() const {
70 return m_frame->width;
71 }
72
73 int GetHeight() const {
74 return m_frame->height;
75 }
76
77 AVPixelFormat GetPixelFormat() const {
78 return static_cast<AVPixelFormat>(m_frame->format);
79 }
80
81 int GetStride(int plane) const {
82 return m_frame->linesize[plane];
83 }
84
85 int* GetStrides() const {
86 return m_frame->linesize;
87 }
88
89 u8* GetData(int plane) const {
90 return m_frame->data[plane];
91 }
92
93 u8** GetPlanes() const {
94 return m_frame->data;
95 }
96
97 void SetFormat(int format) {
98 m_frame->format = format;
99 }
100
101 AVFrame* GetFrame() const {
102 return m_frame;
103 }
104
105private:
106 AVFrame* m_frame{};
107};
108
109// Wraps an AVCodec, a type containing information about a codec.
110class Decoder {
111public:
112 YUZU_NON_COPYABLE(Decoder);
113 YUZU_NON_MOVEABLE(Decoder);
114
115 explicit Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec);
116 ~Decoder() = default;
117
118 bool SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceType type) const;
119
120 const AVCodec* GetCodec() const {
121 return m_codec;
122 }
123
124private:
125 const AVCodec* m_codec{};
126};
127
128// Wraps AVBufferRef for an accelerated decoder.
129class HardwareContext {
130public:
131 YUZU_NON_COPYABLE(HardwareContext);
132 YUZU_NON_MOVEABLE(HardwareContext);
133
134 static std::vector<AVHWDeviceType> GetSupportedDeviceTypes();
135
136 explicit HardwareContext() = default;
137 ~HardwareContext();
138
139 bool InitializeForDecoder(DecoderContext& decoder_context, const Decoder& decoder);
140
141 AVBufferRef* GetBufferRef() const {
142 return m_gpu_decoder;
143 }
144
145private:
146 bool InitializeWithType(AVHWDeviceType type);
147
148 AVBufferRef* m_gpu_decoder{};
149};
150
151// Wraps an AVCodecContext.
152class DecoderContext {
153public:
154 YUZU_NON_COPYABLE(DecoderContext);
155 YUZU_NON_MOVEABLE(DecoderContext);
156
157 explicit DecoderContext(const Decoder& decoder);
158 ~DecoderContext();
159
160 void InitializeHardwareDecoder(const HardwareContext& context, AVPixelFormat hw_pix_fmt);
161 bool OpenContext(const Decoder& decoder);
162 bool SendPacket(const Packet& packet);
163 std::unique_ptr<Frame> ReceiveFrame(bool* out_is_interlaced);
164
165 AVCodecContext* GetCodecContext() const {
166 return m_codec_context;
167 }
168
169private:
170 AVCodecContext* m_codec_context{};
171};
172
173// Wraps an AVFilterGraph.
174class DeinterlaceFilter {
175public:
176 YUZU_NON_COPYABLE(DeinterlaceFilter);
177 YUZU_NON_MOVEABLE(DeinterlaceFilter);
178
179 explicit DeinterlaceFilter(const Frame& frame);
180 ~DeinterlaceFilter();
181
182 bool AddSourceFrame(const Frame& frame);
183 std::unique_ptr<Frame> DrainSinkFrame();
184
185private:
186 AVFilterGraph* m_filter_graph{};
187 AVFilterContext* m_source_context{};
188 AVFilterContext* m_sink_context{};
189 bool m_initialized{};
190};
191
192class DecodeApi {
193public:
194 YUZU_NON_COPYABLE(DecodeApi);
195 YUZU_NON_MOVEABLE(DecodeApi);
196
197 DecodeApi() = default;
198 ~DecodeApi() = default;
199
200 bool Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec);
201 void Reset();
202
203 bool SendPacket(std::span<const u8> packet_data, size_t configuration_size);
204 void ReceiveFrames(std::queue<std::unique_ptr<Frame>>& frame_queue);
205
206private:
207 std::optional<FFmpeg::Decoder> m_decoder;
208 std::optional<FFmpeg::DecoderContext> m_decoder_context;
209 std::optional<FFmpeg::HardwareContext> m_hardware_context;
210 std::optional<FFmpeg::DeinterlaceFilter> m_deinterlace_filter;
211};
212
213} // namespace FFmpeg
diff --git a/src/video_core/host1x/nvdec.cpp b/src/video_core/host1x/nvdec.cpp
index a4bd5b79f..b8f5866d3 100644
--- a/src/video_core/host1x/nvdec.cpp
+++ b/src/video_core/host1x/nvdec.cpp
@@ -28,7 +28,7 @@ void Nvdec::ProcessMethod(u32 method, u32 argument) {
28 } 28 }
29} 29}
30 30
31AVFramePtr Nvdec::GetFrame() { 31std::unique_ptr<FFmpeg::Frame> Nvdec::GetFrame() {
32 return codec->GetCurrentFrame(); 32 return codec->GetCurrentFrame();
33} 33}
34 34
diff --git a/src/video_core/host1x/nvdec.h b/src/video_core/host1x/nvdec.h
index 3949d5181..ddddb8d28 100644
--- a/src/video_core/host1x/nvdec.h
+++ b/src/video_core/host1x/nvdec.h
@@ -23,7 +23,7 @@ public:
23 void ProcessMethod(u32 method, u32 argument); 23 void ProcessMethod(u32 method, u32 argument);
24 24
25 /// Return most recently decoded frame 25 /// Return most recently decoded frame
26 [[nodiscard]] AVFramePtr GetFrame(); 26 [[nodiscard]] std::unique_ptr<FFmpeg::Frame> GetFrame();
27 27
28private: 28private:
29 /// Invoke codec to decode a frame 29 /// Invoke codec to decode a frame
diff --git a/src/video_core/host1x/vic.cpp b/src/video_core/host1x/vic.cpp
index 10d7ef884..2a5eba415 100644
--- a/src/video_core/host1x/vic.cpp
+++ b/src/video_core/host1x/vic.cpp
@@ -82,27 +82,26 @@ void Vic::Execute() {
82 return; 82 return;
83 } 83 }
84 const VicConfig config{host1x.MemoryManager().Read<u64>(config_struct_address + 0x20)}; 84 const VicConfig config{host1x.MemoryManager().Read<u64>(config_struct_address + 0x20)};
85 const AVFramePtr frame_ptr = nvdec_processor->GetFrame(); 85 auto frame = nvdec_processor->GetFrame();
86 const auto* frame = frame_ptr.get();
87 if (!frame) { 86 if (!frame) {
88 return; 87 return;
89 } 88 }
90 const u64 surface_width = config.surface_width_minus1 + 1; 89 const u64 surface_width = config.surface_width_minus1 + 1;
91 const u64 surface_height = config.surface_height_minus1 + 1; 90 const u64 surface_height = config.surface_height_minus1 + 1;
92 if (static_cast<u64>(frame->width) != surface_width || 91 if (static_cast<u64>(frame->GetWidth()) != surface_width ||
93 static_cast<u64>(frame->height) != surface_height) { 92 static_cast<u64>(frame->GetHeight()) != surface_height) {
94 // TODO: Properly support multiple video streams with differing frame dimensions 93 // TODO: Properly support multiple video streams with differing frame dimensions
95 LOG_WARNING(Service_NVDRV, "Frame dimensions {}x{} don't match surface dimensions {}x{}", 94 LOG_WARNING(Service_NVDRV, "Frame dimensions {}x{} don't match surface dimensions {}x{}",
96 frame->width, frame->height, surface_width, surface_height); 95 frame->GetWidth(), frame->GetHeight(), surface_width, surface_height);
97 } 96 }
98 switch (config.pixel_format) { 97 switch (config.pixel_format) {
99 case VideoPixelFormat::RGBA8: 98 case VideoPixelFormat::RGBA8:
100 case VideoPixelFormat::BGRA8: 99 case VideoPixelFormat::BGRA8:
101 case VideoPixelFormat::RGBX8: 100 case VideoPixelFormat::RGBX8:
102 WriteRGBFrame(frame, config); 101 WriteRGBFrame(std::move(frame), config);
103 break; 102 break;
104 case VideoPixelFormat::YUV420: 103 case VideoPixelFormat::YUV420:
105 WriteYUVFrame(frame, config); 104 WriteYUVFrame(std::move(frame), config);
106 break; 105 break;
107 default: 106 default:
108 UNIMPLEMENTED_MSG("Unknown video pixel format {:X}", config.pixel_format.Value()); 107 UNIMPLEMENTED_MSG("Unknown video pixel format {:X}", config.pixel_format.Value());
@@ -110,10 +109,14 @@ void Vic::Execute() {
110 } 109 }
111} 110}
112 111
113void Vic::WriteRGBFrame(const AVFrame* frame, const VicConfig& config) { 112void Vic::WriteRGBFrame(std::unique_ptr<FFmpeg::Frame> frame, const VicConfig& config) {
114 LOG_TRACE(Service_NVDRV, "Writing RGB Frame"); 113 LOG_TRACE(Service_NVDRV, "Writing RGB Frame");
115 114
116 if (!scaler_ctx || frame->width != scaler_width || frame->height != scaler_height) { 115 const auto frame_width = frame->GetWidth();
116 const auto frame_height = frame->GetHeight();
117 const auto frame_format = frame->GetPixelFormat();
118
119 if (!scaler_ctx || frame_width != scaler_width || frame_height != scaler_height) {
117 const AVPixelFormat target_format = [pixel_format = config.pixel_format]() { 120 const AVPixelFormat target_format = [pixel_format = config.pixel_format]() {
118 switch (pixel_format) { 121 switch (pixel_format) {
119 case VideoPixelFormat::RGBA8: 122 case VideoPixelFormat::RGBA8:
@@ -129,27 +132,26 @@ void Vic::WriteRGBFrame(const AVFrame* frame, const VicConfig& config) {
129 132
130 sws_freeContext(scaler_ctx); 133 sws_freeContext(scaler_ctx);
131 // Frames are decoded into either YUV420 or NV12 formats. Convert to desired RGB format 134 // Frames are decoded into either YUV420 or NV12 formats. Convert to desired RGB format
132 scaler_ctx = sws_getContext(frame->width, frame->height, 135 scaler_ctx = sws_getContext(frame_width, frame_height, frame_format, frame_width,
133 static_cast<AVPixelFormat>(frame->format), frame->width, 136 frame_height, target_format, 0, nullptr, nullptr, nullptr);
134 frame->height, target_format, 0, nullptr, nullptr, nullptr); 137 scaler_width = frame_width;
135 scaler_width = frame->width; 138 scaler_height = frame_height;
136 scaler_height = frame->height;
137 converted_frame_buffer.reset(); 139 converted_frame_buffer.reset();
138 } 140 }
139 if (!converted_frame_buffer) { 141 if (!converted_frame_buffer) {
140 const size_t frame_size = frame->width * frame->height * 4; 142 const size_t frame_size = frame_width * frame_height * 4;
141 converted_frame_buffer = AVMallocPtr{static_cast<u8*>(av_malloc(frame_size)), av_free}; 143 converted_frame_buffer = AVMallocPtr{static_cast<u8*>(av_malloc(frame_size)), av_free};
142 } 144 }
143 const std::array<int, 4> converted_stride{frame->width * 4, frame->height * 4, 0, 0}; 145 const std::array<int, 4> converted_stride{frame_width * 4, frame_height * 4, 0, 0};
144 u8* const converted_frame_buf_addr{converted_frame_buffer.get()}; 146 u8* const converted_frame_buf_addr{converted_frame_buffer.get()};
145 sws_scale(scaler_ctx, frame->data, frame->linesize, 0, frame->height, &converted_frame_buf_addr, 147 sws_scale(scaler_ctx, frame->GetPlanes(), frame->GetStrides(), 0, frame_height,
146 converted_stride.data()); 148 &converted_frame_buf_addr, converted_stride.data());
147 149
148 // Use the minimum of surface/frame dimensions to avoid buffer overflow. 150 // Use the minimum of surface/frame dimensions to avoid buffer overflow.
149 const u32 surface_width = static_cast<u32>(config.surface_width_minus1) + 1; 151 const u32 surface_width = static_cast<u32>(config.surface_width_minus1) + 1;
150 const u32 surface_height = static_cast<u32>(config.surface_height_minus1) + 1; 152 const u32 surface_height = static_cast<u32>(config.surface_height_minus1) + 1;
151 const u32 width = std::min(surface_width, static_cast<u32>(frame->width)); 153 const u32 width = std::min(surface_width, static_cast<u32>(frame_width));
152 const u32 height = std::min(surface_height, static_cast<u32>(frame->height)); 154 const u32 height = std::min(surface_height, static_cast<u32>(frame_height));
153 const u32 blk_kind = static_cast<u32>(config.block_linear_kind); 155 const u32 blk_kind = static_cast<u32>(config.block_linear_kind);
154 if (blk_kind != 0) { 156 if (blk_kind != 0) {
155 // swizzle pitch linear to block linear 157 // swizzle pitch linear to block linear
@@ -169,23 +171,23 @@ void Vic::WriteRGBFrame(const AVFrame* frame, const VicConfig& config) {
169 } 171 }
170} 172}
171 173
172void Vic::WriteYUVFrame(const AVFrame* frame, const VicConfig& config) { 174void Vic::WriteYUVFrame(std::unique_ptr<FFmpeg::Frame> frame, const VicConfig& config) {
173 LOG_TRACE(Service_NVDRV, "Writing YUV420 Frame"); 175 LOG_TRACE(Service_NVDRV, "Writing YUV420 Frame");
174 176
175 const std::size_t surface_width = config.surface_width_minus1 + 1; 177 const std::size_t surface_width = config.surface_width_minus1 + 1;
176 const std::size_t surface_height = config.surface_height_minus1 + 1; 178 const std::size_t surface_height = config.surface_height_minus1 + 1;
177 const std::size_t aligned_width = (surface_width + 0xff) & ~0xffUL; 179 const std::size_t aligned_width = (surface_width + 0xff) & ~0xffUL;
178 // Use the minimum of surface/frame dimensions to avoid buffer overflow. 180 // Use the minimum of surface/frame dimensions to avoid buffer overflow.
179 const auto frame_width = std::min(surface_width, static_cast<size_t>(frame->width)); 181 const auto frame_width = std::min(surface_width, static_cast<size_t>(frame->GetWidth()));
180 const auto frame_height = std::min(surface_height, static_cast<size_t>(frame->height)); 182 const auto frame_height = std::min(surface_height, static_cast<size_t>(frame->GetHeight()));
181 183
182 const auto stride = static_cast<size_t>(frame->linesize[0]); 184 const auto stride = static_cast<size_t>(frame->GetStride(0));
183 185
184 luma_buffer.resize_destructive(aligned_width * surface_height); 186 luma_buffer.resize_destructive(aligned_width * surface_height);
185 chroma_buffer.resize_destructive(aligned_width * surface_height / 2); 187 chroma_buffer.resize_destructive(aligned_width * surface_height / 2);
186 188
187 // Populate luma buffer 189 // Populate luma buffer
188 const u8* luma_src = frame->data[0]; 190 const u8* luma_src = frame->GetData(0);
189 for (std::size_t y = 0; y < frame_height; ++y) { 191 for (std::size_t y = 0; y < frame_height; ++y) {
190 const std::size_t src = y * stride; 192 const std::size_t src = y * stride;
191 const std::size_t dst = y * aligned_width; 193 const std::size_t dst = y * aligned_width;
@@ -196,16 +198,16 @@ void Vic::WriteYUVFrame(const AVFrame* frame, const VicConfig& config) {
196 198
197 // Chroma 199 // Chroma
198 const std::size_t half_height = frame_height / 2; 200 const std::size_t half_height = frame_height / 2;
199 const auto half_stride = static_cast<size_t>(frame->linesize[1]); 201 const auto half_stride = static_cast<size_t>(frame->GetStride(1));
200 202
201 switch (frame->format) { 203 switch (frame->GetPixelFormat()) {
202 case AV_PIX_FMT_YUV420P: { 204 case AV_PIX_FMT_YUV420P: {
203 // Frame from FFmpeg software 205 // Frame from FFmpeg software
204 // Populate chroma buffer from both channels with interleaving. 206 // Populate chroma buffer from both channels with interleaving.
205 const std::size_t half_width = frame_width / 2; 207 const std::size_t half_width = frame_width / 2;
206 u8* chroma_buffer_data = chroma_buffer.data(); 208 u8* chroma_buffer_data = chroma_buffer.data();
207 const u8* chroma_b_src = frame->data[1]; 209 const u8* chroma_b_src = frame->GetData(1);
208 const u8* chroma_r_src = frame->data[2]; 210 const u8* chroma_r_src = frame->GetData(2);
209 for (std::size_t y = 0; y < half_height; ++y) { 211 for (std::size_t y = 0; y < half_height; ++y) {
210 const std::size_t src = y * half_stride; 212 const std::size_t src = y * half_stride;
211 const std::size_t dst = y * aligned_width; 213 const std::size_t dst = y * aligned_width;
@@ -219,7 +221,7 @@ void Vic::WriteYUVFrame(const AVFrame* frame, const VicConfig& config) {
219 case AV_PIX_FMT_NV12: { 221 case AV_PIX_FMT_NV12: {
220 // Frame from VA-API hardware 222 // Frame from VA-API hardware
221 // This is already interleaved so just copy 223 // This is already interleaved so just copy
222 const u8* chroma_src = frame->data[1]; 224 const u8* chroma_src = frame->GetData(1);
223 for (std::size_t y = 0; y < half_height; ++y) { 225 for (std::size_t y = 0; y < half_height; ++y) {
224 const std::size_t src = y * stride; 226 const std::size_t src = y * stride;
225 const std::size_t dst = y * aligned_width; 227 const std::size_t dst = y * aligned_width;
diff --git a/src/video_core/host1x/vic.h b/src/video_core/host1x/vic.h
index 3d9753047..6c868f062 100644
--- a/src/video_core/host1x/vic.h
+++ b/src/video_core/host1x/vic.h
@@ -39,9 +39,9 @@ public:
39private: 39private:
40 void Execute(); 40 void Execute();
41 41
42 void WriteRGBFrame(const AVFrame* frame, const VicConfig& config); 42 void WriteRGBFrame(std::unique_ptr<FFmpeg::Frame> frame, const VicConfig& config);
43 43
44 void WriteYUVFrame(const AVFrame* frame, const VicConfig& config); 44 void WriteYUVFrame(std::unique_ptr<FFmpeg::Frame> frame, const VicConfig& config);
45 45
46 Host1x& host1x; 46 Host1x& host1x;
47 std::shared_ptr<Tegra::Host1x::Nvdec> nvdec_processor; 47 std::shared_ptr<Tegra::Host1x::Nvdec> nvdec_processor;