Move FFMPEG Decoder to lib

CMakeLists.txt

@@ -16,6 +16,12 @@ option(CHIAKI_ENABLE_ANDROID "Enable Android (Use only as part of the Gradle Pro
 option(CHIAKI_ENABLE_BOREALIS "Enable Borealis GUI (For Nintendo Switch or PC)" OFF)
 tri_option(CHIAKI_ENABLE_SETSU "Enable libsetsu for touchpad input from controller" AUTO)
 option(CHIAKI_LIB_ENABLE_OPUS "Use Opus as part of Chiaki Lib" ON)
+if(CHIAKI_ENABLE_GUI OR CHIAKI_ENABLE_BOREALIS)
+	set(CHIAKI_FFMPEG_DEFAULT ON)
+else()
+	set(CHIAKI_FFMPEG_DEFAULT AUTO)
+endif()
+tri_option(CHIAKI_ENABLE_FFMPEG_DECODER "Enable FFMPEG video decoder" ${CHIAKI_FFMPEG_DEFAULT})
 tri_option(CHIAKI_ENABLE_PI_DECODER "Enable Raspberry Pi-specific video decoder (requires libraspberrypi0 and libraspberrypi-doc)" AUTO)
 option(CHIAKI_LIB_ENABLE_MBEDTLS "Use mbedtls instead of OpenSSL as part of Chiaki Lib" OFF)
 option(CHIAKI_LIB_OPENSSL_EXTERNAL_PROJECT "Use OpenSSL as CMake external project" OFF)
@@ -85,6 +91,24 @@ if(CHIAKI_LIB_ENABLE_MBEDTLS)
 	add_definitions(-DCHIAKI_LIB_ENABLE_MBEDTLS)
 endif()
 
+if(CHIAKI_ENABLE_FFMPEG_DECODER)
+	find_package(FFMPEG COMPONENTS avcodec avutil)
+	if(FFMPEG_FOUND)
+		set(CHIAKI_ENABLE_FFMPEG_DECODER ON)
+	else()
+		if(NOT CHIAKI_ENABLE_FFMPEG_DECODER STREQUAL AUTO)
+			message(FATAL_ERROR "CHIAKI_ENABLE_FFMPEG_DECODER is set to ON, but ffmpeg could not be found.")
+		endif()
+		set(CHIAKI_ENABLE_FFMPEG_DECODER OFF)
+	endif()
+endif()
+
+if(CHIAKI_ENABLE_FFMPEG_DECODER)
+	message(STATUS "FFMPEG Decoder enabled")
+else()
+	message(STATUS "FFMPEG Decoder disabled")
+endif()
+
 if(CHIAKI_ENABLE_PI_DECODER)
 	find_package(ILClient)
 	if(ILClient_FOUND)
@@ -105,7 +129,6 @@ else()
 	message(STATUS "Pi Decoder disabled")
 endif()
 
-
 add_subdirectory(lib)
 
 if(CHIAKI_ENABLE_CLI)

gui/CMakeLists.txt

@@ -14,8 +14,6 @@ if(WIN32)
 	add_definitions(-DWIN32_LEAN_AND_MEAN)
 endif()
 
-find_package(FFMPEG REQUIRED COMPONENTS avcodec avutil)
-
 set(RESOURCE_FILES "")
 
 if(APPLE)
@@ -28,8 +26,6 @@ add_executable(chiaki WIN32
 		src/main.cpp
 		include/streamwindow.h
 		src/streamwindow.cpp
-		include/videodecoder.h
-		src/videodecoder.cpp
 		include/mainwindow.h
 		src/mainwindow.cpp
 		include/dynamicgridwidget.h
@@ -73,7 +69,6 @@ if(CHIAKI_ENABLE_CLI)
 	target_link_libraries(chiaki chiaki-cli-lib)
 endif()
 
-target_link_libraries(chiaki FFMPEG::avcodec FFMPEG::avutil)
 target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::Concurrent Qt5::Multimedia Qt5::OpenGL Qt5::Svg)
 if(APPLE)
 	target_link_libraries(chiaki Qt5::MacExtras)
@@ -87,6 +82,9 @@ if(CHIAKI_ENABLE_SETSU)
 	target_link_libraries(chiaki setsu)
 	target_compile_definitions(chiaki PRIVATE CHIAKI_GUI_ENABLE_SETSU)
 endif()
+if(NOT CHIAKI_ENABLE_FFMPEG_DECODER)
+	message(FATAL_ERROR "Chiaki GUI requires CHIAKI_ENABLE_FFMPEG_DECODER=ON")
+endif()
 
 set_target_properties(chiaki PROPERTIES
 		MACOSX_BUNDLE TRUE

gui/include/avopenglframeuploader.h

@@ -6,8 +6,10 @@
 #include <QObject>
 #include <QOpenGLWidget>
 
+#include <chiaki/ffmpegdecoder.h>
+
+class StreamSession;
 class AVOpenGLWidget;
-class VideoDecoder;
 class QSurface;
 
 class AVOpenGLFrameUploader: public QObject
@@ -15,16 +17,16 @@ class AVOpenGLFrameUploader: public QObject
 	Q_OBJECT
 
 	private:
-		VideoDecoder *decoder;
+		StreamSession *session;
 		AVOpenGLWidget *widget;
 		QOpenGLContext *context;
 		QSurface *surface;
 
 	private slots:
-		void UpdateFrame();
+		void UpdateFrameFromDecoder();
 
 	public:
-		AVOpenGLFrameUploader(VideoDecoder *decoder, AVOpenGLWidget *widget, QOpenGLContext *context, QSurface *surface);
+		AVOpenGLFrameUploader(StreamSession *session, AVOpenGLWidget *widget, QOpenGLContext *context, QSurface *surface);
 };
 
 #endif // CHIAKI_AVOPENGLFRAMEUPLOADER_H

gui/include/avopenglwidget.h

@@ -15,7 +15,7 @@ extern "C"
 
 #define MAX_PANES 3
 
-class VideoDecoder;
+class StreamSession;
 class AVOpenGLFrameUploader;
 class QOffscreenSurface;
 
@@ -53,7 +53,7 @@ class AVOpenGLWidget: public QOpenGLWidget
 	Q_OBJECT
 
 	private:
-		VideoDecoder *decoder;
+		StreamSession *session;
 
 		GLuint program;
 		GLuint vbo;
@@ -74,7 +74,7 @@ class AVOpenGLWidget: public QOpenGLWidget
 	public:
 		static QSurfaceFormat CreateSurfaceFormat();
 
-		explicit AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent = nullptr);
+		explicit AVOpenGLWidget(StreamSession *session, QWidget *parent = nullptr);
 		~AVOpenGLWidget() override;
 
 		void SwapFrames();

gui/include/logsniffer.h

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: LicenseRef-GPL-3.0-or-later-OpenSSL
+
+#ifndef CHIAKI_LOG_SNIFFER_H
+#define CHIAKI_LOG_SNIFFER_H
+
+#include <chiaki/controller.h>
+
+#include <QObject>
+#include <QSet>
+#include <QMap>
+#include <QString>
+
+#ifdef CHIAKI_GUI_ENABLE_SDL_GAMECONTROLLER
+#include <SDL.h>
+#endif
+
+class Controller;
+
+class ControllerManager : public QObject
+{
+	Q_OBJECT
+
+	friend class Controller;
+
+	private:
+#ifdef CHIAKI_GUI_ENABLE_SDL_GAMECONTROLLER
+		QSet<SDL_JoystickID> available_controllers;
+#endif
+		QMap<int, Controller *> open_controllers;
+
+		void ControllerClosed(Controller *controller);
+
+	private slots:
+		void UpdateAvailableControllers();
+		void HandleEvents();
+		void ControllerEvent(int device_id);
+
+	public:
+		static ControllerManager *GetInstance();
+
+		ControllerManager(QObject *parent = nullptr);
+		~ControllerManager();
+
+		QSet<int> GetAvailableControllers();
+		Controller *OpenController(int device_id);
+
+	signals:
+		void AvailableControllersUpdated();
+};
+
+class Controller : public QObject
+{
+	Q_OBJECT
+
+	friend class ControllerManager;
+
+	private:
+		Controller(int device_id, ControllerManager *manager);
+
+		void UpdateState();
+
+		ControllerManager *manager;
+		int id;
+
+#ifdef CHIAKI_GUI_ENABLE_SDL_GAMECONTROLLER
+		SDL_GameController *controller;
+#endif
+
+	public:
+		~Controller();
+
+		bool IsConnected();
+		int GetDeviceID();
+		QString GetName();
+		ChiakiControllerState GetState();
+
+	signals:
+		void StateChanged();
+};
+
+#endif // CHIAKI_CONTROLLERMANAGER_H

gui/include/settings.h

@@ -6,7 +6,6 @@
 #include <chiaki/session.h>
 
 #include "host.h"
-#include "videodecoder.h"
 
 #include <QSettings>
 #include <QAudioDeviceInfo>
@@ -79,8 +78,8 @@ class Settings : public QObject
 		Decoder GetDecoder() const;
 		void SetDecoder(Decoder decoder);
 
-		HardwareDecodeEngine GetHardwareDecodeEngine() const;
+		QString GetHardwareDecoder() const;
-		void SetHardwareDecodeEngine(HardwareDecodeEngine enabled);
+		void SetHardwareDecoder(const QString &hw_decoder);
 
 		unsigned int GetAudioBufferSizeDefault() const;
 

gui/include/settingsdialog.h

@@ -27,7 +27,7 @@ class SettingsDialog : public QDialog
 		QLineEdit *audio_buffer_size_edit;
 		QComboBox *audio_device_combo_box;
 		QCheckBox *pi_decoder_check_box;
-		QComboBox *hardware_decode_combo_box;
+		QComboBox *hw_decoder_combo_box;
 
 		QListWidget *registered_hosts_list_widget;
 		QPushButton *delete_registered_host_button;

gui/include/streamsession.h

@@ -5,6 +5,7 @@
 
 #include <chiaki/session.h>
 #include <chiaki/opusdecoder.h>
+#include <chiaki/ffmpegdecoder.h>
 
 #if CHIAKI_LIB_ENABLE_PI_DECODER
 #include <chiaki/pidecoder.h>
@@ -14,7 +15,6 @@
 #include <setsu.h>
 #endif
 
-#include "videodecoder.h"
 #include "exception.h"
 #include "sessionlog.h"
 #include "controllermanager.h"
@@ -41,7 +41,7 @@ struct StreamSessionConnectInfo
 	Settings *settings;
 	QMap<Qt::Key, int> key_map;
 	Decoder decoder;
-	HardwareDecodeEngine hw_decode_engine;
+	QString hw_decoder;
 	QString audio_out_device;
 	uint32_t log_level_mask;
 	QString log_file;
@@ -78,7 +78,8 @@ class StreamSession : public QObject
 
 		ChiakiControllerState keyboard_state;
 
-		VideoDecoder *video_decoder;
+		ChiakiFfmpegDecoder *ffmpeg_decoder;
+		void TriggerFfmpegFrameAvailable();
 #if CHIAKI_LIB_ENABLE_PI_DECODER
 		ChiakiPiDecoder *pi_decoder;
 #endif
@@ -91,7 +92,6 @@ class StreamSession : public QObject
 		QMap<Qt::Key, int> key_map;
 
 		void PushAudioFrame(int16_t *buf, size_t samples_count);
-		void PushVideoSample(uint8_t *buf, size_t buf_size);
 		void Event(ChiakiEvent *event);
 #if CHIAKI_GUI_ENABLE_SETSU
 		void HandleSetsuEvent(SetsuEvent *event);
@@ -112,8 +112,9 @@ class StreamSession : public QObject
 
 		void SetLoginPIN(const QString &pin);
 
+		ChiakiLog *GetChiakiLog()				{ return log.GetChiakiLog(); }
 		QList<Controller *> GetControllers()	{ return controllers.values(); }
-		VideoDecoder *GetVideoDecoder()	{ return video_decoder; }
+		ChiakiFfmpegDecoder *GetFfmpegDecoder()	{ return ffmpeg_decoder; }
 #if CHIAKI_LIB_ENABLE_PI_DECODER
 		ChiakiPiDecoder *GetPiDecoder()	{ return pi_decoder; }
 #endif
@@ -122,7 +123,7 @@ class StreamSession : public QObject
 		void HandleMouseEvent(QMouseEvent *event);
 
 	signals:
-		void CurrentImageUpdated();
+		void FfmpegFrameAvailable();
 		void SessionQuit(ChiakiQuitReason reason, const QString &reason_str);
 		void LoginPINRequested(bool incorrect);
 

gui/include/videodecoder.h

@@ -1,77 +0,0 @@
-// SPDX-License-Identifier: LicenseRef-AGPL-3.0-only-OpenSSL
-
-#ifndef CHIAKI_VIDEODECODER_H
-#define CHIAKI_VIDEODECODER_H
-
-#include <chiaki/log.h>
-
-#include "exception.h"
-
-#include <QMap>
-#include <QMutex>
-#include <QObject>
-
-extern "C"
-{
-#include <libavcodec/avcodec.h>
-}
-
-#include <cstdint>
-
-
-typedef enum {
-	HW_DECODE_NONE = 0,
-	HW_DECODE_VAAPI = 1,
-	HW_DECODE_VDPAU = 2,
-	HW_DECODE_VIDEOTOOLBOX = 3,
-	HW_DECODE_CUDA = 4,
-} HardwareDecodeEngine;
-
-
-static const QMap<HardwareDecodeEngine, const char *> hardware_decode_engine_names = {
-	{ HW_DECODE_NONE, "none"},
-	{ HW_DECODE_VAAPI, "vaapi"},
-	{ HW_DECODE_VDPAU, "vdpau"},
-	{ HW_DECODE_VIDEOTOOLBOX, "videotoolbox"},
-	{ HW_DECODE_CUDA, "cuda"},
-};
-
-class VideoDecoderException: public Exception
-{
-	public:
-		explicit VideoDecoderException(const QString &msg) : Exception(msg) {};
-};
-
-class VideoDecoder: public QObject
-{
-	Q_OBJECT
-
-	public:
-		VideoDecoder(HardwareDecodeEngine hw_decode_engine, ChiakiLog *log);
-		~VideoDecoder();
-
-		void PushFrame(uint8_t *buf, size_t buf_size);
-		AVFrame *PullFrame();
-		AVFrame *GetFromHardware(AVFrame *hw_frame);
-
-		ChiakiLog *GetChiakiLog()	{ return log; }
-
-		enum AVPixelFormat PixelFormat() { return hw_decode_engine?AV_PIX_FMT_NV12:AV_PIX_FMT_YUV420P; }
-
-	signals:
-		void FramesAvailable();
-
-	private:
-		HardwareDecodeEngine hw_decode_engine;
-
-		ChiakiLog *log;
-		QMutex mutex;
-
-		AVCodec *codec;
-		AVCodecContext *codec_context;
-
-		enum AVPixelFormat hw_pix_fmt;
-		AVBufferRef *hw_device_ctx;
-};
-
-#endif // CHIAKI_VIDEODECODER_H

gui/src/avopenglframeuploader.cpp

@@ -2,31 +2,38 @@
 
 #include <avopenglframeuploader.h>
 #include <avopenglwidget.h>
-#include <videodecoder.h>
+#include <streamsession.h>
 
 #include <QOpenGLContext>
 #include <QOpenGLFunctions>
 
-AVOpenGLFrameUploader::AVOpenGLFrameUploader(VideoDecoder *decoder, AVOpenGLWidget *widget, QOpenGLContext *context, QSurface *surface)
+AVOpenGLFrameUploader::AVOpenGLFrameUploader(StreamSession *session, AVOpenGLWidget *widget, QOpenGLContext *context, QSurface *surface)
 	: QObject(nullptr),
-	decoder(decoder),
+	session(session),
 	widget(widget),
 	context(context),
 	surface(surface)
 {
-	connect(decoder, SIGNAL(FramesAvailable()), this, SLOT(UpdateFrame()));
+	connect(session, &StreamSession::FfmpegFrameAvailable, this, &AVOpenGLFrameUploader::UpdateFrameFromDecoder);
 }
 
-void AVOpenGLFrameUploader::UpdateFrame()
+void AVOpenGLFrameUploader::UpdateFrameFromDecoder()
 {
+	ChiakiFfmpegDecoder *decoder = session->GetFfmpegDecoder();
+	if(!decoder)
+	{
+		CHIAKI_LOGE(session->GetChiakiLog(), "Session has no ffmpeg decoder");
+		return;
+	}
+
 	if(QOpenGLContext::currentContext() != context)
 		context->makeCurrent(surface);
 
-	AVFrame *next_frame = decoder->PullFrame();
+	AVFrame *next_frame = chiaki_ffmpeg_decoder_pull_frame(decoder);
 	if(!next_frame)
 		return;
 
-	bool success = widget->GetBackgroundFrame()->Update(next_frame, decoder->GetChiakiLog());
+	bool success = widget->GetBackgroundFrame()->Update(next_frame, decoder->log);
 	av_frame_free(&next_frame);
 
 	if(success)

gui/src/avopenglwidget.cpp

@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: LicenseRef-AGPL-3.0-only-OpenSSL
 
 #include <avopenglwidget.h>
-#include <videodecoder.h>
 #include <avopenglframeuploader.h>
+#include <streamsession.h>
 
 #include <QOffscreenSurface>
 #include <QOpenGLContext>
@@ -123,14 +123,15 @@ QSurfaceFormat AVOpenGLWidget::CreateSurfaceFormat()
 	return format;
 }
 
-AVOpenGLWidget::AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent)
+AVOpenGLWidget::AVOpenGLWidget(StreamSession *session, QWidget *parent)
 	: QOpenGLWidget(parent),
-	decoder(decoder)
+	session(session)
 {
+	enum AVPixelFormat pixel_format = chiaki_ffmpeg_decoder_get_pixel_format(session->GetFfmpegDecoder());
 	conversion_config = nullptr;
 	for(auto &cc: conversion_configs)
 	{
-		if(decoder->PixelFormat() == cc.pixel_format)
+		if(pixel_format == cc.pixel_format)
 		{
 			conversion_config = &cc;
 			break;
@@ -245,7 +246,7 @@ void AVOpenGLWidget::initializeGL()
 	auto f = QOpenGLContext::currentContext()->extraFunctions();
 
 	const char *gl_version = (const char *)f->glGetString(GL_VERSION);
-	CHIAKI_LOGI(decoder->GetChiakiLog(), "OpenGL initialized with version \"%s\"", gl_version ? gl_version : "(null)");
+	CHIAKI_LOGI(session->GetChiakiLog(), "OpenGL initialized with version \"%s\"", gl_version ? gl_version : "(null)");
 
 #ifdef DEBUG_OPENGL
 	auto logger = new QOpenGLDebugLogger(this);
@@ -266,7 +267,7 @@ void AVOpenGLWidget::initializeGL()
 		QVector<GLchar> info_log(info_log_size);
 		f->glGetShaderInfoLog(shader, info_log_size, &info_log_size, info_log.data());
 		f->glDeleteShader(shader);
-		CHIAKI_LOGE(decoder->GetChiakiLog(), "Failed to Compile Shader:\n%s", info_log.data());
+		CHIAKI_LOGE(session->GetChiakiLog(), "Failed to Compile Shader:\n%s", info_log.data());
 	};
 
 	GLuint shader_vert = f->glCreateShader(GL_VERTEX_SHADER);
@@ -294,7 +295,7 @@ void AVOpenGLWidget::initializeGL()
 		QVector<GLchar> info_log(info_log_size);
 		f->glGetProgramInfoLog(program, info_log_size, &info_log_size, info_log.data());
 		f->glDeleteProgram(program);
-		CHIAKI_LOGE(decoder->GetChiakiLog(), "Failed to Link Shader Program:\n%s", info_log.data());
+		CHIAKI_LOGE(session->GetChiakiLog(), "Failed to Link Shader Program:\n%s", info_log.data());
 		return;
 	}
 
@@ -346,14 +347,14 @@ void AVOpenGLWidget::initializeGL()
 	frame_uploader_context->setShareContext(context());
 	if(!frame_uploader_context->create())
 	{
-		CHIAKI_LOGE(decoder->GetChiakiLog(), "Failed to create upload OpenGL context");
+		CHIAKI_LOGE(session->GetChiakiLog(), "Failed to create upload OpenGL context");
 		return;
 	}
 
 	frame_uploader_surface = new QOffscreenSurface();
 	frame_uploader_surface->setFormat(context()->format());
 	frame_uploader_surface->create();
-	frame_uploader = new AVOpenGLFrameUploader(decoder, this, frame_uploader_context, frame_uploader_surface);
+	frame_uploader = new AVOpenGLFrameUploader(session, this, frame_uploader_context, frame_uploader_surface);
 	frame_fg = 0;
 
 	frame_uploader_thread = new QThread(this);

gui/src/main.cpp

@@ -4,7 +4,6 @@ int real_main(int argc, char *argv[]);
 int main(int argc, char *argv[]) { return real_main(argc, argv); }
 
 #include <streamwindow.h>
-#include <videodecoder.h>
 #include <mainwindow.h>
 #include <streamsession.h>
 #include <settings.h>

gui/src/settings.cpp

@@ -37,6 +37,10 @@ static void MigrateSettingsTo2(QSettings *settings)
 		i++;
 	}
 	settings->endArray();
+	QString hw_decoder = settings->value("settings/hw_decode_engine").toString();
+	settings->remove("settings/hw_decode_engine");
+	if(hw_decoder != "none")
+		settings->setValue("settings/hw_decoder", hw_decoder);
 }
 
 static void MigrateSettings(QSettings *settings)
@@ -160,25 +164,14 @@ void Settings::SetDecoder(Decoder decoder)
 	settings.setValue("settings/decoder", decoder_values[decoder]);
 }
 
-static const QMap<HardwareDecodeEngine, QString> hw_decode_engine_values = {
+QString Settings::GetHardwareDecoder() const
-	{ HW_DECODE_NONE, "none" },
-	{ HW_DECODE_VAAPI, "vaapi" },
-	{ HW_DECODE_VDPAU, "vdpau" },
-	{ HW_DECODE_VIDEOTOOLBOX, "videotoolbox" },
-	{ HW_DECODE_CUDA, "cuda" }
-};
-
-static const HardwareDecodeEngine hw_decode_engine_default = HW_DECODE_NONE;
-
-HardwareDecodeEngine Settings::GetHardwareDecodeEngine() const
 {
-	auto v = settings.value("settings/hw_decode_engine", hw_decode_engine_values[hw_decode_engine_default]).toString();
+	return settings.value("settings/hw_decoder").toString();
-	return hw_decode_engine_values.key(v, hw_decode_engine_default);
 }
 
-void Settings::SetHardwareDecodeEngine(HardwareDecodeEngine engine)
+void Settings::SetHardwareDecoder(const QString &hw_decoder)
 {
-	settings.setValue("settings/hw_decode_engine", hw_decode_engine_values[engine]);
+	settings.setValue("settings/hw_decoder", hw_decoder);
 }
 
 unsigned int Settings::GetAudioBufferSize() const

gui/src/settingsdialog.cpp

@@ -5,7 +5,6 @@
 #include <settingskeycapturedialog.h>
 #include <registdialog.h>
 #include <sessionlog.h>
-#include <videodecoder.h>
 
 #include <QHBoxLayout>
 #include <QVBoxLayout>
@@ -23,6 +22,7 @@
 #include <QFutureWatcher>
 
 #include <chiaki/config.h>
+#include <chiaki/ffmpegdecoder.h>
 
 const char * const about_string =
 	"<h1>Chiaki</h1> by thestr4ng3r, version " CHIAKI_VERSION
@@ -202,22 +202,22 @@ SettingsDialog::SettingsDialog(Settings *settings, QWidget *parent) : QDialog(pa
 	pi_decoder_check_box = nullptr;
 #endif
 
-	hardware_decode_combo_box = new QComboBox(this);
+	hw_decoder_combo_box = new QComboBox(this);
-	static const QList<QPair<HardwareDecodeEngine, const char *>> hardware_decode_engines = {
+	hw_decoder_combo_box->addItem("none", QString());
-		{ HW_DECODE_NONE, "none"},
+	auto current_hw_decoder = settings->GetHardwareDecoder();
-		{ HW_DECODE_VAAPI, "vaapi"},
+	enum AVHWDeviceType hw_dev = AV_HWDEVICE_TYPE_NONE;
-		{ HW_DECODE_VIDEOTOOLBOX, "videotoolbox"},
+	while(true)
-		{ HW_DECODE_CUDA, "cuda"}
-	};
-	auto current_hardware_decode_engine = settings->GetHardwareDecodeEngine();
-	for(const auto &p : hardware_decode_engines)
 	{
-		hardware_decode_combo_box->addItem(p.second, (int)p.first);
+		hw_dev = av_hwdevice_iterate_types(hw_dev);
-		if(current_hardware_decode_engine == p.first)
+		if(hw_dev == AV_HWDEVICE_TYPE_NONE)
-			hardware_decode_combo_box->setCurrentIndex(hardware_decode_combo_box->count() - 1);
+			break;
+		QString name = QString::fromUtf8(av_hwdevice_get_type_name(hw_dev));
+		hw_decoder_combo_box->addItem(name, name);
+		if(current_hw_decoder == name)
+			hw_decoder_combo_box->setCurrentIndex(hw_decoder_combo_box->count() - 1);
 	}
-	connect(hardware_decode_combo_box, SIGNAL(currentIndexChanged(int)), this, SLOT(HardwareDecodeEngineSelected()));
+	connect(hw_decoder_combo_box, SIGNAL(currentIndexChanged(int)), this, SLOT(HardwareDecodeEngineSelected()));
-	decode_settings_layout->addRow(tr("Hardware decode method:"), hardware_decode_combo_box);
+	decode_settings_layout->addRow(tr("Hardware decode method:"), hw_decoder_combo_box);
 	UpdateHardwareDecodeEngineComboBox();
 
 	// Registered Consoles
@@ -329,12 +329,12 @@ void SettingsDialog::AudioOutputSelected()
 
 void SettingsDialog::HardwareDecodeEngineSelected()
 {
-	settings->SetHardwareDecodeEngine((HardwareDecodeEngine)hardware_decode_combo_box->currentData().toInt());
+	settings->SetHardwareDecoder(hw_decoder_combo_box->currentData().toString());
 }
 
 void SettingsDialog::UpdateHardwareDecodeEngineComboBox()
 {
-	hardware_decode_combo_box->setEnabled(settings->GetDecoder() == Decoder::Ffmpeg);
+	hw_decoder_combo_box->setEnabled(settings->GetDecoder() == Decoder::Ffmpeg);
 }
 
 void SettingsDialog::UpdateBitratePlaceholder()

gui/src/streamsession.cpp

@@ -19,7 +19,7 @@ StreamSessionConnectInfo::StreamSessionConnectInfo(Settings *settings, ChiakiTar
 {
 	key_map = settings->GetControllerMappingForDecoding();
 	decoder = settings->GetDecoder();
-	hw_decode_engine = settings->GetHardwareDecodeEngine();
+	hw_decoder = settings->GetHardwareDecoder();
 	audio_out_device = settings->GetAudioOutDevice();
 	log_level_mask = settings->GetLogLevelMask();
 	log_file = CreateLogFilename();
@@ -35,16 +35,16 @@ StreamSessionConnectInfo::StreamSessionConnectInfo(Settings *settings, ChiakiTar
 
 static void AudioSettingsCb(uint32_t channels, uint32_t rate, void *user);
 static void AudioFrameCb(int16_t *buf, size_t samples_count, void *user);
-static bool VideoSampleCb(uint8_t *buf, size_t buf_size, void *user);
 static void EventCb(ChiakiEvent *event, void *user);
 #if CHIAKI_GUI_ENABLE_SETSU
 static void SessionSetsuCb(SetsuEvent *event, void *user);
 #endif
+static void FfmpegFrameCb(ChiakiFfmpegDecoder *decoder, void *user);
 
 StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObject *parent)
 	: QObject(parent),
 	log(this, connect_info.log_level_mask, connect_info.log_file),
-	video_decoder(nullptr),
+	ffmpeg_decoder(nullptr),
 #if CHIAKI_LIB_ENABLE_PI_DECODER
 	pi_decoder(nullptr),
 #endif
@@ -63,7 +63,22 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
 	else
 	{
 #endif
-		video_decoder = new VideoDecoder(connect_info.hw_decode_engine, log.GetChiakiLog());
+		ffmpeg_decoder = new ChiakiFfmpegDecoder;
+		ChiakiLogSniffer sniffer;
+		chiaki_log_sniffer_init(&sniffer, CHIAKI_LOG_ALL, GetChiakiLog());
+		ChiakiErrorCode err = chiaki_ffmpeg_decoder_init(ffmpeg_decoder,
+				chiaki_log_sniffer_get_log(&sniffer),
+				connect_info.video_profile.codec,
+				connect_info.hw_decoder.isEmpty() ? NULL : connect_info.hw_decoder.toUtf8().constData(),
+				FfmpegFrameCb, this);
+		if(err != CHIAKI_ERR_SUCCESS)
+		{
+			QString log = QString::fromUtf8(chiaki_log_sniffer_get_buffer(&sniffer));
+			chiaki_log_sniffer_fini(&sniffer);
+			throw ChiakiException("Failed to initialize FFMPEG Decoder:\n" + log);
+		}
+		chiaki_log_sniffer_fini(&sniffer);
+		ffmpeg_decoder->log = GetChiakiLog();
 #if CHIAKI_LIB_ENABLE_PI_DECODER
 	}
 #endif
@@ -101,7 +116,7 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
 
 	chiaki_controller_state_set_idle(&keyboard_state);
 
-	ChiakiErrorCode err = chiaki_session_init(&session, &chiaki_connect_info, log.GetChiakiLog());
+	err = chiaki_session_init(&session, &chiaki_connect_info, GetChiakiLog());
 	if(err != CHIAKI_ERR_SUCCESS)
 		throw ChiakiException("Chiaki Session Init failed: " + QString::fromLocal8Bit(chiaki_error_string(err)));
 
@@ -116,7 +131,7 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
 	else
 	{
 #endif
-		chiaki_session_set_video_sample_cb(&session, VideoSampleCb, this);
+		chiaki_session_set_video_sample_cb(&session, chiaki_ffmpeg_decoder_video_sample_cb, ffmpeg_decoder);
 #if CHIAKI_LIB_ENABLE_PI_DECODER
 	}
 #endif
@@ -160,7 +175,11 @@ StreamSession::~StreamSession()
 		free(pi_decoder);
 	}
 #endif
-	delete video_decoder;
+	if(ffmpeg_decoder)
+	{
+		chiaki_ffmpeg_decoder_fini(ffmpeg_decoder);
+		delete ffmpeg_decoder;
+	}
 }
 
 void StreamSession::Start()
@@ -346,11 +365,6 @@ void StreamSession::PushAudioFrame(int16_t *buf, size_t samples_count)
 	audio_io->write((const char *)buf, static_cast<qint64>(samples_count * 2 * 2));
 }
 
-void StreamSession::PushVideoSample(uint8_t *buf, size_t buf_size)
-{
-	video_decoder->PushFrame(buf, buf_size);
-}
-
 void StreamSession::Event(ChiakiEvent *event)
 {
 	switch(event->type)
@@ -431,6 +445,11 @@ void StreamSession::HandleSetsuEvent(SetsuEvent *event)
 }
 #endif
 
+void StreamSession::TriggerFfmpegFrameAvailable()
+{
+	emit FfmpegFrameAvailable();
+}
+
 class StreamSessionPrivate
 {
 	public:
@@ -440,11 +459,11 @@ class StreamSessionPrivate
 		}
 
 		static void PushAudioFrame(StreamSession *session, int16_t *buf, size_t samples_count)	{ session->PushAudioFrame(buf, samples_count); }
-		static void PushVideoSample(StreamSession *session, uint8_t *buf, size_t buf_size)		{ session->PushVideoSample(buf, buf_size); }
 		static void Event(StreamSession *session, ChiakiEvent *event)							{ session->Event(event); }
 #if CHIAKI_GUI_ENABLE_SETSU
 		static void HandleSetsuEvent(StreamSession *session, SetsuEvent *event)					{ session->HandleSetsuEvent(event); }
 #endif
+		static void TriggerFfmpegFrameAvailable(StreamSession *session)							{ session->TriggerFfmpegFrameAvailable(); }
 };
 
 static void AudioSettingsCb(uint32_t channels, uint32_t rate, void *user)
@@ -459,13 +478,6 @@ static void AudioFrameCb(int16_t *buf, size_t samples_count, void *user)
 	StreamSessionPrivate::PushAudioFrame(session, buf, samples_count);
 }
 
-static bool VideoSampleCb(uint8_t *buf, size_t buf_size, void *user)
-{
-	auto session = reinterpret_cast<StreamSession *>(user);
-	StreamSessionPrivate::PushVideoSample(session, buf, buf_size);
-	return true;
-}
-
 static void EventCb(ChiakiEvent *event, void *user)
 {
 	auto session = reinterpret_cast<StreamSession *>(user);
@@ -479,3 +491,9 @@ static void SessionSetsuCb(SetsuEvent *event, void *user)
 	StreamSessionPrivate::HandleSetsuEvent(session, event);
 }
 #endif
+
+static void FfmpegFrameCb(ChiakiFfmpegDecoder *decoder, void *user)
+{
+	auto session = reinterpret_cast<StreamSession *>(user);
+	StreamSessionPrivate::TriggerFfmpegFrameAvailable(session);
+}

gui/src/streamwindow.cpp

@@ -47,9 +47,9 @@ void StreamWindow::Init()
 	connect(session, &StreamSession::SessionQuit, this, &StreamWindow::SessionQuit);
 	connect(session, &StreamSession::LoginPINRequested, this, &StreamWindow::LoginPINRequested);
 
-	if(session->GetVideoDecoder())
+	if(session->GetFfmpegDecoder())
 	{
-		av_widget = new AVOpenGLWidget(session->GetVideoDecoder(), this);
+		av_widget = new AVOpenGLWidget(session, this);
 		setCentralWidget(av_widget);
 	}
 	else

gui/src/videodecoder.cpp

@@ -1,177 +0,0 @@
-// SPDX-License-Identifier: LicenseRef-AGPL-3.0-only-OpenSSL
-
-#include <videodecoder.h>
-
-#include <libavcodec/avcodec.h>
-
-#include <QImage>
-
-VideoDecoder::VideoDecoder(HardwareDecodeEngine hw_decode_engine, ChiakiLog *log) : hw_decode_engine(hw_decode_engine), log(log)
-{
-	enum AVHWDeviceType type;
-	hw_device_ctx = nullptr;
-
-	#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
-	avcodec_register_all();
-	#endif
-	codec = avcodec_find_decoder(AV_CODEC_ID_H264);
-	if(!codec)
-		throw VideoDecoderException("H264 Codec not available");
-
-	codec_context = avcodec_alloc_context3(codec);
-	if(!codec_context)
-		throw VideoDecoderException("Failed to alloc codec context");
-
-	if(hw_decode_engine)
-	{
-		if(!hardware_decode_engine_names.contains(hw_decode_engine))
-			throw VideoDecoderException("Unknown hardware decode engine!");
-
-		const char *hw_dec_eng = hardware_decode_engine_names[hw_decode_engine];
-		CHIAKI_LOGI(log, "Using hardware decode %s", hw_dec_eng);
-		type = av_hwdevice_find_type_by_name(hw_dec_eng);
-		if (type == AV_HWDEVICE_TYPE_NONE)
-			throw VideoDecoderException("Can't initialize vaapi");
-
-		for(int i = 0;; i++) {
-			const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);
-			if(!config)
-				throw VideoDecoderException("avcodec_get_hw_config failed");
-			if(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
-				config->device_type == type)
-			{
-				hw_pix_fmt = config->pix_fmt;
-				break;
-			}
-		}
-
-		if(av_hwdevice_ctx_create(&hw_device_ctx, type, NULL, NULL, 0) < 0)
-			throw VideoDecoderException("Failed to create hwdevice context");
-		codec_context->hw_device_ctx = av_buffer_ref(hw_device_ctx);
-	}
-
-	if(avcodec_open2(codec_context, codec, nullptr) < 0)
-	{
-		avcodec_free_context(&codec_context);
-		throw VideoDecoderException("Failed to open codec context");
-	}
-}
-
-VideoDecoder::~VideoDecoder()
-{
-	avcodec_close(codec_context);
-	avcodec_free_context(&codec_context);
-	if(hw_device_ctx)
-	{
-		av_buffer_unref(&hw_device_ctx);
-	}
-}
-
-void VideoDecoder::PushFrame(uint8_t *buf, size_t buf_size)
-{
-	{
-		QMutexLocker locker(&mutex);
-
-		AVPacket packet;
-		av_init_packet(&packet);
-		packet.data = buf;
-		packet.size = buf_size;
-		int r;
-send_packet:
-		r = avcodec_send_packet(codec_context, &packet);
-		if(r != 0)
-		{
-			if(r == AVERROR(EAGAIN))
-			{
-				CHIAKI_LOGE(log, "AVCodec internal buffer is full removing frames before pushing");
-				AVFrame *frame = av_frame_alloc();
-				if(!frame)
-				{
-					CHIAKI_LOGE(log, "Failed to alloc AVFrame");
-					return;
-				}
-				r = avcodec_receive_frame(codec_context, frame);
-				av_frame_free(&frame);
-				if(r != 0)
-				{
-					CHIAKI_LOGE(log, "Failed to pull frame");
-					return;
-				}
-				goto send_packet;
-			}
-			else
-			{
-				char errbuf[128];
-				av_make_error_string(errbuf, sizeof(errbuf), r);
-				CHIAKI_LOGE(log, "Failed to push frame: %s", errbuf);
-				return;
-			}
-		}
-	}
-
-	emit FramesAvailable();
-}
-
-AVFrame *VideoDecoder::PullFrame()
-{
-	QMutexLocker locker(&mutex);
-
-	// always try to pull as much as possible and return only the very last frame
-	AVFrame *frame_last = nullptr;
-	AVFrame *sw_frame = nullptr;
-	AVFrame *frame = nullptr; 
-	while(true)
-	{
-		AVFrame *next_frame;
-		if(frame_last)
-		{
-			av_frame_unref(frame_last);
-			next_frame = frame_last;
-		}
-		else
-		{
-			next_frame = av_frame_alloc();
-			if(!next_frame)
-				return frame;
-		}
-		frame_last = frame;
-		frame = next_frame;
-		int r = avcodec_receive_frame(codec_context, frame);
-		if(r == 0)
-		{
-			frame = hw_decode_engine ? GetFromHardware(frame) : frame;
-		}
-		else
-		{
-			if(r != AVERROR(EAGAIN))
-				CHIAKI_LOGE(log, "Decoding with FFMPEG failed");
-			av_frame_free(&frame);
-			return frame_last;
-		}
-	}
-}
-
-AVFrame *VideoDecoder::GetFromHardware(AVFrame *hw_frame)
-{
-	AVFrame *frame;
-	AVFrame *sw_frame;
-
-	sw_frame = av_frame_alloc();
-
-	int ret = av_hwframe_transfer_data(sw_frame, hw_frame, 0);
-
-	if(ret < 0)
-	{
-		CHIAKI_LOGE(log, "Failed to transfer frame from hardware");
-	}
-
-	av_frame_unref(hw_frame);
-
-	if(sw_frame->width <= 0)
-	{
-		av_frame_unref(sw_frame);
-		return nullptr;
-	}
-
-	return sw_frame;
-}

lib/CMakeLists.txt

@@ -75,6 +75,12 @@ set(SOURCE_FILES
 		src/regist.c
 		src/opusdecoder.c)
 
+if(CHIAKI_ENABLE_FFMPEG_DECODER)
+	list(APPEND HEADER_FILES include/chiaki/ffmpegdecoder.h)
+	list(APPEND SOURCE_FILES src/ffmpegdecoder.c)
+endif()
+set(CHIAKI_LIB_ENABLE_PI_DECODER "${CHIAKI_ENABLE_FFMPEG_DECODER}")
+
 if(CHIAKI_ENABLE_PI_DECODER)
 	list(APPEND HEADER_FILES include/chiaki/pidecoder.h)
 	list(APPEND SOURCE_FILES src/pidecoder.c)
@@ -127,6 +133,10 @@ endif()
 target_link_libraries(chiaki-lib Nanopb::nanopb)
 target_link_libraries(chiaki-lib Jerasure::Jerasure)
 
+if(CHIAKI_ENABLE_FFMPEG_DECODER)
+	target_link_libraries(chiaki-lib FFMPEG::avcodec FFMPEG::avutil)
+endif()
+
 if(CHIAKI_ENABLE_PI_DECODER)
 	target_link_libraries(chiaki-lib ILClient::ILClient)
 endif()

lib/include/chiaki/common.h

@@ -100,6 +100,8 @@ static inline bool chiaki_codec_is_hdr(ChiakiCodec codec)
 	return codec == CHIAKI_CODEC_H265_HDR;
 }
 
+CHIAKI_EXPORT const char *chiaki_codec_name(ChiakiCodec codec);
+
 #ifdef __cplusplus
 }
 #endif

lib/include/chiaki/ffmpegdecoder.h

@@ -0,0 +1,43 @@
+#ifndef CHIAKI_FFMPEG_DECODER_H
+#define CHIAKI_FFMPEG_DECODER_H
+
+#include <chiaki/config.h>
+#include <chiaki/log.h>
+#include <chiaki/thread.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <libavcodec/avcodec.h>
+
+typedef struct chiaki_ffmpeg_decoder_t ChiakiFfmpegDecoder;
+
+typedef void (*ChiakiFfmpegFrameAvailable)(ChiakiFfmpegDecoder *decover, void *user);
+
+struct chiaki_ffmpeg_decoder_t
+{
+	ChiakiLog *log;
+	ChiakiMutex mutex;
+	AVCodec *av_codec;
+	AVCodecContext *codec_context;
+	enum AVPixelFormat hw_pix_fmt;
+	AVBufferRef *hw_device_ctx;
+	ChiakiMutex cb_mutex;
+	ChiakiFfmpegFrameAvailable frame_available_cb;
+	void *frame_available_cb_user;
+};
+
+CHIAKI_EXPORT ChiakiErrorCode chiaki_ffmpeg_decoder_init(ChiakiFfmpegDecoder *decoder, ChiakiLog *log,
+		ChiakiCodec codec, const char *hw_decoder_name,
+		ChiakiFfmpegFrameAvailable frame_available_cb, void *frame_available_cb_user);
+CHIAKI_EXPORT void chiaki_ffmpeg_decoder_fini(ChiakiFfmpegDecoder *decoder);
+CHIAKI_EXPORT bool chiaki_ffmpeg_decoder_video_sample_cb(uint8_t *buf, size_t buf_size, void *user);
+CHIAKI_EXPORT AVFrame *chiaki_ffmpeg_decoder_pull_frame(ChiakiFfmpegDecoder *decoder);
+CHIAKI_EXPORT enum AVPixelFormat chiaki_ffmpeg_decoder_get_pixel_format(ChiakiFfmpegDecoder *decoder);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // CHIAKI_FFMPEG_DECODER_H

lib/include/chiaki/log.h

@@ -50,6 +50,20 @@ CHIAKI_EXPORT void chiaki_log_hexdump_raw(ChiakiLog *log, ChiakiLogLevel level,
 #define CHIAKI_LOGW(log, ...) do { chiaki_log((log), CHIAKI_LOG_WARNING, __VA_ARGS__); } while(0)
 #define CHIAKI_LOGE(log, ...) do { chiaki_log((log), CHIAKI_LOG_ERROR, __VA_ARGS__); } while(0)
 
+typedef struct chiaki_log_sniffer_t
+{
+	ChiakiLog *forward_log; // The original log, where everything is forwarded
+	ChiakiLog sniff_log; // The log where others will log into
+	uint32_t sniff_level_mask;
+	char *buf; // always null-terminated
+	size_t buf_len; // strlen(buf)
+} ChiakiLogSniffer;
+
+CHIAKI_EXPORT void chiaki_log_sniffer_init(ChiakiLogSniffer *sniffer, uint32_t level_mask, ChiakiLog *forward_log);
+CHIAKI_EXPORT void chiaki_log_sniffer_fini(ChiakiLogSniffer *sniffer);
+static inline ChiakiLog *chiaki_log_sniffer_get_log(ChiakiLogSniffer *sniffer) { return &sniffer->sniff_log; }
+static inline const char *chiaki_log_sniffer_get_buffer(ChiakiLogSniffer *sniffer) { return sniffer->buf; }
+
 #ifdef __cplusplus
 }
 #endif

lib/src/common.c

@@ -105,3 +105,18 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_lib_init()
 
 	return CHIAKI_ERR_SUCCESS;
 }
+
+CHIAKI_EXPORT const char *chiaki_codec_name(ChiakiCodec codec)
+{
+	switch(codec)
+	{
+		case CHIAKI_CODEC_H264:
+			return "H264";
+		case CHIAKI_CODEC_H265:
+			return "H265";
+		case CHIAKI_CODEC_H265_HDR:
+			return "H265/HDR";
+		default:
+			return "unknown";
+	}
+}

lib/src/ffmpegdecoder.c

@@ -0,0 +1,216 @@
+
+#include <chiaki/ffmpegdecoder.h>
+
+#include <libavcodec/avcodec.h>
+
+static enum AVCodecID chiaki_codec_av_codec_id(ChiakiCodec codec)
+{
+	switch(codec)
+	{
+		case CHIAKI_CODEC_H265:
+		case CHIAKI_CODEC_H265_HDR:
+			return AV_CODEC_ID_H265;
+		default:
+			return AV_CODEC_ID_H264;
+	}
+}
+
+CHIAKI_EXPORT ChiakiErrorCode chiaki_ffmpeg_decoder_init(ChiakiFfmpegDecoder *decoder, ChiakiLog *log,
+		ChiakiCodec codec, const char *hw_decoder_name,
+		ChiakiFfmpegFrameAvailable frame_available_cb, void *frame_available_cb_user)
+{
+	decoder->log = log;
+	decoder->frame_available_cb = frame_available_cb;
+	decoder->frame_available_cb_user = frame_available_cb_user;
+
+	ChiakiErrorCode err = chiaki_mutex_init(&decoder->mutex, false);
+	if(err != CHIAKI_ERR_SUCCESS)
+		return err;
+
+	decoder->hw_device_ctx = NULL;
+	decoder->hw_pix_fmt = AV_PIX_FMT_NONE;
+
+#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
+	avcodec_register_all();
+#endif
+	enum AVCodecID av_codec = chiaki_codec_av_codec_id(codec);
+	decoder->av_codec = avcodec_find_decoder(av_codec);
+	if(!decoder->av_codec)
+	{
+		CHIAKI_LOGE(log, "%s Codec not available", chiaki_codec_name(codec));
+		goto error_mutex;
+	}
+
+	decoder->codec_context = avcodec_alloc_context3(decoder->av_codec);
+	if(!decoder->codec_context)
+	{
+		CHIAKI_LOGE(log, "Failed to alloc codec context");
+		goto error_mutex;
+	}
+
+	if(hw_decoder_name)
+	{
+		CHIAKI_LOGI(log, "Using hardware decoder \"%s\"", hw_decoder_name);
+		enum AVHWDeviceType type = av_hwdevice_find_type_by_name(hw_decoder_name);
+		if(type == AV_HWDEVICE_TYPE_NONE)
+		{
+			CHIAKI_LOGE(log, "Hardware decoder \"%s\" not found", hw_decoder_name);
+			goto error_codec_context;
+		}
+
+		for(int i = 0;; i++)
+		{
+			const AVCodecHWConfig *config = avcodec_get_hw_config(decoder->av_codec, i);
+			if(!config)
+			{
+				CHIAKI_LOGE(log, "avcodec_get_hw_config failed");
+				goto error_codec_context;
+			}
+			if(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type)
+			{
+				decoder->hw_pix_fmt = config->pix_fmt;
+				break;
+			}
+		}
+
+		if(av_hwdevice_ctx_create(&decoder->hw_device_ctx, type, NULL, NULL, 0) < 0)
+		{
+			CHIAKI_LOGE(log, "Failed to create hwdevice context");
+			goto error_codec_context;
+		}
+		decoder->codec_context->hw_device_ctx = av_buffer_ref(decoder->hw_device_ctx);
+	}
+
+	if(avcodec_open2(decoder->codec_context, decoder->av_codec, NULL) < 0)
+	{
+		CHIAKI_LOGE(log, "Failed to open codec context");
+		goto error_codec_context;
+	}
+
+	return CHIAKI_ERR_SUCCESS;
+error_codec_context:
+	if(decoder->hw_device_ctx)
+		av_buffer_unref(&decoder->hw_device_ctx);
+	avcodec_free_context(&decoder->codec_context);
+error_mutex:
+	chiaki_mutex_fini(&decoder->mutex);
+	return CHIAKI_ERR_UNKNOWN;
+}
+
+CHIAKI_EXPORT void chiaki_ffmpeg_decoder_fini(ChiakiFfmpegDecoder *decoder)
+{
+	avcodec_close(decoder->codec_context);
+	avcodec_free_context(&decoder->codec_context);
+	if(decoder->hw_device_ctx)
+		av_buffer_unref(&decoder->hw_device_ctx);
+}
+
+CHIAKI_EXPORT bool chiaki_ffmpeg_decoder_video_sample_cb(uint8_t *buf, size_t buf_size, void *user)
+{
+	ChiakiFfmpegDecoder *decoder = user;
+
+	chiaki_mutex_lock(&decoder->mutex);
+	AVPacket packet;
+	av_init_packet(&packet);
+	packet.data = buf;
+	packet.size = buf_size;
+	int r;
+send_packet:
+	r = avcodec_send_packet(decoder->codec_context, &packet);
+	if(r != 0)
+	{
+		if(r == AVERROR(EAGAIN))
+		{
+			CHIAKI_LOGE(decoder->log, "AVCodec internal buffer is full removing frames before pushing");
+			AVFrame *frame = av_frame_alloc();
+			if(!frame)
+			{
+				CHIAKI_LOGE(decoder->log, "Failed to alloc AVFrame");
+				goto hell;
+			}
+			r = avcodec_receive_frame(decoder->codec_context, frame);
+			av_frame_free(&frame);
+			if(r != 0)
+			{
+				CHIAKI_LOGE(decoder->log, "Failed to pull frame");
+				goto hell;
+			}
+			goto send_packet;
+		}
+		else
+		{
+			char errbuf[128];
+			av_make_error_string(errbuf, sizeof(errbuf), r);
+			CHIAKI_LOGE(decoder->log, "Failed to push frame: %s", errbuf);
+			goto hell;
+		}
+	}
+	chiaki_mutex_unlock(&decoder->mutex);
+
+	decoder->frame_available_cb(decoder, decoder->frame_available_cb_user);
+	return true;
+hell:
+	chiaki_mutex_unlock(&decoder->mutex);
+	return false;
+}
+
+static AVFrame *pull_from_hw(ChiakiFfmpegDecoder *decoder, AVFrame *hw_frame)
+{
+	AVFrame *sw_frame = av_frame_alloc();
+	if(av_hwframe_transfer_data(sw_frame, hw_frame, 0) < 0)
+	{
+		CHIAKI_LOGE(decoder->log, "Failed to transfer frame from hardware");
+		av_frame_unref(sw_frame);
+		sw_frame = NULL;
+	}
+	av_frame_unref(hw_frame);
+	return sw_frame;
+}
+
+CHIAKI_EXPORT AVFrame *chiaki_ffmpeg_decoder_pull_frame(ChiakiFfmpegDecoder *decoder)
+{
+	chiaki_mutex_lock(&decoder->mutex);
+	// always try to pull as much as possible and return only the very last frame
+	AVFrame *frame_last = NULL;
+	AVFrame *frame = NULL;
+	while(true)
+	{
+		AVFrame *next_frame;
+		if(frame_last)
+		{
+			av_frame_unref(frame_last);
+			next_frame = frame_last;
+		}
+		else
+		{
+			next_frame = av_frame_alloc();
+			if(!next_frame)
+				break;
+		}
+		frame_last = frame;
+		frame = next_frame;
+		int r = avcodec_receive_frame(decoder->codec_context, frame);
+		if(!r)
+			frame = decoder->hw_device_ctx ? pull_from_hw(decoder, frame) : frame;
+		else
+		{
+			if(r != AVERROR(EAGAIN))
+				CHIAKI_LOGE(decoder->log, "Decoding with FFMPEG failed");
+			av_frame_free(&frame);
+			frame = frame_last;
+			break;
+		}
+	}
+	chiaki_mutex_unlock(&decoder->mutex);
+
+	return frame;
+}
+
+CHIAKI_EXPORT enum AVPixelFormat chiaki_ffmpeg_decoder_get_pixel_format(ChiakiFfmpegDecoder *decoder)
+{
+	// TODO: this is probably very wrong, especially for hdr
+	return decoder->hw_device_ctx
+		? AV_PIX_FMT_NV12
+		: AV_PIX_FMT_YUV420P;
+}
+

lib/src/log.c

@@ -4,6 +4,7 @@
 
 #include <stdio.h>
 #include <stdarg.h>
+#include <string.h>
 
 CHIAKI_EXPORT char chiaki_log_level_char(ChiakiLogLevel level)
 {
@@ -177,3 +178,50 @@ CHIAKI_EXPORT void chiaki_log_hexdump_raw(ChiakiLog *log, ChiakiLogLevel level,
 	chiaki_log(log, level, "%s", str);
 	free(str);
 }
+
+static void log_sniffer_cb(ChiakiLogLevel level, const char *msg, void *user);
+
+CHIAKI_EXPORT void chiaki_log_sniffer_init(ChiakiLogSniffer *sniffer, uint32_t level_mask, ChiakiLog *forward_log)
+{
+	sniffer->forward_log = forward_log;
+	// level_mask is applied later and everything is forwarded unmasked, so use ALL here:
+	chiaki_log_init(&sniffer->sniff_log, CHIAKI_LOG_ALL, log_sniffer_cb, sniffer);
+	sniffer->sniff_level_mask = level_mask;
+	sniffer->buf = calloc(1, 1);
+	sniffer->buf_len = '\0';
+}
+
+CHIAKI_EXPORT void chiaki_log_sniffer_fini(ChiakiLogSniffer *sniffer)
+{
+	free(sniffer->buf);
+}
+
+static void log_sniffer_push(ChiakiLogSniffer *sniffer, ChiakiLogLevel level, const char *msg)
+{
+	size_t len = strlen(msg);
+	if(!len)
+		return;
+	bool nl = sniffer->buf_len != 0;
+	char *new_buf = realloc(sniffer->buf, sniffer->buf_len + (nl ? 1 : 0) + 4 + len + 1);
+	if(!new_buf)
+		return;
+	sniffer->buf = new_buf;
+	if(nl)
+		sniffer->buf[sniffer->buf_len++] = '\n';
+	sniffer->buf[sniffer->buf_len++] = '[';
+	sniffer->buf[sniffer->buf_len++] = chiaki_log_level_char(level);
+	sniffer->buf[sniffer->buf_len++] = ']';
+	sniffer->buf[sniffer->buf_len++] = ' ';
+	memcpy(sniffer->buf + sniffer->buf_len, msg, len);
+	sniffer->buf_len += len;
+	sniffer->buf[sniffer->buf_len] = '\0';
+}
+
+static void log_sniffer_cb(ChiakiLogLevel level, const char *msg, void *user)
+{
+	ChiakiLogSniffer *sniffer = user;
+	if(level & sniffer->sniff_level_mask)
+		log_sniffer_push(sniffer, level, msg);
+	if(sniffer->forward_log)
+		chiaki_log(sniffer->forward_log, level, "%s", msg);
+}