Add AVOpenGLWidget

gui/CMakeLists.txt

@@ -5,7 +5,7 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTOUIC ON)
 set(CMAKE_AUTORCC ON)
-find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui Multimedia)
+find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui Multimedia OpenGL)
 if(CHIAKI_GUI_ENABLE_QT_GAMEPAD)
 	find_package(Qt5 REQUIRED COMPONENTS Gamepad)
 endif()
@@ -29,12 +29,14 @@ add_executable(chiaki
 		include/discoverymanager.h
 		src/discoverymanager.cpp
 		include/streamsession.h
-		src/streamsession.cpp)
+		src/streamsession.cpp
+		include/avopenglwidget.h
+		src/avopenglwidget.cpp)
 target_include_directories(chiaki PRIVATE include)
 
 target_link_libraries(chiaki chiaki-lib)
 target_link_libraries(chiaki FFMPEG::avcodec)
-target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::Multimedia)
+target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::Multimedia Qt5::OpenGL)
 if(CHIAKI_GUI_ENABLE_QT_GAMEPAD)
 	target_link_libraries(chiaki Qt5::Gamepad)
 	target_compile_definitions(chiaki PRIVATE CHIAKI_GUI_ENABLE_QT_GAMEPAD)

gui/include/avopenglwidget.h

@@ -0,0 +1,54 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef CHIAKI_AVOPENGLWIDGET_H
+#define CHIAKI_AVOPENGLWIDGET_H
+
+#include <QOpenGLWidget>
+
+extern "C"
+{
+#include <libavcodec/avcodec.h>
+}
+
+class VideoDecoder;
+
+class AVOpenGLWidget: public QOpenGLWidget
+{
+	Q_OBJECT
+
+	private:
+		VideoDecoder *decoder;
+
+		GLuint program;
+		GLuint vbo;
+		GLuint vao;
+		GLuint tex[3];
+		unsigned int frame_width, frame_height;
+
+		void UpdateTextures(AVFrame *frame);
+
+	public:
+		explicit AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent = nullptr);
+
+	protected:
+		void initializeGL() override;
+		void resizeGL(int w, int h) override;
+		void paintGL() override;
+};
+
+#endif // CHIAKI_AVOPENGLWIDGET_H

gui/include/streamwindow.h

@@ -23,6 +23,7 @@
 #include "streamsession.h"
 
 class QLabel;
+class AVOpenGLWidget;
 
 class StreamWindow: public QMainWindow
 {
@@ -30,14 +31,12 @@ class StreamWindow: public QMainWindow
 
 	public:
 		explicit StreamWindow(const StreamSessionConnectInfo &connect_info, QWidget *parent = nullptr);
-		~StreamWindow();
+		~StreamWindow() override;
 
 	private:
 		StreamSession *session;
 
-		QLabel *imageLabel;
-
-		void SetImage(const QImage &image);
+		AVOpenGLWidget *av_widget;
 
 	protected:
 		void keyPressEvent(QKeyEvent *event) override;
@@ -45,7 +44,6 @@ class StreamWindow: public QMainWindow
 		void closeEvent(QCloseEvent *event) override;
 
 	private slots:
-		void FramesAvailable();
 		void SessionQuit(ChiakiQuitReason reason, const QString &reason_str);
 };
 

gui/include/videodecoder.h

@@ -24,7 +24,7 @@
 extern "C"
 {
 #include <libavcodec/avcodec.h>
-};
+}
 
 #include <cstdint>
 
@@ -36,8 +36,8 @@ class VideoDecoder: public QObject
 		VideoDecoder();
 		~VideoDecoder();
 
-		void PutFrame(uint8_t *buf, size_t buf_size);
-		QImage PullFrame();
+		void PushFrame(uint8_t *buf, size_t buf_size);
+		AVFrame *PullFrame();
 
 	signals:
 		void FramesAvailable();

gui/src/avopenglwidget.cpp

@@ -0,0 +1,242 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <avopenglwidget.h>
+#include <videodecoder.h>
+
+#include <QOpenGLContext>
+#include <QOpenGLExtraFunctions>
+#include <QOpenGLDebugLogger>
+
+//#define DEBUG_OPENGL
+
+static const char *shader_vert_glsl = R"glsl(
+#version 150 core
+
+in vec2 pos_attr;
+
+out vec2 uv_var;
+
+void main()
+{
+	uv_var = pos_attr;
+	gl_Position = vec4(pos_attr * vec2(2.0, -2.0) + vec2(-1.0, 1.0), 0.0, 1.0);
+}
+)glsl";
+
+static const char *shader_frag_glsl = R"glsl(
+#version 150 core
+
+uniform sampler2D tex_y;
+uniform sampler2D tex_u;
+uniform sampler2D tex_v;
+
+in vec2 uv_var;
+
+out vec4 out_color;
+
+void main()
+{
+	vec3 yuv = vec3(
+		texture2D(tex_y, uv_var).r,
+		texture2D(tex_u, uv_var).r - 0.5,
+		texture2D(tex_v, uv_var).r - 0.5);
+	vec3 rgb = mat3(
+		1.0,		1.0,		1.0,
+		0.0,		-0.39393,	2.02839,
+		1.14025,	-0.58081,	0.0) * yuv;
+	out_color = vec4(rgb, 1.0);
+}
+)glsl";
+
+static const float vert_pos[] = {
+	0.0f, 0.0f,
+	0.0f, 1.0f,
+	1.0f, 0.0f,
+	1.0f, 1.0f
+};
+
+AVOpenGLWidget::AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent)
+	: QOpenGLWidget(parent),
+	decoder(decoder)
+{
+	QSurfaceFormat format;
+	format.setDepthBufferSize(0);
+	format.setStencilBufferSize(0);
+	format.setVersion(3, 2);
+	format.setProfile(QSurfaceFormat::CoreProfile);
+#ifdef DEBUG_OPENGL
+	format.setOption(QSurfaceFormat::DebugContext, true);
+#endif
+	setFormat(format);
+
+	frame_width = 0;
+	frame_height = 0;
+
+	connect(decoder, SIGNAL(FramesAvailable()), this, SLOT(update()));
+}
+
+void AVOpenGLWidget::initializeGL()
+{
+	auto f = QOpenGLContext::currentContext()->extraFunctions();
+
+#ifdef DEBUG_OPENGL
+	auto logger = new QOpenGLDebugLogger(this);
+	logger->initialize();
+	connect(logger, &QOpenGLDebugLogger::messageLogged, this, [](const QOpenGLDebugMessage &msg) {
+		qDebug() << msg;
+	});
+	logger->startLogging();
+#endif
+
+	GLuint shader_vert = f->glCreateShader(GL_VERTEX_SHADER);
+	f->glShaderSource(shader_vert, 1, &shader_vert_glsl, nullptr);
+	f->glCompileShader(shader_vert);
+
+	GLuint shader_frag = f->glCreateShader(GL_FRAGMENT_SHADER);
+	f->glShaderSource(shader_frag, 1, &shader_frag_glsl, nullptr);
+	f->glCompileShader(shader_frag);
+
+	program = f->glCreateProgram();
+	f->glAttachShader(program, shader_vert);
+	f->glAttachShader(program, shader_frag);
+	f->glBindAttribLocation(program, 0, "pos_attr");
+	f->glLinkProgram(program);
+
+	GLint linked = 0;
+	f->glGetProgramiv(program, GL_LINK_STATUS, &linked);
+	if(linked != GL_TRUE)
+	{
+		GLint info_log_size = 0;
+		f->glGetProgramiv(program, GL_INFO_LOG_LENGTH, &info_log_size);
+		std::vector<GLchar> info_log(info_log_size);
+		f->glGetProgramInfoLog(program, info_log_size, &info_log_size, info_log.data());
+		f->glDeleteProgram(program);
+		// TODO: log to somewhere else
+		printf("Failed to Link Shader Program:\n%s\n", info_log.data());
+		return;
+	}
+
+	f->glGenTextures(3, tex);
+	uint8_t uv_default = 127;
+	for(int i=0; i<3; i++)
+	{
+		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
+		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+		f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, i > 0 ? &uv_default : nullptr);
+	}
+
+	f->glUseProgram(program);
+	f->glUniform1i(f->glGetUniformLocation(program, "tex_y"), 0);
+	f->glUniform1i(f->glGetUniformLocation(program, "tex_u"), 1);
+	f->glUniform1i(f->glGetUniformLocation(program, "tex_v"), 2);
+
+	f->glGenVertexArrays(1, &vao);
+	f->glBindVertexArray(vao);
+
+	f->glGenBuffers(1, &vbo);
+	f->glBindBuffer(GL_ARRAY_BUFFER, vbo);
+	f->glBufferData(GL_ARRAY_BUFFER, sizeof(vert_pos), vert_pos, GL_STATIC_DRAW);
+
+	f->glBindBuffer(GL_ARRAY_BUFFER, vbo);
+	f->glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, nullptr);
+	f->glEnableVertexAttribArray(0);
+
+	f->glCullFace(GL_BACK);
+	f->glEnable(GL_CULL_FACE);
+	f->glClearColor(0.0, 0.0, 0.0, 1.0);
+}
+
+void AVOpenGLWidget::resizeGL(int w, int h)
+{
+}
+
+void AVOpenGLWidget::UpdateTextures(AVFrame *frame)
+{
+	auto f = QOpenGLContext::currentContext()->functions();
+
+	if(frame->format != AV_PIX_FMT_YUV420P)
+	{
+		// TODO: log to somewhere else
+		printf("Invalid Format\n");
+		return;
+	}
+
+	frame_width = frame->width;
+	frame_height = frame->height;
+
+	for(int i=0; i<3; i++)
+	{
+		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
+		int width = frame->width;
+		int height = frame->height;
+		if(i > 0)
+		{
+			width /= 2;
+			height /= 2;
+		}
+		f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, frame->data[i]);
+	}
+}
+
+void AVOpenGLWidget::paintGL()
+{
+	auto f = QOpenGLContext::currentContext()->extraFunctions();
+
+	f->glClear(GL_COLOR_BUFFER_BIT);
+
+	AVFrame *next_frame = decoder->PullFrame();
+	if(next_frame)
+	{
+		UpdateTextures(next_frame);
+		av_frame_free(&next_frame);
+	}
+
+	GLsizei vp_width, vp_height;
+	if(!frame_width || !frame_height)
+	{
+		vp_width = width();
+		vp_height = height();
+	}
+	else
+	{
+		float aspect = (float)frame_width / (float)frame_height;
+		if(aspect < (float)width() / (float)height())
+		{
+			vp_height = height();
+			vp_width = (GLsizei)(vp_height * aspect);
+		}
+		else
+		{
+			vp_width = width();
+			vp_height = (GLsizei)(vp_width / aspect);
+		}
+	}
+
+	f->glViewport((width() - vp_width) / 2, (height() - vp_height) / 2, vp_width, vp_height);
+
+	for(int i=0; i<3; i++)
+	{
+		f->glActiveTexture(GL_TEXTURE0 + i);
+		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
+	}
+
+	f->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+}

gui/src/main.cpp

@@ -71,8 +71,8 @@ int main(int argc, char *argv[])
 		connect_info.did = parser.value(did_option);
 
 		chiaki_connect_video_profile_preset(&connect_info.video_profile,
-				CHIAKI_VIDEO_RESOLUTION_PRESET_360p,
-				CHIAKI_VIDEO_FPS_PRESET_30);
+				CHIAKI_VIDEO_RESOLUTION_PRESET_720p,
+				CHIAKI_VIDEO_FPS_PRESET_60);
 
 		if(connect_info.registkey.isEmpty() || connect_info.ostype.isEmpty() || connect_info.auth.isEmpty() || connect_info.morning.isEmpty() || connect_info.did.isEmpty())
 			parser.showHelp(1);

gui/src/streamsession.cpp

@@ -249,7 +249,7 @@ void StreamSession::PushAudioFrame(int16_t *buf, size_t samples_count)
 
 void StreamSession::PushVideoSample(uint8_t *buf, size_t buf_size)
 {
-	video_decoder.PutFrame(buf, buf_size);
+	video_decoder.PushFrame(buf, buf_size);
 }
 
 void StreamSession::Event(ChiakiEvent *event)

gui/src/streamwindow.cpp

@@ -17,6 +17,7 @@
 
 #include <streamwindow.h>
 #include <streamsession.h>
+#include <avopenglwidget.h>
 
 #include <QLabel>
 #include <QMessageBox>
@@ -24,19 +25,12 @@
 StreamWindow::StreamWindow(const StreamSessionConnectInfo &connect_info, QWidget *parent)
 	: QMainWindow(parent)
 {
-	imageLabel = new QLabel(this);
-	setCentralWidget(imageLabel);
-
-	imageLabel->setBackgroundRole(QPalette::Base);
-	imageLabel->setSizePolicy(QSizePolicy::Ignored, QSizePolicy::Ignored);
-	imageLabel->setScaledContents(true);
-
 	session = new StreamSession(connect_info, this);
 
 	connect(session, &StreamSession::SessionQuit, this, &StreamWindow::SessionQuit);
 
-	connect(session->GetVideoDecoder(), &VideoDecoder::FramesAvailable, this, &StreamWindow::FramesAvailable);
-	FramesAvailable();
+	av_widget = new AVOpenGLWidget(session->GetVideoDecoder(), this);
+	setCentralWidget(av_widget);
 
 	grabKeyboard();
 
@@ -45,11 +39,8 @@ StreamWindow::StreamWindow(const StreamSessionConnectInfo &connect_info, QWidget
 
 StreamWindow::~StreamWindow()
 {
-}
-
-void StreamWindow::SetImage(const QImage &image)
-{
-	imageLabel->setPixmap(QPixmap::fromImage(image));
+	// make sure av_widget is always deleted before the session
+	delete av_widget;
 }
 
 void StreamWindow::keyPressEvent(QKeyEvent *event)
@@ -67,22 +58,6 @@ void StreamWindow::closeEvent(QCloseEvent *)
 	session->Stop();
 }
 
-void StreamWindow::FramesAvailable()
-{
-	QImage prev;
-	QImage image;
-	do
-	{
-		prev = image;
-		image = session->GetVideoDecoder()->PullFrame();
-	} while(!image.isNull());
-
-	if(!prev.isNull())
-	{
-		SetImage(prev);
-	}
-}
-
 void StreamWindow::SessionQuit(ChiakiQuitReason reason, const QString &reason_str)
 {
 	if(reason == CHIAKI_QUIT_REASON_STOPPED)

gui/src/videodecoder.cpp

@@ -39,7 +39,7 @@ VideoDecoder::~VideoDecoder()
 	// TODO: free codec?
 }
 
-void VideoDecoder::PutFrame(uint8_t *buf, size_t buf_size)
+void VideoDecoder::PushFrame(uint8_t *buf, size_t buf_size)
 {
 	{
 		QMutexLocker locker(&mutex);
@@ -54,48 +54,36 @@ void VideoDecoder::PutFrame(uint8_t *buf, size_t buf_size)
 	emit FramesAvailable();
 }
 
-QImage VideoDecoder::PullFrame()
+AVFrame *VideoDecoder::PullFrame()
 {
 	QMutexLocker locker(&mutex);
 
-	AVFrame *frame = av_frame_alloc(); // TODO: handle !frame
-	int r = avcodec_receive_frame(codec_context, frame);
-
-	if(r != 0)
+	// always try to pull as much as possible and return only the very last frame
+	AVFrame *frame_last = nullptr;
+	AVFrame *frame = nullptr;
+	while(true)
 	{
-		if(r != AVERROR(EAGAIN))
-			printf("decoding with ffmpeg failed!!\n");
-		av_frame_free(&frame);
-		return QImage();
-	}
-
-	switch(frame->format)
-	{
-		case AV_PIX_FMT_YUV420P:
-			break;
-		default:
-			printf("unknown format %d\n", frame->format);
-			av_frame_free(&frame);
-			return QImage();
-	}
-
-
-	QImage image(frame->width, frame->height, QImage::Format_RGB32);
-	for(int y=0; y<frame->height; y++)
-	{
-		for(int x=0; x<frame->width; x++)
+		AVFrame *next_frame;
+		if(frame_last)
 		{
-			int Y = frame->data[0][y * frame->linesize[0] + x] - 16;
-			int U = frame->data[1][(y/2) * frame->linesize[1] + (x/2)] - 128;
-			int V = frame->data[2][(y/2) * frame->linesize[2] + (x/2)] - 128;
-			int r = qBound(0, (298 * Y + 409 * V + 128) >> 8, 255);
-			int g = qBound(0, (298 * Y - 100 * U - 208 * V + 128) >> 8, 255);
-			int b = qBound(0, (298 * Y + 516 * U + 128) >> 8, 255);
-			image.setPixel(x, y, qRgb(r, g, b));
+			av_frame_unref(frame_last);
+			next_frame = frame_last;
+		}
+		else
+		{
+			next_frame = av_frame_alloc();
+			if(!next_frame)
+				return frame;
+		}
+		frame_last = frame;
+		frame = next_frame;
+		int r = avcodec_receive_frame(codec_context, frame);
+		if(r != 0)
+		{
+			if(r != AVERROR(EAGAIN))
+				printf("decoding with ffmpeg failed!!\n"); // TODO: log somewhere else
+			av_frame_free(&frame);
+			return frame_last;
 		}
 	}
-
-	av_frame_free(&frame);
-
-	return image;
-}
+}