Add Double-Buffering Frame Uploader

gui/CMakeLists.txt

@@ -31,7 +31,9 @@ add_executable(chiaki
 		include/streamsession.h
 		src/streamsession.cpp
 		include/avopenglwidget.h
-		src/avopenglwidget.cpp)
+		src/avopenglwidget.cpp
+		include/avopenglframeuploader.h
+		src/avopenglframeuploader.cpp)
 target_include_directories(chiaki PRIVATE include)
 
 target_link_libraries(chiaki chiaki-lib)

gui/include/avopenglframeuploader.h

@@ -0,0 +1,42 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef CHIAKI_AVOPENGLFRAMEUPLOADER_H
+#define CHIAKI_AVOPENGLFRAMEUPLOADER_H
+
+#include <QObject>
+#include <QOpenGLWidget>
+
+class AVOpenGLWidget;
+class VideoDecoder;
+
+class AVOpenGLFrameUploader: public QObject
+{
+	Q_OBJECT
+
+	private:
+		VideoDecoder *decoder;
+		AVOpenGLWidget *widget;
+
+	private slots:
+		void UpdateFrame();
+
+	public:
+		AVOpenGLFrameUploader(VideoDecoder *decoder, AVOpenGLWidget *widget);
+};
+
+#endif // CHIAKI_AVOPENGLFRAMEUPLOADER_H

gui/include/avopenglwidget.h

@@ -19,6 +19,7 @@
 #define CHIAKI_AVOPENGLWIDGET_H
 
 #include <QOpenGLWidget>
+#include <QMutex>
 
 extern "C"
 {
@@ -26,6 +27,16 @@ extern "C"
 }
 
 class VideoDecoder;
+class AVOpenGLFrameUploader;
+
+struct AVOpenGLFrame
+{
+	GLuint tex[3];
+	unsigned int width;
+	unsigned int height;
+
+	bool Update(AVFrame *frame);
+};
 
 class AVOpenGLWidget: public QOpenGLWidget
 {
@@ -37,13 +48,18 @@ class AVOpenGLWidget: public QOpenGLWidget
 		GLuint program;
 		GLuint vbo;
 		GLuint vao;
-		GLuint tex[3];
-		unsigned int frame_width, frame_height;
 
-		void UpdateTextures(AVFrame *frame);
+		AVOpenGLFrame frames[2];
+		int frame_fg;
+		QMutex frames_mutex;
+		AVOpenGLFrameUploader *frame_uploader;
 
 	public:
 		explicit AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent = nullptr);
+		~AVOpenGLWidget() override;
+
+		void SwapFrames();
+		AVOpenGLFrame *GetBackgroundFrame()	{ return &frames[1 - frame_fg]; }
 
 	protected:
 		void initializeGL() override;

gui/src/avopenglframeuploader.cpp

@@ -0,0 +1,45 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <avopenglframeuploader.h>
+#include <avopenglwidget.h>
+#include <videodecoder.h>
+
+#include <QOpenGLContext>
+#include <QOpenGLFunctions>
+
+AVOpenGLFrameUploader::AVOpenGLFrameUploader(VideoDecoder *decoder, AVOpenGLWidget *widget)
+	: QObject(nullptr),
+	decoder(decoder),
+	widget(widget)
+{
+
+	connect(decoder, SIGNAL(FramesAvailable()), this, SLOT(UpdateFrame()));
+}
+
+void AVOpenGLFrameUploader::UpdateFrame()
+{
+	AVFrame *next_frame = decoder->PullFrame();
+	if(!next_frame)
+		return;
+
+	bool success = widget->GetBackgroundFrame()->Update(next_frame);
+	av_frame_free(&next_frame);
+
+	if(success)
+		widget->SwapFrames();
+}

gui/src/avopenglwidget.cpp

@@ -17,6 +17,7 @@
 
 #include <avopenglwidget.h>
 #include <videodecoder.h>
+#include <avopenglframeuploader.h>
 
 #include <QOpenGLContext>
 #include <QOpenGLExtraFunctions>
@@ -84,10 +85,51 @@ AVOpenGLWidget::AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent)
 #endif
 	setFormat(format);
 
-	frame_width = 0;
+	frame_uploader = nullptr;
-	frame_height = 0;
+}
 
-	connect(decoder, SIGNAL(FramesAvailable()), this, SLOT(update()));
+AVOpenGLWidget::~AVOpenGLWidget()
+{
+	delete frame_uploader;
+}
+
+void AVOpenGLWidget::SwapFrames()
+{
+	QMutexLocker lock(&frames_mutex);
+	frame_fg = 1 - frame_fg;
+	QMetaObject::invokeMethod(this, "update");
+}
+
+bool AVOpenGLFrame::Update(AVFrame *frame)
+{
+	auto f = QOpenGLContext::currentContext()->functions();
+
+	if(frame->format != AV_PIX_FMT_YUV420P)
+	{
+		// TODO: log to somewhere else
+		printf("Invalid Format\n");
+		return false;
+	}
+
+	width = frame->width;
+	height = frame->height;
+
+	for(int i=0; i<3; i++)
+	{
+		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
+		int width = frame->width;
+		int height = frame->height;
+		if(i > 0)
+		{
+			width /= 2;
+			height /= 2;
+		}
+		f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, frame->data[i]);
+	}
+
+	f->glFinish(); // TODO: fence
+
+	return true;
 }
 
 void AVOpenGLWidget::initializeGL()
@@ -131,16 +173,21 @@ void AVOpenGLWidget::initializeGL()
 		return;
 	}
 
-	f->glGenTextures(3, tex);
+	for(int i=0; i<2; i++)
-	uint8_t uv_default = 127;
-	for(int i=0; i<3; i++)
 	{
-		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
+		f->glGenTextures(3, frames[i].tex);
-		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+		uint8_t uv_default = 127;
-		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+		for(int j=0; j<3; j++)
-		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+		{
-		f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+			f->glBindTexture(GL_TEXTURE_2D, frames[i].tex[j]);
-		f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, i > 0 ? &uv_default : nullptr);
+			f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+			f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+			f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+			f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+			f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, j > 0 ? &uv_default : nullptr);
+		}
+		frames[i].width = 0;
+		frames[i].height = 0;
 	}
 
 	f->glUseProgram(program);
@@ -162,62 +209,33 @@ void AVOpenGLWidget::initializeGL()
 	f->glCullFace(GL_BACK);
 	f->glEnable(GL_CULL_FACE);
 	f->glClearColor(0.0, 0.0, 0.0, 1.0);
+
+	frame_uploader = new AVOpenGLFrameUploader(decoder, this);
+	frame_fg = 0;
 }
 
 void AVOpenGLWidget::resizeGL(int w, int h)
 {
 }
 
-void AVOpenGLWidget::UpdateTextures(AVFrame *frame)
-{
-	auto f = QOpenGLContext::currentContext()->functions();
-
-	if(frame->format != AV_PIX_FMT_YUV420P)
-	{
-		// TODO: log to somewhere else
-		printf("Invalid Format\n");
-		return;
-	}
-
-	frame_width = frame->width;
-	frame_height = frame->height;
-
-	for(int i=0; i<3; i++)
-	{
-		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
-		int width = frame->width;
-		int height = frame->height;
-		if(i > 0)
-		{
-			width /= 2;
-			height /= 2;
-		}
-		f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, frame->data[i]);
-	}
-}
-
 void AVOpenGLWidget::paintGL()
 {
 	auto f = QOpenGLContext::currentContext()->extraFunctions();
 
 	f->glClear(GL_COLOR_BUFFER_BIT);
 
-	AVFrame *next_frame = decoder->PullFrame();
+	QMutexLocker lock(&frames_mutex);
-	if(next_frame)
+	AVOpenGLFrame *frame = &frames[frame_fg];
-	{
-		UpdateTextures(next_frame);
-		av_frame_free(&next_frame);
-	}
 
 	GLsizei vp_width, vp_height;
-	if(!frame_width || !frame_height)
+	if(!frame->width || !frame->height)
 	{
 		vp_width = width();
 		vp_height = height();
 	}
 	else
 	{
-		float aspect = (float)frame_width / (float)frame_height;
+		float aspect = (float)frame->width / (float)frame->height;
 		if(aspect < (float)width() / (float)height())
 		{
 			vp_height = height();
@@ -235,8 +253,10 @@ void AVOpenGLWidget::paintGL()
 	for(int i=0; i<3; i++)
 	{
 		f->glActiveTexture(GL_TEXTURE0 + i);
-		f->glBindTexture(GL_TEXTURE_2D, tex[i]);
+		f->glBindTexture(GL_TEXTURE_2D, frame->tex[i]);
 	}
 
 	f->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+	f->glFinish(); // TODO: fence
 }