Very dirty Video Display

gui/CMakeLists.txt

@@ -3,19 +3,18 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTOUIC ON)
 set(CMAKE_AUTORCC ON)
-find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui Multimedia OpenGL)
+find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui Multimedia)
 
-find_package(QtAV REQUIRED)
+find_package(FFMPEG REQUIRED COMPONENTS avcodec)
 
 add_executable(chiaki
 		src/main.cpp
 		include/streamwindow.h
-		src/streamwindow.cpp)
+		src/streamwindow.cpp
+		include/videodecoder.h
+		src/videodecoder.cpp)
 target_include_directories(chiaki PRIVATE include)
 
 target_link_libraries(chiaki chiaki-lib)
-
-target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::OpenGL Qt5::Multimedia)
-
-target_include_directories(chiaki PRIVATE ${QTAV_INCLUDE_DIRS} ${QTAVWIDGETS_INCLUDE_DIRS})
-target_link_libraries(chiaki ${QTAV_LIBRARIES} ${QTAVWIDGETS_LIBRARIES})
+target_link_libraries(chiaki FFMPEG::avcodec)
+target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::Multimedia)

gui/include/streamwindow.h

@@ -20,79 +20,7 @@
 
 #include <QMainWindow>
 
-namespace QtAV
-{
-	class VideoOutput;
-	class AVPlayer;
-}
-
-#include <QBuffer>
-#include <QMutex>
-#include <QThread>
-#include <QFile>
-
-class StreamRelayIODevice : public QIODevice
-{
-	private:
-		QMutex *mutex;
-		QByteArray buffer;
-
-	public:
-		explicit StreamRelayIODevice(QObject *parent = nullptr)
-			: QIODevice(parent),
-			mutex(new QMutex(QMutex::Recursive))
-		{
-			setOpenMode(OpenModeFlag::ReadOnly);
-		}
-
-		~StreamRelayIODevice() override = default;
-
-		void PushSample(uint8_t *buf, size_t size)
-		{
-			{
-				QMutexLocker locker(mutex);
-				printf("push sample %zu\n", size);
-				buffer.append(reinterpret_cast<const char *>(buf), static_cast<int>(size));
-			}
-			emit readyRead();
-		}
-
-
-		qint64 pos() const override { return 0; }
-		bool open(QIODevice::OpenMode mode) override { return true; }
-		bool isSequential() const override { return true; }
-
-		qint64 bytesAvailable() const override
-		{
-			QMutexLocker locker(mutex);
-			return buffer.size() + QIODevice::bytesAvailable();
-		}
-
-	protected:
-		qint64 readData(char *data, qint64 maxSize)
-		{
-			while(true)
-			{
-				{
-					QMutexLocker locker(mutex);
-					if(buffer.size() >= maxSize)
-					{
-						printf("read %lld\n", maxSize);
-						memcpy(data, buffer.constData(), maxSize);
-						buffer.remove(0, maxSize);
-						return maxSize;
-					}
-				}
-				QThread::msleep(100);
-			}
-		}
-
-		qint64 writeData(const char *data, qint64 maxSize)
-		{
-			return -1;
-		}
-};
-
+class QLabel;
 
 class StreamWindow: public QMainWindow
 {
@@ -102,14 +30,10 @@ class StreamWindow: public QMainWindow
 		explicit StreamWindow(QWidget *parent = nullptr);
 		~StreamWindow();
 
-		StreamRelayIODevice *GetIODevice()	{ return io_device; }
+		void SetImage(const QImage &image);
 
 	private:
-		QtAV::VideoOutput *video_output;
-		QtAV::AVPlayer *av_player;
-
-		StreamRelayIODevice *io_device;
-
+		QLabel *imageLabel;
 };
 
 #endif // CHIAKI_GUI_STREAMWINDOW_H

gui/include/videodecoder.h

@@ -0,0 +1,52 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef CHIAKI_VIDEODECODER_H
+#define CHIAKI_VIDEODECODER_H
+
+#include <QMutex>
+#include <QObject>
+
+extern "C"
+{
+#include <libavcodec/avcodec.h>
+};
+
+#include <cstdint>
+
+class VideoDecoder: public QObject
+{
+	Q_OBJECT
+
+	public:
+		VideoDecoder();
+		~VideoDecoder();
+
+		void PutFrame(uint8_t *buf, size_t buf_size);
+		QImage PullFrame();
+
+	signals:
+		void FramesAvailable();
+
+	private:
+		QMutex mutex;
+
+		AVCodec *codec;
+		AVCodecContext *codec_context;
+};
+
+#endif // CHIAKI_VIDEODECODER_H

gui/src/main.cpp

@@ -1,5 +1,6 @@
 
 #include <streamwindow.h>
+#include <videodecoder.h>
 
 #include <chiaki/session.h>
 #include <chiaki/base64.h>
@@ -15,8 +16,7 @@
 QAudioOutput *audio_out;
 QIODevice *audio_io;
 
-QFile *video_out_file;
-size_t file_size = 0;
+VideoDecoder video_decoder;
 
 
 void audio_frame_cb(int16_t *buf, size_t samples_count, void *user)
@@ -26,12 +26,13 @@ void audio_frame_cb(int16_t *buf, size_t samples_count, void *user)
 
 void video_sample_cb(uint8_t *buf, size_t buf_size, void *user)
 {
-	if(!video_out_file)
+	video_decoder.PutFrame(buf, buf_size);
+	/*if(!video_out_file)
 		return;
 	printf("writing %#zx to file, start: %#zx\n", buf_size, file_size);
 	chiaki_log_hexdump(nullptr, CHIAKI_LOG_DEBUG, buf, buf_size);
 	file_size += buf_size;
-	video_out_file->write((const char *)buf, buf_size);
+	video_out_file->write((const char *)buf, buf_size);*/
 	//StreamRelayIODevice *io_device = reinterpret_cast<StreamRelayIODevice *>(user);
 	//io_device->PushSample(buf, buf_size);
 }
@@ -98,13 +99,28 @@ int main(int argc, char *argv[])
 	audio_io = audio_out->start();
 
 
-	video_out_file = nullptr;
+	//video_out_file = nullptr;
 	//video_out_file->open(QFile::ReadWrite);
 
+	QObject::connect(&video_decoder, &VideoDecoder::FramesAvailable, &window, [&window]() {
+		QImage prev;
+		QImage image;
+		do
+		{
+			prev = image;
+			image = video_decoder.PullFrame();
+		} while(!image.isNull());
+
+		if(!prev.isNull())
+		{
+			window.SetImage(prev);
+		}
+	});
+
 	ChiakiSession session;
 	chiaki_session_init(&session, &connect_info);
-	chiaki_session_set_audio_frame_cb(&session, audio_frame_cb, NULL);
-	chiaki_session_set_video_sample_cb(&session, video_sample_cb, window.GetIODevice());
+	chiaki_session_set_audio_frame_cb(&session, audio_frame_cb, nullptr);
+	chiaki_session_set_video_sample_cb(&session, video_sample_cb, nullptr);
 	chiaki_session_start(&session);
 
 	app.setQuitOnLastWindowClosed(true);
@@ -113,7 +129,7 @@ int main(int argc, char *argv[])
 
 	//video_out_file->close();
 
-	printf("CLOSED!!! filesize: %zu\n", file_size);
+	//printf("CLOSED!!! filesize: %zu\n", file_size);
 
 	chiaki_session_join(&session);
 	chiaki_session_fini(&session);

gui/src/streamwindow.cpp

@@ -17,42 +17,25 @@
 
 #include <streamwindow.h>
 
-#include <QtAV>
-#include <QtAVWidgets>
-
-#include <QProcessEnvironment>
-
+#include <QLabel>
 
 StreamWindow::StreamWindow(QWidget *parent)
 	: QMainWindow(parent)
 {
-	video_output = new QtAV::VideoOutput(QtAV::VideoRendererId_GLWidget2, this);
-	setCentralWidget(video_output->widget());
+	imageLabel = new QLabel(this);
+	setCentralWidget(imageLabel);
 
-	av_player = new QtAV::AVPlayer(this);
+	imageLabel->setBackgroundRole(QPalette::Base);
+	imageLabel->setSizePolicy(QSizePolicy::Ignored, QSizePolicy::Ignored);
+	imageLabel->setScaledContents(true);
 
-	connect(av_player, &QtAV::AVPlayer::stateChanged, this, [](QtAV::AVPlayer::State state){
-		printf("state changed to %d\n", state);
-	});
-
-	av_player->setRenderer(video_output);
-
-	io_device = new StreamRelayIODevice(this);
-	io_device->open(QIODevice::ReadOnly);
-
-	QString test_filename = QProcessEnvironment::systemEnvironment().value("CHIAKI_TEST_VIDEO");
-	if(!test_filename.isEmpty())
-	{
-		QFile *test_file = new QFile(test_filename, this);
-		test_file->open(QIODevice::OpenModeFlag::ReadOnly);
-		QByteArray sample = test_file->readAll();
-		io_device->PushSample((uint8_t *)sample.constData(), (size_t)sample.size());
-		test_file->close();
-		av_player->setIODevice(io_device);
-		av_player->play();
-	}
 }
 
 StreamWindow::~StreamWindow()
 {
+}
+
+void StreamWindow::SetImage(const QImage &image)
+{
+	imageLabel->setPixmap(QPixmap::fromImage(image));
 }

gui/src/videodecoder.cpp

@@ -0,0 +1,101 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <videodecoder.h>
+
+#include <libavcodec/avcodec.h>
+
+#include <QImage>
+
+VideoDecoder::VideoDecoder()
+{
+	codec = avcodec_find_decoder(AV_CODEC_ID_H264); // TODO: handle !codec
+	codec_context = avcodec_alloc_context3(codec); // TODO: handle !codec_context
+
+	if(codec->capabilities & AV_CODEC_CAP_TRUNCATED)
+		codec_context->flags |= AV_CODEC_FLAG_TRUNCATED;
+
+	avcodec_open2(codec_context, codec, nullptr); // TODO: handle < 0
+}
+
+VideoDecoder::~VideoDecoder()
+{
+	avcodec_close(codec_context);
+	avcodec_free_context(&codec_context); // TODO: does close by itself too?
+	// TODO: free codec?
+}
+
+void VideoDecoder::PutFrame(uint8_t *buf, size_t buf_size)
+{
+	{
+		QMutexLocker locker(&mutex);
+
+		AVPacket packet;
+		av_init_packet(&packet);
+		packet.data = buf;
+		packet.size = buf_size;
+		avcodec_send_packet(codec_context, &packet);
+	}
+
+	emit FramesAvailable();
+}
+
+QImage VideoDecoder::PullFrame()
+{
+	QMutexLocker locker(&mutex);
+
+	AVFrame *frame = av_frame_alloc(); // TODO: handle !frame
+	int r = avcodec_receive_frame(codec_context, frame);
+
+	if(r != 0)
+	{
+		if(r != AVERROR(EAGAIN))
+			printf("decoding with ffmpeg failed!!\n");
+		av_frame_free(&frame);
+		return QImage();
+	}
+
+	switch(frame->format)
+	{
+		case AV_PIX_FMT_YUV420P:
+			break;
+		default:
+			printf("unknown format %d\n", frame->format);
+			av_frame_free(&frame);
+			return QImage();
+	}
+
+
+	QImage image(frame->width, frame->height, QImage::Format_RGB32);
+	for(int y=0; y<frame->height; y++)
+	{
+		for(int x=0; x<frame->width; x++)
+		{
+			int Y = frame->data[0][y * frame->linesize[0] + x] - 16;
+			int U = frame->data[1][(y/2) * frame->linesize[1] + (x/2)] - 128;
+			int V = frame->data[2][(y/2) * frame->linesize[2] + (x/2)] - 128;
+			int r = qBound(0, (298 * Y + 409 * V + 128) >> 8, 255);
+			int g = qBound(0, (298 * Y - 100 * U - 208 * V + 128) >> 8, 255);
+			int b = qBound(0, (298 * Y + 516 * U + 128) >> 8, 255);
+			image.setPixel(x, y, qRgb(r, g, b));
+		}
+	}
+
+	av_frame_free(&frame);
+
+	return image;
+}